Fix JWT_SECRET env var, setup database, update spider web and DAILY BUGLE colors to #e62020

2026-06-24 15:08:48 +02:00
commit 1e332fb50d
942 changed files with 447438 additions and 0 deletions
@@ -0,0 +1,100 @@
+const express = require('express');
+const { chromium } = require('playwright');
+
+const app = express();
+app.use(express.json());
+const PORT = process.env.PORT || 3007;
+
+app.get('/health', (req, res) => {
+  res.json({ status: 'ok' });
+});
+
+app.post('/scrape/indeed', async (req, res) => {
+  const results = [];
+  let browser;
+  try {
+    browser = await chromium.launch({ headless: true });
+    const context = await browser.newContext({
+      userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
+    });
+
+    const queries = [
+      'need a website built',
+      'looking for someone to build my website',
+      'need web designer',
+      'looking for web developer',
+      'need help with my website',
+      'need ecommerce website',
+      'need wordpress website',
+      'looking for website designer',
+    ];
+
+    for (const q of queries) {
+      const url = `https://www.indeed.com/jobs?q=${encodeURIComponent(q)}&sort=date`;
+      try {
+        const page = await context.newPage();
+        await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 20000 });
+        await page.waitForTimeout(2000);
+
+        const items = await page.evaluate(() => {
+          const cards = document.querySelectorAll('.job_seen_beacon');
+          return Array.from(cards).slice(0, 8).map(card => {
+            const titleEl = card.querySelector('.jcs-JobTitle');
+            const companyEl = card.querySelector('[data-testid="inlineHeader-companyName"]');
+            const snippetEl = card.querySelector('.job-snippet');
+            return {
+              title: titleEl?.textContent?.trim() || '',
+              url: titleEl?.href || '',
+              company: companyEl?.textContent?.trim() || '',
+              snippet: snippetEl?.textContent?.trim()?.slice(0, 200) || '',
+            };
+          });
+        });
+
+        for (const item of items) {
+          if (item.title && item.url && !results.some(r => r.url === item.url)) {
+            results.push({
+              title: item.title,
+              url: item.url,
+              author: item.company,
+              date: '',
+              content: item.snippet,
+              source: 'indeed',
+            });
+          }
+        }
+        await page.close();
+      } catch (e) {
+        console.error(`Indeed failed ${q}:`, e.message);
+      }
+    }
+
+    await browser.close();
+    res.json(results.slice(0, 50));
+  } catch (e) {
+    if (browser) await browser.close().catch(() => {});
+    console.error('Indeed error:', e.message);
+    res.status(500).json({ error: e.message });
+  }
+});
+
+app.post('/scrape/all', async (req, res) => {
+  try {
+    const [indeed] = await Promise.allSettled([
+      fetch(`http://localhost:${PORT}/scrape/indeed`, { method: 'POST' }).then(r => r.json()).catch(() => []),
+    ]);
+
+    const all = [
+      ...(indeed.status === 'fulfilled' ? indeed.value : []),
+    ];
+    console.log(`Scrape all: ${all.length} leads`);
+    res.json(all);
+  } catch (e) {
+    console.error('Scrape all error:', e.message);
+    res.status(500).json({ error: e.message });
+  }
+});
+
+app.listen(PORT, () => {
+  console.log(`Scraper service running on port ${PORT}`);
+});