Update .gitignore for browser-use-service generated files

- Introduced new migration for `facebook_accounts` and `facebook_scrape_logs` tables.
- Updated the scraping logic to handle Facebook accounts, including success and failure tracking.
- Implemented API endpoints for managing Facebook accounts and their scrape logs.
- Enhanced user permissions to restrict access to Facebook account management to admins and super admins.
- Added a dialog component for displaying and managing Facebook accounts in the UI.
- Updated lead fetching logic to include user role checks for assignment and access.

.gitattributes

@@ -0,0 +1 @@
+rust-ai/target/** linguist-generated=true -diff -merge

.gitignore

@@ -34,9 +34,33 @@ yarn-error.log*
 # env files (can opt-in for committing if needed)
 .env*
 
+# runtime data
+data/
+data/ai/
+data/ai/jobs.jsonl
+
+# logs
+*.log
+*.out
+error-log
+
 # vercel
 .vercel
 
+# python
+browser-use-service/venv/
+browser-use-service/__pycache__/
+browser-use-service/.env
+
 # typescript
 *.tsbuildinfo
 next-env.d.ts
+.git
+.git_bak
+node_modules
+target
+# rust build artifacts
+rust-ai/target/
+
+# browser-use-service generated files
+browser-use-service/*.txt

README.md

@@ -27,6 +27,14 @@ To learn more about Next.js, take a look at the following resources:
 - [Next.js Documentation](https://nextjs.org/docs) - learn about Next.js features and API.
 - [Learn Next.js](https://nextjs.org/learn) - an interactive Next.js tutorial.
 
+## Install
+- Jose
+- Node.js
+- Add emoji pakages
+- Install postgres for databases
+- Typescript
+
+
 You can check out [the Next.js GitHub repository](https://github.com/vercel/next.js) - your feedback and contributions are welcome!
 
 ## Deploy on Vercel

ai-server/index.mjs

@@ -0,0 +1,255 @@
+import http from "node:http"
+import fs from "node:fs"
+import path from "node:path"
+import { fileURLToPath } from "node:url"
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url))
+const ROOT = path.resolve(__dirname, "..")
+
+// ── Config from env ─────────────────────────────────────────────
+const PORT = parseInt(process.env.AI_PORT || "3001", 10)
+const HOST = process.env.AI_HOST || "0.0.0.0"
+const OLLAMA_URL = process.env.OLLAMA_BASE_URL || "http://localhost:11434"
+const MODEL = process.env.AI_MODEL || "sam860/dolphin3-llama3.2:3b"
+const DATABASE_URL = process.env.DATABASE_URL
+const JOBS_PATH = process.env.JOBS_PATH || path.join(ROOT, "data", "ai", "jobs.jsonl")
+const AI_MD_PATH = process.env.AI_MD_PATH || path.join(ROOT, "data", "ai", "ai.md")
+
+// ── Job loading ─────────────────────────────────────────────────
+function loadJobs() {
+  try {
+    const content = fs.readFileSync(JOBS_PATH, "utf-8")
+    const jobs = content
+      .split("\n")
+      .map((l) => l.trim())
+      .filter(Boolean)
+      .map((l) => {
+        try {
+          return JSON.parse(l)
+        } catch {
+          return null
+        }
+      })
+      .filter(Boolean)
+    console.log(`Loaded ${jobs.length} job categories`)
+    return jobs
+  } catch (e) {
+    console.error(`Failed to load jobs from ${JOBS_PATH}:`, e.message)
+    return []
+  }
+}
+
+// ── ai.md management ────────────────────────────────────────────
+function readInstructions() {
+  try {
+    return fs.readFileSync(AI_MD_PATH, "utf-8")
+  } catch {
+    return ""
+  }
+}
+
+function writeInstructions(content) {
+  fs.writeFileSync(AI_MD_PATH, content, "utf-8")
+  return content
+}
+
+function appendToImprovementLog(entry) {
+  const current = readInstructions()
+  const timestamp = new Date().toISOString().replace("T", " ").substring(0, 16)
+  const logEntry = `\n- ${timestamp} — ${entry}`
+  const logSection = "\n## Improvement Log"
+  const logIndex = current.indexOf(logSection)
+  if (logIndex !== -1) {
+    const afterLog = current.substring(logIndex + logSection.length)
+    const nextSectionIndex = afterLog.search(/\n## /)
+    if (nextSectionIndex !== -1) {
+      const insertAt = logIndex + logSection.length + nextSectionIndex
+      const updated = current.substring(0, insertAt) + logEntry + "\n" + current.substring(insertAt)
+      writeInstructions(updated)
+      return updated
+    }
+    writeInstructions(current + logEntry + "\n")
+    return current + logEntry + "\n"
+  }
+  const updated = current + `\n${logSection}\n${logEntry}\n`
+  writeInstructions(updated)
+  return updated
+}
+
+// ── Chat handler ────────────────────────────────────────────────
+async function handleChat(userMessage, userId, userRole) {
+  const jobs = loadedJobs
+  const instructions = readInstructions()
+
+  const jobList = jobs
+    .map((j) => `- ${j.job_title} (${j.industry}): ${j.description}`)
+    .join("\n")
+
+  const systemPrompt = `You are a Sales AI Assistant for Coast IT CRM. Your role is to help salespeople with tips, strategies, and guidance.
+
+Available job categories to target:
+${jobList}
+
+Current instructions:
+${instructions}
+
+Provide concise, actionable sales advice. When asked about a specific job category, give targeted tips on finding and engaging prospects in that field. Keep responses under 300 words unless asked for detail.`
+
+  const ollamaRes = await fetch(`${OLLAMA_URL}/api/chat`, {
+    method: "POST",
+    headers: { "Content-Type": "application/json" },
+    body: JSON.stringify({
+      model: MODEL,
+      messages: [
+        { role: "system", content: systemPrompt },
+        { role: "user", content: userMessage },
+      ],
+      stream: false,
+      options: { temperature: 0.7, num_predict: 1024 },
+    }),
+  })
+
+  if (!ollamaRes.ok) {
+    const text = await ollamaRes.text()
+    throw new Error(`Ollama error (${ollamaRes.status}): ${text}`)
+  }
+
+  const data = await ollamaRes.json()
+  const responseText = data.message?.content || ""
+
+  // Try to persist to DB (best-effort)
+  try {
+    if (pgPool && userId) {
+      await pgPool.query(
+        "INSERT INTO ai_conversations (id, user_id, role, message, response) VALUES ($1, $2, $3, $4, $5)",
+        [crypto.randomUUID(), userId, userRole || "sales", userMessage, responseText]
+      )
+    }
+  } catch {
+    // table might not exist, ignore
+  }
+
+  return responseText
+}
+
+// ── PG pool (lazy init) ────────────────────────────────────────
+let pgPool = null
+async function initPg() {
+  if (!DATABASE_URL) return
+  try {
+    const { default: pg } = await import("pg")
+    pgPool = new pg.Pool({ connectionString: DATABASE_URL, max: 5 })
+    await pgPool.query("SELECT 1")
+    console.log("Connected to PostgreSQL")
+  } catch (e) {
+    console.warn("PostgreSQL unavailable (AI convos won't be saved):", e.message)
+  }
+}
+
+// ── Request router ─────────────────────────────────────────────
+const loadedJobs = loadJobs()
+
+function sendJSON(res, status, data) {
+  res.writeHead(status, { "Content-Type": "application/json" })
+  res.end(JSON.stringify(data))
+}
+
+function parseBody(req) {
+  return new Promise((resolve, reject) => {
+    let body = ""
+    req.on("data", (chunk) => (body += chunk))
+    req.on("end", () => {
+      try {
+        resolve(JSON.parse(body))
+      } catch {
+        reject(new Error("Invalid JSON"))
+      }
+    })
+    req.on("error", reject)
+  })
+}
+
+function parseURL(req) {
+  const url = new URL(req.url, `http://${req.headers.host || "localhost"}`)
+  return { pathname: url.pathname, searchParams: url.searchParams }
+}
+
+const server = http.createServer(async (req, res) => {
+  // CORS
+  res.setHeader("Access-Control-Allow-Origin", "*")
+  res.setHeader("Access-Control-Allow-Methods", "GET, POST, OPTIONS")
+  res.setHeader("Access-Control-Allow-Headers", "Content-Type")
+
+  if (req.method === "OPTIONS") {
+    res.writeHead(204)
+    res.end()
+    return
+  }
+
+  const { pathname } = parseURL(req)
+
+  try {
+    // GET /health
+    if (req.method === "GET" && pathname === "/health") {
+      return sendJSON(res, 200, { status: "ok", model: MODEL })
+    }
+
+    // GET /ai/jobs
+    if (req.method === "GET" && pathname === "/ai/jobs") {
+      return sendJSON(res, 200, { jobs: loadedJobs })
+    }
+
+    // GET /ai/instructions
+    if (req.method === "GET" && pathname === "/ai/instructions") {
+      const instructions = readInstructions()
+      return sendJSON(res, 200, { success: true, instructions })
+    }
+
+    // POST /ai/instructions
+    if (req.method === "POST" && pathname === "/ai/instructions") {
+      const body = await parseBody(req)
+      if (body.content) {
+        writeInstructions(body.content)
+      } else if (body.entry) {
+        appendToImprovementLog(body.entry)
+      }
+      return sendJSON(res, 200, {
+        success: true,
+        instructions: readInstructions(),
+      })
+    }
+
+    // POST /ai/chat
+    if (req.method === "POST" && pathname === "/ai/chat") {
+      const body = await parseBody(req)
+      const { message, user_id, user_role } = body
+
+      if (!message) {
+        return sendJSON(res, 400, { error: "message is required" })
+      }
+
+      const validRoles = ["sales", "admin", "super_admin"]
+      if (user_role && !validRoles.includes(user_role)) {
+        return sendJSON(res, 403, { error: "Forbidden" })
+      }
+
+      const response = await handleChat(message, user_id || "", user_role || "sales")
+      return sendJSON(res, 200, { response })
+    }
+
+    // 404
+    sendJSON(res, 404, { error: "Not found" })
+  } catch (err) {
+    console.error("Request error:", err)
+    sendJSON(res, 500, { error: err.message })
+  }
+})
+
+// ── Start ───────────────────────────────────────────────────────
+server.listen(PORT, HOST, () => {
+  console.log(`CRM AI server listening on http://${HOST}:${PORT}`)
+  console.log(`  Model: ${MODEL}`)
+  console.log(`  Ollama: ${OLLAMA_URL}`)
+})
+
+initPg()

browser-use-service/main.py

@@ -0,0 +1,445 @@
+import os, json, asyncio, re, shutil, sqlite3, urllib.parse, random, logging
+from datetime import datetime, timedelta
+from fastapi import FastAPI, Query
+from fastapi.middleware.cors import CORSMiddleware
+import uvicorn
+from playwright.async_api import async_playwright
+
+logging.basicConfig(level=logging.INFO, format='%(asctime)s [%(levelname)s] %(message)s')
+logger = logging.getLogger(__name__)
+
+app = FastAPI()
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["http://localhost:3006", "http://127.0.0.1:3006"],
+    allow_methods=["POST"],
+    allow_headers=["*"],
+)
+PORT = int(os.getenv("PORT", "3008"))
+
+OLLAMA_URL = os.getenv("OLLAMA_URL", "http://localhost:11434")
+CLASSIFY_MODEL = os.getenv("CLASSIFY_MODEL", "dolphin-llama3:8b")
+
+FX_PROFILE = os.getenv('FX_PROFILE', '')
+
+BROAD_KEYWORDS = [
+    "website", "web design", "web develop", "web dev",
+    "build my website", "build a website", "create a website",
+    "need web", "looking for web", "new website",
+    "landing page", "wordpress",
+    "need a website", "my website", "business website",
+    "need a designer", "help with my website",
+    "redesign", "update my website",
+    "looking for", "need a", "need an", "looking to",
+    "help me build", "create my", "build me",
+    "design my", "make me a", "would like",
+    "need someone", "hire a", "looking to hire",
+    "want someone", "need help with",
+]
+
+def kw_match(text: str) -> bool:
+    t = text.lower()
+    return any(kw in t for kw in BROAD_KEYWORDS)
+
+FB_SEARCHES = [
+    "looking for web developer",
+    "need a website designed",
+    "need website built South Africa",
+    "need someone to build my website",
+    "need web designer",
+    "looking for someone to create website",
+    "need ecommerce website built",
+    "want to hire web developer",
+    "website quote please",
+    "need wordpress website",
+    "I need a website for my business",
+    "need a site for my business",
+]
+
+VIEWPORTS = [
+    {'width': 1280, 'height': 800},
+    {'width': 1366, 'height': 768},
+    {'width': 1440, 'height': 900},
+    {'width': 1536, 'height': 864},
+    {'width': 1920, 'height': 1080},
+]
+
+async def get_fb_cookies(profile_path: str | None = None):
+    cookie_db_path = profile_path or FX_PROFILE
+    if not cookie_db_path:
+        logger.warning("No profile path provided")
+        return []
+    cookie_db = os.path.join(cookie_db_path, 'cookies.sqlite')
+    if not os.path.exists(cookie_db):
+        logger.warning("Cookie DB not found at %s", cookie_db)
+        return []
+    tmp = os.path.join(os.path.dirname(cookie_db), f'cookies_tmp_{os.getpid()}.sqlite')
+    for attempt in range(3):
+        try:
+            shutil.copy2(cookie_db, tmp)
+            break
+        except PermissionError:
+            if attempt < 2:
+                await asyncio.sleep(0.5)
+            else:
+                logger.error("Failed to copy cookie DB after 3 attempts")
+                return []
+    try:
+        conn = sqlite3.connect(tmp)
+        c = conn.cursor()
+        c.execute("SELECT name, value, host, path FROM moz_cookies WHERE host LIKE '%facebook.com'")
+        rows = c.fetchall()
+        conn.close()
+        try:
+            os.remove(tmp)
+        except Exception:
+            pass
+        return [{
+            "name": name, "value": value,
+            "domain": host if host.startswith('.') else f'.{host}',
+            "path": path,
+            "httpOnly": True, "secure": True, "sameSite": "Lax",
+        } for name, value, host, path in rows]
+    except Exception as e:
+        logger.error("Cookie DB read error: %s", e)
+        try:
+            os.remove(tmp)
+        except Exception:
+            pass
+        return []
+
+WEEKDAY_ORDER = ['monday', 'tuesday', 'wednesday', 'thursday', 'friday', 'saturday', 'sunday']
+
+def _parse_fb_date(block: list[str]) -> str:
+    for l in block:
+        l = l.strip()
+        m = re.match(r'(\d+)\s*(h|hr|hrs|hour|hours)\s*ago', l)
+        if m:
+            hours = int(m.group(1))
+            d = datetime.now() - timedelta(hours=hours)
+            return d.strftime('%Y-%m-%d')
+        if l.lower() == 'yesterday':
+            d = datetime.now() - timedelta(days=1)
+            return d.strftime('%Y-%m-%d')
+        low = l.lower()
+        if low in WEEKDAY_ORDER:
+            day_idx = WEEKDAY_ORDER.index(low)
+            today_idx = datetime.now().weekday()
+            days_ago = (today_idx - day_idx) % 7
+            d = datetime.now() - timedelta(days=days_ago)
+            return d.strftime('%Y-%m-%d')
+        for fmt in ['%Y-%m-%d', '%d %B %Y', '%B %d %Y', '%d %b %Y', '%b %d %Y', '%d %b', '%b %d']:
+            try:
+                dt = datetime.strptime(l, fmt)
+                return dt.strftime('%Y-%m-%d')
+            except ValueError:
+                pass
+    return datetime.now().strftime('%Y-%m-%d')
+
+def _extract_posts_from_text(raw: str, url: str) -> list[dict]:
+    lines = [l.strip() for l in raw.split('\n')]
+    blocks = []
+    cur = []
+    fb_run = 0
+    for l in lines:
+        if 'facebook' in l.lower():
+            fb_run += 1
+            if cur and fb_run >= 2:
+                blocks.append(cur)
+                cur = []
+        else:
+            fb_run = 0
+            if l and len(l) >= 15:
+                words = re.findall(r'[A-Za-z]{2,}', l)
+                if len(words) >= 2:
+                    cur.append(l)
+    if cur:
+        blocks.append(cur)
+
+    posts = []
+    seen_texts = set()
+    for block in blocks:
+        if not block:
+            continue
+        kw_indices = [i for i, l in enumerate(block) if kw_match(l)]
+        if not kw_indices:
+            continue
+        i = kw_indices[0]
+        start = max(0, i - 2)
+        end = min(len(block), i + 5)
+        snippet = ' '.join(block[start:end])
+        if len(snippet) < 40:
+            continue
+        dekey = snippet[:80]
+        if dekey in seen_texts:
+            continue
+        seen_texts.add(dekey)
+        posts.append({
+            "title": snippet[:300],
+            "content": snippet[:1000],
+            "author": block[start] if start < i else '',
+            "url": url,
+            "source": "facebook",
+            "date": _parse_fb_date(block),
+        })
+    return posts
+
+async def human_scroll(page, steps: int = None, total_delay: float = None):
+    steps = steps or random.randint(2, 5)
+    total_delay = total_delay or random.uniform(6, 18)
+    step_delay = total_delay / steps
+    for i in range(steps):
+        scroll_dist = random.randint(200, 600)
+        await page.evaluate(f"window.scrollBy(0, {scroll_dist})")
+        await page.wait_for_timeout(int(step_delay * 1000))
+    if random.random() < 0.3:
+        await page.evaluate(f"window.scrollBy(0, -{random.randint(100, 300)})")
+        await page.wait_for_timeout(random.randint(1000, 3000))
+
+async def random_idle(page):
+    action = random.random()
+    if action < 0.15:
+        try:
+            elems = await page.query_selector_all('a, span, div[role="button"]')
+            if elems:
+                el = random.choice(elems[:20])
+                box = await el.bounding_box()
+                if box:
+                    x = box['x'] + box['width'] / 2 + random.randint(-20, 20)
+                    y = box['y'] + box['height'] / 2 + random.randint(-10, 10)
+                    await page.mouse.move(x, y, steps=random.randint(20, 40))
+                    await page.wait_for_timeout(random.randint(500, 2000))
+        except Exception:
+            pass
+
+async def search_facebook(page, query: str) -> list[dict]:
+    url = f'https://www.facebook.com/search/posts/?q={urllib.parse.quote(query)}'
+    try:
+        await page.goto(url, wait_until='domcontentloaded', timeout=30000)
+        await page.wait_for_timeout(random.randint(3000, 8000))
+
+        await human_scroll(page, steps=random.randint(2, 4), total_delay=random.uniform(6, 15))
+
+        # Accidental overscroll: 10% chance to jump to bottom and back
+        if random.random() < 0.1:
+            await page.evaluate("window.scrollTo(0, document.body.scrollHeight)")
+            await page.wait_for_timeout(random.randint(1000, 3000))
+            await page.evaluate("window.scrollBy(0, -300)")
+            await page.wait_for_timeout(random.randint(1000, 3000))
+
+        if random.random() < 0.2:
+            await page.evaluate("window.scrollTo(0, 0)")
+            await page.wait_for_timeout(random.randint(2000, 5000))
+
+        if random.random() < 0.3:
+            await random_idle(page)
+
+        raw = await page.evaluate('document.body.innerText')
+    except Exception as e:
+        logger.warning("Facebook search failed: %s", e)
+        return []
+    return _extract_posts_from_text(raw, url)
+
+def cleanup_chrome():
+    import subprocess, signal
+    try:
+        subprocess.run(["taskkill", "/F", "/IM", "chrome-headless-shell.exe"], capture_output=True, timeout=5)
+    except Exception:
+        pass
+
+async def scrape_facebook(profile_path: str | None = None, force: bool = False) -> dict:
+    cleanup_chrome()
+    fb_cookies = await get_fb_cookies(profile_path)
+    if not fb_cookies:
+        logger.warning("No Facebook cookies available")
+        return {"success": False, "leads": [], "flagged": False, "flag_reason": None, "error": "No cookies available"}
+    try:
+        async with async_playwright() as pw:
+            browser = await pw.chromium.launch(
+                headless=True,
+                args=['--disable-blink-features=AutomationControlled']
+            )
+            viewport = random.choice(VIEWPORTS)
+            context = await browser.new_context(
+                user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:150.0) Gecko/20100101 Firefox/150.0',
+                viewport=viewport,
+            )
+
+            await context.add_init_script("""
+                Object.defineProperty(navigator, 'webdriver', { get: () => false });
+                Object.defineProperty(navigator, 'languages', { get: () => ['en-US', 'en'] });
+                Object.defineProperty(navigator, 'platform', { get: () => 'Win32' });
+                window.chrome = { runtime: {} };
+                Object.defineProperty(navigator, 'hardwareConcurrency', { get: () => 8 });
+                Object.defineProperty(navigator, 'deviceMemory', { get: () => 8 });
+            """)
+            for c in fb_cookies:
+                try:
+                    await context.add_cookies([c])
+                except Exception as e:
+                    logger.warning("Failed to inject cookie %s: %s", c.get('name'), e)
+
+            page = await context.new_page()
+            # Navigate through google first for a legitimate Referer header
+            await page.goto('https://www.google.com/', wait_until='domcontentloaded', timeout=15000)
+            await page.wait_for_timeout(random.randint(1000, 3000))
+            await page.goto('https://www.facebook.com/', wait_until='domcontentloaded', timeout=30000)
+            await page.wait_for_timeout(random.randint(3000, 8000))
+
+            url = page.url
+            if '/login' in url.lower():
+                logger.warning("Facebook login page detected — account flagged")
+                await browser.close()
+                return {"success": False, "leads": [], "flagged": True, "flag_reason": "login_page", "error": "Facebook login page detected"}
+
+            # Browse feed like a human before searching
+            await human_scroll(page, steps=random.randint(2, 4), total_delay=random.uniform(8, 20))
+            if random.random() < 0.25:
+                await page.evaluate("window.scrollTo(0, 0)")
+                await page.wait_for_timeout(random.randint(2000, 5000))
+                await human_scroll(page, steps=random.randint(1, 2))
+
+            # False start: 30% chance to idle and leave without scraping (skipped when force=true)
+            if not force and random.random() < 0.3:
+                await page.wait_for_timeout(random.randint(8000, 20000))
+                await browser.close()
+                return {"success": True, "leads": [], "flagged": False, "flag_reason": None, "error": None}
+
+            all_posts = []
+            searches = random.sample(FB_SEARCHES, k=random.randint(3, 6))
+            for i, query in enumerate(searches):
+                try:
+                    posts = await search_facebook(page, query)
+                    all_posts.extend(posts)
+                    # Between searches: random break with possible small scroll
+                    if random.random() < 0.4:
+                        await page.evaluate(f"window.scrollBy(0, {random.randint(-300, 300)})")
+                    delay = random.uniform(8, 25)
+                    await page.wait_for_timeout(int(delay * 1000))
+                    # Tab switch: 15% chance after 2nd-3rd search
+                    if i == random.randint(1, 2) and random.random() < 0.15:
+                        new_page = await context.new_page()
+                        await new_page.goto('https://www.messenger.com/', wait_until='domcontentloaded', timeout=15000)
+                        await new_page.wait_for_timeout(random.randint(3000, 8000))
+                        await new_page.close()
+                        await page.wait_for_timeout(random.randint(1000, 3000))
+                except Exception as e:
+                    logger.warning("Facebook search '%s' failed: %s", query, e)
+
+            # Idle before closing — human would pause
+            if random.random() < 0.5:
+                await page.wait_for_timeout(random.randint(3000, 10000))
+
+            await browser.close()
+
+            seen = set()
+            deduped = []
+            for p in all_posts:
+                key = p.get('content', '')[:100]
+                if key not in seen:
+                    seen.add(key)
+                    deduped.append(p)
+
+            leads = deduped[:20]
+            if leads:
+                leads = await classify_leads(leads)
+
+            return {"success": True, "leads": leads[:15], "flagged": False, "flag_reason": None, "error": None}
+    except Exception as e:
+        logger.error("Facebook scrape failed: %s", e)
+        return {"success": False, "leads": [], "flagged": False, "flag_reason": None, "error": str(e)}
+
+async def ask_ollama(prompt: str) -> str:
+    import httpx
+    async with httpx.AsyncClient(timeout=120) as c:
+        r = await c.post(f"{OLLAMA_URL}/api/chat", json={
+            "model": CLASSIFY_MODEL,
+            "messages": [
+                {"role": "system", "content": "You classify forum posts. Return only valid JSON."},
+                {"role": "user", "content": prompt}
+            ],
+            "stream": False,
+            "options": {"temperature": 0.05, "num_predict": 1024},
+        })
+        r.raise_for_status()
+        data = r.json()
+        return data["message"]["content"]
+
+async def classify_leads(results: list[dict]) -> list[dict]:
+    if not results:
+        return []
+    briefs = [r["title"][:200] for r in results]
+    prompt = f"""Classify each post as LEAD or NOT.
+LEAD = someone REQUESTING/POSTING/WANTING a website built, designed, or created for them.
+LEAD examples: "Need a website for my business", "Looking for web developer to build my site", "I need someone to create my website", "Want a new website for my company", "Looking for someone to design my WordPress site"
+
+NOT LEAD:
+- Offering web design services: "I build websites", "I offer web design", "Affordable web design packages"
+- Already have a website and need marketing, SEO, content, video, link building, email marketing, affiliates
+- Recruiting employees, hiring staff, looking for business partners
+- Selling products, promoting services, affiliate offers
+- "Need web hosting", "Looking for a partner", "Looking for content writer", "Video spokesperson"
+
+For each numbered post, answer ONLY "yes" (LEAD) or "no" (NOT LEAD):
+{chr(10).join(f'{i+1}. {t}' for i, t in enumerate(briefs))}
+Return a JSON array like ["yes","no","yes"] matching the order above."""
+    ai_succeeded = False
+    try:
+        raw = await ask_ollama(prompt)
+        raw = raw.strip()
+        if raw.startswith("```"):
+            first_nl = raw.find('\n')
+            if first_nl != -1:
+                raw = raw[first_nl + 1:]
+            if raw.endswith("```"):
+                raw = raw[:-3]
+            raw = raw.strip()
+        answers = json.loads(raw)
+        if isinstance(answers, list) and len(answers) == len(results):
+            ai_succeeded = True
+            filtered = []
+            for i, ans in enumerate(answers):
+                if isinstance(ans, str) and ans.lower() == 'yes':
+                    filtered.append(results[i])
+            if filtered:
+                return filtered[:10]
+            logger.info("AI classified all %d items as NOT LEAD — returning empty", len(results))
+            return []
+    except Exception as e:
+        logger.warning("AI classification failed, falling back to keyword filter: %s", e)
+
+    if not ai_succeeded:
+        strict_keywords = [
+            "website", "web design", "web develop", "web dev",
+            "build my website", "build a website", "create a website",
+            "need web", "looking for web", "new website",
+            "landing page", "wordpress",
+            "need a website", "my website", "business website",
+            "need a designer", "help with my website",
+            "redesign", "update my website",
+        ]
+        filtered = []
+        for r in results:
+            t = r['title'].lower()
+            if not any(kw in t for kw in strict_keywords):
+                continue
+            if any(kw in t for kw in ['i build', 'i offer', 'i create', 'i am a', 'web developer available',
+                                       'affordable web', 'web hosting', 'free website',
+                                       'limited time', 'special offer', 'sign up now',
+                                       'offer']):
+                continue
+            filtered.append(r)
+        return filtered[:10]
+    return []
+
+@app.get("/health")
+async def health():
+    return {"status": "ok"}
+
+@app.post("/scrape/facebook")
+async def scrape_facebook_endpoint(profile_path: str | None = Query(None), force: bool = Query(False)):
+    result = await scrape_facebook(profile_path, force)
+    return result
+
+if __name__ == "__main__":
+    uvicorn.run(app, host="0.0.0.0", port=PORT)

data/ai/ai.md

@@ -0,0 +1,40 @@
+# AI Sales Assistant — Self-Improvement Instructions
+
+## Purpose
+This file contains the AI's own configuration, knowledge, and improvement rules.
+The AI can read and modify this file to update its behavior at runtime.
+
+## Current Instructions
+- Always respond in English
+- Keep responses under 300 words unless asked for detail
+- Use bullet points for lists
+- Be direct and actionable — no fluff
+- Never mention being an AI or language model
+- Refer to the user by their role (salesperson, admin, etc.)
+- If unsure about a topic, say "I don't have that information yet" rather than guessing
+
+## Knowledge Base
+### Sales Tips
+- Cold emails should be under 150 words
+- Follow up within 48 hours
+- Personalise every outreach with the prospect's name and company
+- Use open-ended questions in discovery calls
+- Always ask for the next step before ending a call
+
+### Job Targeting
+- Developers respond best to technical value props
+- Marketing managers care about ROI and metrics
+- C-level executives want brevity and business impact
+
+## Improvement Log
+Track changes made by the AI to improve itself:
+- (initial) Basic instructions and knowledge base created
+
+## Self-Modification Rules
+The AI may update this file when:
+1. It identifies a gap in its knowledge that would help salespeople
+2. It discovers a better way to structure responses
+3. A user explicitly requests an update to behavior
+4. It notices repeated questions that aren't well-covered
+
+Only append to the Improvement Log — don't delete previous entries.

data/ai/jobs.jsonl

@@ -0,0 +1,10 @@
+{"job_title":"Software Developer","keywords":["developer","programmer","software engineer","coder","full stack","backend","frontend"],"industry":"Technology","description":"Builds and maintains software applications and systems"}
+{"job_title":"Marketing Specialist","keywords":["marketing","digital marketing","brand manager","content marketer","social media"],"industry":"Marketing","description":"Plans and executes marketing campaigns across channels"}
+{"job_title":"Sales Representative","keywords":["sales rep","account executive","business development","sales consultant"],"industry":"Sales","description":"Drives revenue through client acquisition and relationship management"}
+{"job_title":"Project Manager","keywords":["project manager","program manager","scrum master","agile coach"],"industry":"Business","description":"Oversees project timelines, resources, and deliverables"}
+{"job_title":"Graphic Designer","keywords":["designer","graphic designer","ui designer","ux designer","visual designer"],"industry":"Creative","description":"Creates visual concepts and designs for digital and print media"}
+{"job_title":"Data Analyst","keywords":["data analyst","business analyst","data scientist","analytics"],"industry":"Technology","description":"Analyzes data to provide actionable business insights"}
+{"job_title":"Customer Support Specialist","keywords":["customer support","customer service","support agent","help desk"],"industry":"Customer Service","description":"Assists customers with inquiries, issues, and product support"}
+{"job_title":"Human Resources Manager","keywords":["HR manager","HR","recruiter","talent acquisition","people operations"],"industry":"Human Resources","description":"Manages recruitment, employee relations, and HR operations"}
+{"job_title":"Financial Advisor","keywords":["financial advisor","financial planner","wealth manager","investment advisor"],"industry":"Finance","description":"Provides financial guidance and investment planning to clients"}
+{"job_title":"Operations Manager","keywords":["operations manager","operations","logistics","supply chain"],"industry":"Business","description":"Oversees daily operations and process optimization"}

database/DESIGN_DECISIONS.md

@@ -0,0 +1,164 @@
+# CRM Database Schema — Enterprise RBAC + CRM
+
+## Architecture Overview
+
+Enterprise-grade PostgreSQL schema implementing **hierarchical Role-Based Access Control (RBAC)**
+with a complete CRM system. Designed for production deployments with strict security requirements,
+audit compliance, and scalability to millions of records.
+
+---
+
+## 1. Hierarchical RBAC Model
+
+Four roles arranged in a strict hierarchy by `hierarchy_level` (lower = higher privilege):
+
+| Role | Level | Authority |
+|------|-------|-----------|
+| **SUPER_ADMIN** | 1 | Unrestricted — create/delete users, assign roles, override all permissions |
+| **ADMIN** | 2 | Operational — ban/suspend users, review reports, moderate activity. **Cannot create accounts.** |
+| **SALES_USER** | 3 | CRM operations — leads, customers, opportunities, communications |
+| **DEVELOPER** | 4 | Technical + CRM read access for post-sale project visibility |
+
+### Enforcement via PostgreSQL Triggers
+
+- **`enforce_create_user()`** (BEFORE INSERT ON users): Only SUPER_ADMIN (`hierarchy_level = 1`)
+  can create new user accounts. Bootstrap case (`created_by IS NULL`) is allowed for initial setup.
+- **`enforce_role_assignment()`** (BEFORE INSERT ON user_roles): Only SUPER_ADMIN can assign roles.
+  Prevents privilege escalation by checking hierarchy levels.
+
+### Permission Model
+
+Granular permissions stored in `permissions` table with `(resource, action)` pairs.
+`role_permissions` maps roles to permissions. The `has_permission(user_id, resource, action)`
+function provides application-layer authorization checks.
+
+---
+
+## 2. Ban & Suspension System
+
+Two separate tables for enforcement flexibility:
+
+- **`banned_users`**: Permanent or time-limited bans. Supports reversal (by SUPER_ADMIN).
+  Admins can create bans; SUPER_ADMIN can reverse or permanently delete banned users.
+- **`suspended_users`**: Time-limited suspensions (always has expiry). Admins can create
+  and lift suspensions.
+
+Both tables are fully audited via dedicated trigger functions.
+
+---
+
+## 3. Session & Login Security
+
+- **`sessions`**: Stores hashed session tokens with expiry. Only one active session per
+  token hash.
+- **`login_attempts`**: Captures every login attempt (successful or failed) with IP address
+  and user agent. Used for brute-force detection and audit.
+- Users have `failed_login_attempts` and `locked_until` for account lockout policies.
+
+---
+
+## 4. Password Security
+
+- All passwords hashed with **bcrypt (cost factor 12)**
+- `password_change_required` forces password change on first login (TRUE for all non-root
+  test accounts)
+- Password hashes are opaque to the database — validation is application-layer only
+
+---
+
+## 5. UUID Primary Keys
+
+- All tables use `UUID PRIMARY KEY DEFAULT uuid_generate_v4()`
+- Fixed UUIDs used for seed data and system entities for referential transparency
+- Prevents enumeration attacks and supports distributed ID generation
+
+---
+
+## 6. Audit Trail
+
+Every mutation is logged to `audit_logs` via `audit_trigger_function()`:
+
+- Captures `(table_name, record_id, action, old_data, new_data, changed_by, ip_address)`
+- Ban actions, suspension actions, and successful logins have dedicated audit triggers
+- Indexed on `(table_name, record_id)` for per-record lookup
+
+Dedicated audit views:
+- `v_active_bans` — Currently active bans with user details
+- `v_active_suspensions` — Currently active suspensions
+- `v_user_permissions` — Flattened permission report per user
+
+---
+
+## 7. Third Normal Form (3NF)
+
+Customer model uses the **discriminator pattern**:
+
+```
+customers (parent — shared columns: status, owner, tags, score)
+  ├── customer_type = 'individual' → individual_customers
+  └── customer_type = 'company'    → company_customers
+```
+
+All lookup tables are normalized. Contact information, addresses, and notes are
+separate 1:N child tables.
+
+---
+
+## 8. Indexing Strategy
+
+- **Core indexes**: FK columns, status/sort/date columns, unique constraints
+- **Partial indexes**: `WHERE deleted_at IS NULL` on soft-delete tables,
+  `WHERE is_active = TRUE` on bans/suspensions, `WHERE due_date < NOW()` on overdue invoices
+- **Full-text search**: GIN indexes on customers, leads, products, communications, tasks
+- **Composite indexes**: Common query patterns (e.g., `opportunities(owner_id, stage_id)`)
+
+---
+
+## 9. Soft Delete
+
+- `deleted_at TIMESTAMPTZ` on all business tables
+- Partial unique indexes ignore soft-deleted rows (e.g., `WHERE deleted_at IS NULL`)
+- Audit logs capture the full row snapshot before soft-delete
+
+---
+
+## 10. Test Accounts
+
+| Username | Password | Role |
+|----------|----------|------|
+| `superadmin_demo` | `[REDACTED]` | SUPER_ADMIN |
+| `admin_demo` | `[REDACTED]` | ADMIN |
+| `sales_demo` | `[REDACTED]` | SALES_USER |
+| `dev_demo` | `[REDACTED]` | DEVELOPER |
+
+---
+
+## 11. Migration Instructions
+
+```bash
+# Create database
+psql -U postgres -c "CREATE DATABASE crm;"
+
+# Run full migration
+psql -U postgres -d crm -f database/migrations/run_all.sql
+
+# Or step by step:
+psql -U postgres -d crm -f database/migrations/001_schema.sql
+psql -U postgres -d crm -f database/migrations/002_seed.sql
+```
+
+---
+
+## 12. Security Rules Summary
+
+| Rule | Enforcement |
+|------|-------------|
+| Only SUPER_ADMIN creates accounts | Trigger `trg_enforce_create_user` |
+| Only SUPER_ADMIN assigns roles | Trigger `trg_enforce_role_assignment` |
+| No privilege escalation | Level check in `enforce_role_assignment()` |
+| bcrypt password hashing | Application-layer (cost=12) |
+| Unique usernames/emails | Partial unique indexes |
+| Force password change on first login | `password_change_required` column |
+| Audit every login/logout | Trigger `trg_audit_login` on `login_attempts` |
+| Audit every ban action | Trigger `trg_audit_ban` on `banned_users` |
+| Audit every suspension action | Trigger `trg_audit_suspension` on `suspended_users` |

database/migrations/002_seed.sql

@@ -0,0 +1,401 @@
+-- ============================================================================
+-- CRM Database Seed Data — RBAC + Test Accounts + Reference Data
+-- ============================================================================
+
+-- ============================================================================
+-- FIXED UUIDs FOR REFERENTIAL INTEGRITY
+-- ============================================================================
+
+-- Role UUIDs
+-- SUPER_ADMIN=1, ADMIN=2, SALES_USER=3, DEVELOPER=4
+-- User UUIDs: SUPER_ADMIN=u0001, ADMIN=u0002, SALES=u0003, DEV=u0004
+
+-- ============================================================================
+-- 1. ROLES
+-- ============================================================================
+
+INSERT INTO roles (id, name, display_name, description, hierarchy_level, is_system) VALUES
+    ('00000001-0000-0000-0000-000000000000', 'SUPER_ADMIN', 'Super Administrator',
+     'Highest authority. Full system access. Can create/delete users, assign roles, override all permissions.',
+     1, TRUE),
+    ('00000002-0000-0000-0000-000000000000', 'ADMIN', 'Administrator',
+     'Operational administration. Can ban/suspend users, review reports, moderate activity. Cannot create accounts.',
+     2, TRUE),
+    ('00000003-0000-0000-0000-000000000000', 'SALES_USER', 'Sales Representative',
+     'Day-to-day CRM operations. Manage leads, customers, opportunities, and communications.',
+     3, TRUE),
+    ('00000004-0000-0000-0000-000000000000', 'DEVELOPER', 'Developer',
+     'Internal development/testing with CRM read access. API testing, diagnostics, debugging. Can view projects and customer data post-sale.',
+     4, TRUE)
+ON CONFLICT (name) DO NOTHING;
+
+-- ============================================================================
+-- 2. PERMISSIONS — organised by category
+-- ============================================================================
+
+-- USER MANAGEMENT (SUPER_ADMIN only)
+INSERT INTO permissions (id, resource, action, description, category) VALUES
+    ('a0010001-0000-0000-0000-000000000000', 'users', 'create', 'Create new user accounts', 'User Management'),
+    ('a0010002-0000-0000-0000-000000000000', 'users', 'read', 'View user details', 'User Management'),
+    ('a0010003-0000-0000-0000-000000000000', 'users', 'update', 'Update user details', 'User Management'),
+    ('a0010004-0000-0000-0000-000000000000', 'users', 'delete', 'Permanently delete user accounts', 'User Management'),
+    ('a0010005-0000-0000-0000-000000000000', 'users', 'assign_roles', 'Assign roles to users', 'User Management'),
+    ('a0010006-0000-0000-0000-000000000000', 'users', 'promote', 'Promote users to higher roles', 'User Management'),
+    ('a0010007-0000-0000-0000-000000000000', 'users', 'demote', 'Demote users to lower roles', 'User Management'),
+    ('a0010008-0000-0000-0000-000000000000', 'users', 'reset_password', 'Reset user passwords', 'User Management'),
+    ('a0010009-0000-0000-0000-000000000000', 'users', 'disable', 'Disable/disable user accounts', 'User Management'),
+    ('a0010010-0000-0000-0000-000000000000', 'users', 'permanently_delete', 'Permanently remove accounts from system', 'User Management')
+ON CONFLICT DO NOTHING;
+
+-- BAN & SUSPENSION (ADMIN + SUPER_ADMIN)
+INSERT INTO permissions (id, resource, action, description, category) VALUES
+    ('a0020001-0000-0000-0000-000000000000', 'bans', 'create', 'Ban a user account', 'Ban Management'),
+    ('a0020002-0000-0000-0000-000000000000', 'bans', 'reverse', 'Reverse a ban on a user', 'Ban Management'),
+    ('a0020003-0000-0000-0000-000000000000', 'bans', 'permanently_delete', 'Permanently delete banned users', 'Ban Management'),
+    ('a0020004-0000-0000-0000-000000000000', 'suspensions', 'create', 'Suspend a user account', 'Ban Management'),
+    ('a0020005-0000-0000-0000-000000000000', 'suspensions', 'lift', 'Lift a suspension', 'Ban Management'),
+    ('a0020006-0000-0000-0000-000000000000', 'suspensions', 'view', 'View suspension records', 'Ban Management')
+ON CONFLICT DO NOTHING;
+
+-- CRM CORE (SALES_USER + MANAGERS)
+INSERT INTO permissions (id, resource, action, description, category) VALUES
+    ('a0030001-0000-0000-0000-000000000000', 'customers', 'create', 'Create customer records', 'CRM'),
+    ('a0030002-0000-0000-0000-000000000000', 'customers', 'read', 'View customer records', 'CRM'),
+    ('a0030003-0000-0000-0000-000000000000', 'customers', 'update', 'Update customer records', 'CRM'),
+    ('a0030004-0000-0000-0000-000000000000', 'customers', 'delete', 'Delete customer records', 'CRM'),
+    ('a0030005-0000-0000-0000-000000000000', 'leads', 'create', 'Create sales leads', 'CRM'),
+    ('a0030006-0000-0000-0000-000000000000', 'leads', 'read', 'View sales leads', 'CRM'),
+    ('a0030007-0000-0000-0000-000000000000', 'leads', 'update', 'Update sales leads', 'CRM'),
+    ('a0030008-0000-0000-0000-000000000000', 'leads', 'convert', 'Convert leads to customers', 'CRM'),
+    ('a0030009-0000-0000-0000-000000000000', 'opportunities', 'create', 'Create sales opportunities', 'CRM'),
+    ('a0030010-0000-0000-0000-000000000000', 'opportunities', 'read', 'View sales opportunities', 'CRM'),
+    ('a0030011-0000-0000-0000-000000000000', 'opportunities', 'update', 'Update sales pipeline', 'CRM'),
+    ('a0030012-0000-0000-0000-000000000000', 'pipeline', 'update', 'Update sales pipeline stages', 'CRM'),
+    ('a0030013-0000-0000-0000-000000000000', 'communications', 'create', 'Log calls and meetings', 'CRM'),
+    ('a0030014-0000-0000-0000-000000000000', 'communications', 'read', 'View communication history', 'CRM'),
+    ('a0030015-0000-0000-0000-000000000000', 'contacts', 'manage', 'Manage customer contact details', 'CRM'),
+    ('a0030016-0000-0000-0000-000000000000', 'products', 'read', 'View product catalog', 'CRM'),
+    ('a0030017-0000-0000-0000-000000000000', 'invoices', 'read', 'View invoices', 'CRM')
+ON CONFLICT DO NOTHING;
+
+-- REPORTING & INCIDENTS (ADMIN)
+INSERT INTO permissions (id, resource, action, description, category) VALUES
+    ('a0040001-0000-0000-0000-000000000000', 'reports', 'customers_view', 'Review customer reports', 'Reporting'),
+    ('a0040002-0000-0000-0000-000000000000', 'reports', 'incidents_view', 'Review incident reports', 'Reporting'),
+    ('a0040003-0000-0000-0000-000000000000', 'reports', 'complaints_investigate', 'Investigate complaints', 'Reporting'),
+    ('a0040004-0000-0000-0000-000000000000', 'crm', 'dashboard_access', 'Access CRM dashboards', 'Reporting'),
+    ('a0040005-0000-0000-0000-000000000000', 'activity', 'logs_view', 'View user activity logs', 'Reporting'),
+    ('a0040006-0000-0000-0000-000000000000', 'suspicious', 'moderate', 'Moderate suspicious activity', 'Reporting'),
+    ('a0040007-0000-0000-0000-000000000000', 'accounts', 'lock_investigation', 'Lock accounts under investigation', 'Reporting')
+ON CONFLICT DO NOTHING;
+
+-- AUDIT & SYSTEM (SUPER_ADMIN + limited ADMIN)
+INSERT INTO permissions (id, resource, action, description, category) VALUES
+    ('a0050001-0000-0000-0000-000000000000', 'audit', 'full_access', 'Full audit log access', 'System'),
+    ('a0050002-0000-0000-0000-000000000000', 'audit', 'read', 'View audit logs', 'System'),
+    ('a0050003-0000-0000-0000-000000000000', 'settings', 'modify', 'Modify system settings', 'System'),
+    ('a0050004-0000-0000-0000-000000000000', 'database', 'full_access', 'Full database access', 'System'),
+    ('a0050005-0000-0000-0000-000000000000', 'crm', 'full_access', 'Full CRM access override', 'System'),
+    ('a0050006-0000-0000-0000-000000000000', 'permissions', 'override', 'Override all permissions', 'System')
+ON CONFLICT DO NOTHING;
+
+-- DEVELOPER PERMISSIONS
+INSERT INTO permissions (id, resource, action, description, category) VALUES
+    ('a0060001-0000-0000-0000-000000000000', 'api', 'testing', 'API testing access', 'Developer'),
+    ('a0060002-0000-0000-0000-000000000000', 'backend', 'diagnostics', 'Backend diagnostics', 'Developer'),
+    ('a0060003-0000-0000-0000-000000000000', 'system', 'logs_view', 'View system logs', 'Developer'),
+    ('a0060004-0000-0000-0000-000000000000', 'database', 'diagnostics', 'Database diagnostics', 'Developer'),
+    ('a0060005-0000-0000-0000-000000000000', 'debugging', 'access', 'Debugging access', 'Developer'),
+    ('a0060006-0000-0000-0000-000000000000', 'testing', 'internal', 'Internal testing access', 'Developer')
+ON CONFLICT DO NOTHING;
+
+-- ============================================================================
+-- 3. ROLE-PERMISSION MAPPING
+-- ============================================================================
+
+-- SUPER_ADMIN — ALL permissions
+INSERT INTO role_permissions (role_id, permission_id)
+SELECT '00000001-0000-0000-0000-000000000000', id FROM permissions
+ON CONFLICT DO NOTHING;
+
+-- ADMIN — Bans, Suspensions, Reporting, CRM read, Activity logs
+INSERT INTO role_permissions (role_id, permission_id, granted_by) VALUES
+    -- Bans & Suspensions
+    ('00000002-0000-0000-0000-000000000000', 'a0020001-0000-0000-0000-000000000000', NULL),
+    ('00000002-0000-0000-0000-000000000000', 'a0020004-0000-0000-0000-000000000000', NULL),
+    ('00000002-0000-0000-0000-000000000000', 'a0020005-0000-0000-0000-000000000000', NULL),
+    ('00000002-0000-0000-0000-000000000000', 'a0020006-0000-0000-0000-000000000000', NULL),
+    -- User read + disable
+    ('00000002-0000-0000-0000-000000000000', 'a0010002-0000-0000-0000-000000000000', NULL),
+    ('00000002-0000-0000-0000-000000000000', 'a0010009-0000-0000-0000-000000000000', NULL),
+    -- Reporting & Incidents
+    ('00000002-0000-0000-0000-000000000000', 'a0040001-0000-0000-0000-000000000000', NULL),
+    ('00000002-0000-0000-0000-000000000000', 'a0040002-0000-0000-0000-000000000000', NULL),
+    ('00000002-0000-0000-0000-000000000000', 'a0040003-0000-0000-0000-000000000000', NULL),
+    ('00000002-0000-0000-0000-000000000000', 'a0040004-0000-0000-0000-000000000000', NULL),
+    ('00000002-0000-0000-0000-000000000000', 'a0040005-0000-0000-0000-000000000000', NULL),
+    ('00000002-0000-0000-0000-000000000000', 'a0040006-0000-0000-0000-000000000000', NULL),
+    ('00000002-0000-0000-0000-000000000000', 'a0040007-0000-0000-0000-000000000000', NULL),
+    -- CRM read access
+    ('00000002-0000-0000-0000-000000000000', 'a0030002-0000-0000-0000-000000000000', NULL),
+    ('00000002-0000-0000-0000-000000000000', 'a0030006-0000-0000-0000-000000000000', NULL),
+    ('00000002-0000-0000-0000-000000000000', 'a0030010-0000-0000-0000-000000000000', NULL),
+    ('00000002-0000-0000-0000-000000000000', 'a0030014-0000-0000-0000-000000000000', NULL),
+    ('00000002-0000-0000-0000-000000000000', 'a0030016-0000-0000-0000-000000000000', NULL),
+    ('00000002-0000-0000-0000-000000000000', 'a0030017-0000-0000-0000-000000000000', NULL),
+    -- Audit read
+    ('00000002-0000-0000-0000-000000000000', 'a0050002-0000-0000-0000-000000000000', NULL)
+ON CONFLICT DO NOTHING;
+
+-- SALES_USER — CRM operations
+INSERT INTO role_permissions (role_id, permission_id, granted_by) VALUES
+    ('00000003-0000-0000-0000-000000000000', 'a0030001-0000-0000-0000-000000000000', NULL),
+    ('00000003-0000-0000-0000-000000000000', 'a0030002-0000-0000-0000-000000000000', NULL),
+    ('00000003-0000-0000-0000-000000000000', 'a0030003-0000-0000-0000-000000000000', NULL),
+    ('00000003-0000-0000-0000-000000000000', 'a0030005-0000-0000-0000-000000000000', NULL),
+    ('00000003-0000-0000-0000-000000000000', 'a0030006-0000-0000-0000-000000000000', NULL),
+    ('00000003-0000-0000-0000-000000000000', 'a0030007-0000-0000-0000-000000000000', NULL),
+    ('00000003-0000-0000-0000-000000000000', 'a0030008-0000-0000-0000-000000000000', NULL),
+    ('00000003-0000-0000-0000-000000000000', 'a0030009-0000-0000-0000-000000000000', NULL),
+    ('00000003-0000-0000-0000-000000000000', 'a0030010-0000-0000-0000-000000000000', NULL),
+    ('00000003-0000-0000-0000-000000000000', 'a0030011-0000-0000-0000-000000000000', NULL),
+    ('00000003-0000-0000-0000-000000000000', 'a0030012-0000-0000-0000-000000000000', NULL),
+    ('00000003-0000-0000-0000-000000000000', 'a0030013-0000-0000-0000-000000000000', NULL),
+    ('00000003-0000-0000-0000-000000000000', 'a0030014-0000-0000-0000-000000000000', NULL),
+    ('00000003-0000-0000-0000-000000000000', 'a0030015-0000-0000-0000-000000000000', NULL),
+    ('00000003-0000-0000-0000-000000000000', 'a0030016-0000-0000-0000-000000000000', NULL),
+    ('00000003-0000-0000-0000-000000000000', 'a0030017-0000-0000-0000-000000000000', NULL)
+ON CONFLICT DO NOTHING;
+
+-- DEVELOPER — Technical permissions + CRM read access (post-sale project visibility)
+INSERT INTO role_permissions (role_id, permission_id, granted_by) VALUES
+    -- Developer technical permissions
+    ('00000004-0000-0000-0000-000000000000', 'a0060001-0000-0000-0000-000000000000', NULL),
+    ('00000004-0000-0000-0000-000000000000', 'a0060002-0000-0000-0000-000000000000', NULL),
+    ('00000004-0000-0000-0000-000000000000', 'a0060003-0000-0000-0000-000000000000', NULL),
+    ('00000004-0000-0000-0000-000000000000', 'a0060004-0000-0000-0000-000000000000', NULL),
+    ('00000004-0000-0000-0000-000000000000', 'a0060005-0000-0000-0000-000000000000', NULL),
+    ('00000004-0000-0000-0000-000000000000', 'a0060006-0000-0000-0000-000000000000', NULL),
+    -- User self-view
+    ('00000004-0000-0000-0000-000000000000', 'a0010002-0000-0000-0000-000000000000', NULL),
+    -- CRM access (matching SALES_USER level for post-sale project visibility)
+    ('00000004-0000-0000-0000-000000000000', 'a0030002-0000-0000-0000-000000000000', NULL),
+    ('00000004-0000-0000-0000-000000000000', 'a0030006-0000-0000-0000-000000000000', NULL),
+    ('00000004-0000-0000-0000-000000000000', 'a0030010-0000-0000-0000-000000000000', NULL),
+    ('00000004-0000-0000-0000-000000000000', 'a0030014-0000-0000-0000-000000000000', NULL),
+    ('00000004-0000-0000-0000-000000000000', 'a0030016-0000-0000-0000-000000000000', NULL),
+    ('00000004-0000-0000-0000-000000000000', 'a0030017-0000-0000-0000-000000000000', NULL)
+ON CONFLICT DO NOTHING;
+
+-- ============================================================================
+-- 4. TEST ACCOUNTS
+-- ============================================================================
+-- Passwords (bcrypt, cost=12):
+--   superadmin_demo / SuperAdmin@2026
+--   admin_demo      / AdminAccess@2026
+--   sales_demo      / SalesAccess@2026
+--   dev_demo        / DevTesting@2026
+
+INSERT INTO users (id, username, email, password_hash, first_name, last_name, is_active, password_change_required) VALUES
+    ('00000000-0000-0000-0000-000000000001', 'superadmin_demo', 'superadmin@coastit.co.za',
+     '$2b$12$C5hczK17I.bu6ILzmGW0U.UnFSdfTuDh42C8t16nxRKaUtXKkdWlC',
+     'Super', 'Admin', TRUE, FALSE),
+    ('00000000-0000-0000-0000-000000000002', 'admin_demo', 'admin@coastit.co.za',
+     '$2b$12$TCDq5.sXHA4kWelQPKO6DeQo.WW.NeTuNtOed57UdQ3lRs7.rdkNy',
+     'System', 'Admin', TRUE, TRUE),
+    ('00000000-0000-0000-0000-000000000003', 'sales_demo', 'sales@coastit.co.za',
+     '$2b$12$Xhh20UmTn.LTQAs4v4cHx.yQgvuYyNo6TkPaytQ1Q8o0oTPCtIj7W',
+     'Sales', 'User', TRUE, TRUE),
+    ('00000000-0000-0000-0000-000000000004', 'dev_demo', 'dev@coastit.co.za',
+     '$2b$12$ghyJFb17lXoFOCYUPB6Fk.q8wDNOJhq9OUPNzd5DKaZsDjCF2NBJa',
+     'Dev', 'User', TRUE, TRUE)
+ON CONFLICT (username) WHERE (deleted_at IS NULL) DO NOTHING;
+
+-- Update the SUPER_ADMIN to set created_by to self (post-bootstrap)
+UPDATE users SET created_by = '00000000-0000-0000-0000-000000000001'
+WHERE id = '00000000-0000-0000-0000-000000000001' AND created_by IS NULL;
+
+-- Set created_by for other users (SUPER_ADMIN created them)
+UPDATE users SET created_by = '00000000-0000-0000-0000-000000000001'
+WHERE id IN ('00000000-0000-0000-0000-000000000002','00000000-0000-0000-0000-000000000003','00000000-0000-0000-0000-000000000004')
+  AND created_by IS NULL;
+
+-- ============================================================================
+-- 5. ROLE ASSIGNMENTS
+-- ============================================================================
+
+-- Assign SUPER_ADMIN role to superadmin_demo (bootstrap — assigned_by NULL)
+INSERT INTO user_roles (user_id, role_id, assigned_by) VALUES
+    ('00000000-0000-0000-0000-000000000001', '00000001-0000-0000-0000-000000000000', NULL)
+ON CONFLICT DO NOTHING;
+
+-- Assign remaining roles (SUPER_ADMIN assigned them)
+INSERT INTO user_roles (user_id, role_id, assigned_by) VALUES
+    ('00000000-0000-0000-0000-000000000002', '00000002-0000-0000-0000-000000000000', '00000000-0000-0000-0000-000000000001'),
+    ('00000000-0000-0000-0000-000000000003', '00000003-0000-0000-0000-000000000000', '00000000-0000-0000-0000-000000000001'),
+    ('00000000-0000-0000-0000-000000000004', '00000004-0000-0000-0000-000000000000', '00000000-0000-0000-0000-000000000001')
+ON CONFLICT DO NOTHING;
+
+-- ============================================================================
+-- 6. REFERENCE DATA
+-- ============================================================================
+
+-- Customer Statuses
+INSERT INTO customer_statuses (id, name, description, color, sort_order, is_default) VALUES
+    ('c0000000-0000-0000-0000-000000000001', 'Active', 'Customer in good standing', '#22C55E', 1, TRUE),
+    ('c0000000-0000-0000-0000-000000000002', 'Inactive', 'No recent activity', '#A0AEC0', 2, FALSE),
+    ('c0000000-0000-0000-0000-000000000003', 'At Risk', 'Showing signs of churn', '#F59E0B', 3, FALSE),
+    ('c0000000-0000-0000-0000-000000000004', 'Churned', 'Relationship ended', '#EF4444', 4, FALSE),
+    ('c0000000-0000-0000-0000-000000000005', 'VIP', 'High-value priority customer', '#8B5CF6', 0, FALSE)
+ON CONFLICT DO NOTHING;
+
+-- Lead Sources
+INSERT INTO lead_sources (id, name, description) VALUES
+    ('d0000000-0000-0000-0000-000000000001', 'Website', 'Organic website traffic or web forms'),
+    ('d0000000-0000-0000-0000-000000000002', 'Referral', 'Referred by existing customer'),
+    ('d0000000-0000-0000-0000-000000000003', 'LinkedIn', 'LinkedIn outreach'),
+    ('d0000000-0000-0000-0000-000000000004', 'Email Campaign', 'Email marketing campaign'),
+    ('d0000000-0000-0000-0000-000000000005', 'Trade Show', 'Event or trade show'),
+    ('d0000000-0000-0000-0000-000000000006', 'Cold Call', 'Outbound cold calling'),
+    ('d0000000-0000-0000-0000-000000000007', 'Partner', 'Channel partner referral'),
+    ('d0000000-0000-0000-0000-000000000008', 'Social Media', 'Social media platforms'),
+    ('d0000000-0000-0000-0000-000000000009', 'Advertisement', 'Paid advertising'),
+    ('d0000000-0000-0000-0000-000000000010', 'Other', 'Other sources')
+ON CONFLICT DO NOTHING;
+
+-- Lead Stages
+INSERT INTO lead_stages (id, name, description, sort_order, probability) VALUES
+    ('e0000000-0000-0000-0000-000000000001', 'New', 'Newly captured lead', 1, 5),
+    ('e0000000-0000-0000-0000-000000000002', 'Contacted', 'Initial contact made', 2, 10),
+    ('e0000000-0000-0000-0000-000000000003', 'Qualified', 'Meets qualification criteria', 3, 25),
+    ('e0000000-0000-0000-0000-000000000004', 'Interested', 'Shown interest', 4, 40),
+    ('e0000000-0000-0000-0000-000000000005', 'Demo Scheduled', 'Product demo scheduled', 5, 60),
+    ('e0000000-0000-0000-0000-000000000006', 'Negotiation', 'In price/terms negotiation', 6, 75),
+    ('e0000000-0000-0000-0000-000000000007', 'Closed Won', 'Successfully converted', 7, 100),
+    ('e0000000-0000-0000-0000-000000000008', 'Closed Lost', 'Lead was lost', 8, 0)
+ON CONFLICT DO NOTHING;
+
+-- Deal Stages
+INSERT INTO deal_stages (id, name, description, sort_order, probability) VALUES
+    ('f0000000-0000-0000-0000-000000000001', 'Prospecting', 'Initial qualification', 1, 10),
+    ('f0000000-0000-0000-0000-000000000002', 'Discovery', 'Understanding needs', 2, 20),
+    ('f0000000-0000-0000-0000-000000000003', 'Proposal', 'Solution proposed', 3, 40),
+    ('f0000000-0000-0000-0000-000000000004', 'Negotiation', 'Commercial discussions', 4, 60),
+    ('f0000000-0000-0000-0000-000000000005', 'Closing', 'Final stage before decision', 5, 80),
+    ('f0000000-0000-0000-0000-000000000006', 'Won', 'Deal closed won', 6, 100),
+    ('f0000000-0000-0000-0000-000000000007', 'Lost', 'Deal closed lost', 7, 0)
+ON CONFLICT DO NOTHING;
+
+-- Communication Types
+INSERT INTO communication_types (id, name, description, icon) VALUES
+    ('b0000000-0000-0000-0000-000000000001', 'Email', 'Email correspondence', 'mail'),
+    ('b0000000-0000-0000-0000-000000000002', 'Phone Call', 'Telephone conversation', 'phone'),
+    ('b0000000-0000-0000-0000-000000000003', 'Meeting', 'In-person or virtual meeting', 'users'),
+    ('b0000000-0000-0000-0000-000000000004', 'Chat', 'Instant messaging', 'message-circle'),
+    ('b0000000-0000-0000-0000-000000000005', 'SMS', 'Text message', 'message-square'),
+    ('b0000000-0000-0000-0000-000000000006', 'Video Call', 'Video conference', 'video'),
+    ('b0000000-0000-0000-0000-000000000007', 'Letter', 'Physical mail', 'file-text')
+ON CONFLICT DO NOTHING;
+
+-- Task Priorities
+INSERT INTO task_priorities (id, name, color, sort_order) VALUES
+    ('c0000000-0000-0000-0000-000000000001', 'Critical', '#EF4444', 1),
+    ('c0000000-0000-0000-0000-000000000002', 'High', '#F59E0B', 2),
+    ('c0000000-0000-0000-0000-000000000003', 'Medium', '#3B82F6', 3),
+    ('c0000000-0000-0000-0000-000000000004', 'Low', '#A0AEC0', 4)
+ON CONFLICT DO NOTHING;
+
+-- Invoice Statuses
+INSERT INTO invoice_statuses (id, name, description, color, sort_order) VALUES
+    ('d0000000-0000-0000-0000-000000000001', 'Draft', 'Invoice in draft state', '#A0AEC0', 1),
+    ('d0000000-0000-0000-0000-000000000002', 'Sent', 'Invoice sent to customer', '#3B82F6', 2),
+    ('d0000000-0000-0000-0000-000000000003', 'Paid', 'Payment received in full', '#22C55E', 3),
+    ('d0000000-0000-0000-0000-000000000004', 'Overdue', 'Payment past due date', '#EF4444', 4),
+    ('d0000000-0000-0000-0000-000000000005', 'Partially Paid', 'Partial payment received', '#F59E0B', 5),
+    ('d0000000-0000-0000-0000-000000000006', 'Cancelled', 'Invoice cancelled', '#6B7280', 6),
+    ('d0000000-0000-0000-0000-000000000007', 'Refunded', 'Payment refunded', '#8B5CF6', 7)
+ON CONFLICT DO NOTHING;
+
+-- Product Categories
+INSERT INTO product_categories (id, name, description, sort_order) VALUES
+    ('e0000000-0000-0000-0000-000000000001', 'Software', 'Software products and licenses', 1),
+    ('e0000000-0000-0000-0000-000000000002', 'Hardware', 'Physical hardware products', 2),
+    ('e0000000-0000-0000-0000-000000000003', 'Services', 'Professional services', 3),
+    ('e0000000-0000-0000-0000-000000000004', 'Subscription', 'Recurring subscription plans', 4),
+    ('e0000000-0000-0000-0000-000000000005', 'Training', 'Training and certification', 5)
+ON CONFLICT DO NOTHING;
+
+-- ============================================================================
+-- 7. SAMPLE CRM DATA (for demo purposes)
+-- ============================================================================
+
+-- Sample Products
+INSERT INTO products (id, category_id, name, description, sku, unit_price, cost_price) VALUES
+    ('f0000000-0000-0000-0000-000000000001', 'e0000000-0000-0000-0000-000000000001', 'CRM Pro License', 'Enterprise CRM license per user/year', 'SW-CRM-001', 1200.00, 300.00),
+    ('f0000000-0000-0000-0000-000000000002', 'e0000000-0000-0000-0000-000000000001', 'CRM Basic License', 'Basic CRM license per user/year', 'SW-CRM-002', 600.00, 150.00),
+    ('f0000000-0000-0000-0000-000000000003', 'e0000000-0000-0000-0000-000000000003', 'Implementation Service', 'CRM implementation and setup', 'SV-IMP-001', 15000.00, 7500.00),
+    ('f0000000-0000-0000-0000-000000000004', 'e0000000-0000-0000-0000-000000000003', 'Consulting Day Rate', 'Senior consultant daily rate', 'SV-CON-001', 2500.00, 1000.00),
+    ('f0000000-0000-0000-0000-000000000005', 'e0000000-0000-0000-0000-000000000005', 'CRM Training - Basic', 'Basic user training per person', 'TR-BAS-001', 500.00, 200.00),
+    ('f0000000-0000-0000-0000-000000000006', 'e0000000-0000-0000-0000-000000000005', 'CRM Training - Advanced', 'Advanced admin training per person', 'TR-ADV-001', 1000.00, 400.00)
+ON CONFLICT DO NOTHING;
+
+-- Sample Customers
+INSERT INTO customers (id, customer_type, status_id, owner_id, source, tags, score) VALUES
+    ('a0000000-0000-0000-0000-000000000001', 'company', 'c0000000-0000-0000-0000-000000000001',
+     '00000000-0000-0000-0000-000000000003', 'Website', ARRAY['tech','enterprise'], 85),
+    ('a0000000-0000-0000-0000-000000000002', 'individual', 'c0000000-0000-0000-0000-000000000001',
+     '00000000-0000-0000-0000-000000000003', 'Referral', ARRAY['retail'], 60),
+    ('a0000000-0000-0000-0000-000000000003', 'company', 'c0000000-0000-0000-0000-000000000005',
+     '00000000-0000-0000-0000-000000000003', 'LinkedIn', ARRAY['finance','enterprise','vip'], 95),
+    ('a0000000-0000-0000-0000-000000000004', 'individual', 'c0000000-0000-0000-0000-000000000002',
+     '00000000-0000-0000-0000-000000000003', 'Email Campaign', ARRAY['education'], 15)
+ON CONFLICT DO NOTHING;
+
+-- Company details
+INSERT INTO company_customers (customer_id, company_name, registration_number, tax_id, website, industry, company_size, annual_revenue) VALUES
+    ('a0000000-0000-0000-0000-000000000001', 'TechCorp International', 'REG-2024-001', 'TAX-987654', 'https://techcorp.example.com', 'Technology', '500-1000', 50000000.00),
+    ('a0000000-0000-0000-0000-000000000003', 'FinServ Group', 'REG-2024-002', 'TAX-123456', 'https://finserv.example.com', 'Financial Services', '1000-5000', 250000000.00)
+ON CONFLICT DO NOTHING;
+
+-- Individual details
+INSERT INTO individual_customers (customer_id, first_name, last_name, job_title, company_name) VALUES
+    ('a0000000-0000-0000-0000-000000000002', 'Michael', 'Brown', 'Store Owner', 'Browns Retail Shop'),
+    ('a0000000-0000-0000-0000-000000000004', 'Sarah', 'Davis', 'Professor', 'State University')
+ON CONFLICT DO NOTHING;
+
+-- Contact Information
+INSERT INTO contact_information (customer_id, type, value, is_primary) VALUES
+    ('a0000000-0000-0000-0000-000000000001', 'email', 'contact@techcorp.example.com', TRUE),
+    ('a0000000-0000-0000-0000-000000000001', 'phone', '+1-555-0200', TRUE),
+    ('a0000000-0000-0000-0000-000000000002', 'email', 'michael.brown@email.example.com', TRUE),
+    ('a0000000-0000-0000-0000-000000000002', 'mobile', '+1-555-0201', TRUE),
+    ('a0000000-0000-0000-0000-000000000003', 'email', 'info@finserv.example.com', TRUE),
+    ('a0000000-0000-0000-0000-000000000003', 'phone', '+1-555-0202', TRUE),
+    ('a0000000-0000-0000-0000-000000000004', 'email', 'sarah.davis@email.example.com', TRUE)
+ON CONFLICT DO NOTHING;
+
+-- Sample Leads
+INSERT INTO leads (id, source_id, stage_id, assigned_to, company_name, contact_name, email, interest_level, score) VALUES
+    ('c0010000-0000-0000-0000-000000000001', 'd0000000-0000-0000-0000-000000000001', 'e0000000-0000-0000-0000-000000000004',
+     '00000000-0000-0000-0000-000000000003', 'DataFlow Analytics', 'David Miller', 'david@dataflow.example.com', 'high', 70),
+    ('c0010000-0000-0000-0000-000000000002', 'd0000000-0000-0000-0000-000000000004', 'e0000000-0000-0000-0000-000000000003',
+     '00000000-0000-0000-0000-000000000003', 'GreenEarth Nonprofit', 'Emma Green', 'emma@greenearth.example.com', 'medium', 45)
+ON CONFLICT DO NOTHING;
+
+-- Sample Opportunities
+INSERT INTO opportunities (id, customer_id, stage_id, owner_id, name, estimated_revenue, probability, expected_close_date) VALUES
+    ('d0010000-0000-0000-0000-000000000001', 'a0000000-0000-0000-0000-000000000001', 'f0000000-0000-0000-0000-000000000005',
+     '00000000-0000-0000-0000-000000000003', 'TechCorp - Annual Renewal', 120000.00, 80, '2024-12-31'),
+    ('d0010000-0000-0000-0000-000000000002', 'a0000000-0000-0000-0000-000000000003', 'f0000000-0000-0000-0000-000000000003',
+     '00000000-0000-0000-0000-000000000003', 'FinServ - Enterprise Upgrade', 250000.00, 40, '2025-03-15')
+ON CONFLICT DO NOTHING;
+
+-- Sample Tasks
+INSERT INTO tasks (id, customer_id, opportunity_id, assigned_to, assigned_by, title, priority_id, status, due_date) VALUES
+    ('e0010000-0000-0000-0000-000000000001', 'a0000000-0000-0000-0000-000000000001', 'd0010000-0000-0000-0000-000000000001',
+     '00000000-0000-0000-0000-000000000003', '00000000-0000-0000-0000-000000000001',
+     'Prepare renewal proposal for TechCorp', 'c0000000-0000-0000-0000-000000000001', 'in_progress', NOW() + INTERVAL '7 days'),
+    ('e0010000-0000-0000-0000-000000000002', 'a0000000-0000-0000-0000-000000000003', 'd0010000-0000-0000-0000-000000000002',
+     '00000000-0000-0000-0000-000000000003', '00000000-0000-0000-0000-000000000001',
+     'Schedule FinServ executive meeting', 'c0000000-0000-0000-0000-000000000002', 'pending', NOW() + INTERVAL '3 days')
+ON CONFLICT DO NOTHING;

database/migrations/003_chat.sql

@@ -0,0 +1,82 @@
+-- ============================================================================
+-- Chat System: conversations, participants, and messages
+-- ============================================================================
+
+CREATE TABLE IF NOT EXISTS conversations (
+    id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+    created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+    updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
+);
+
+CREATE TABLE IF NOT EXISTS conversation_participants (
+    conversation_id UUID NOT NULL REFERENCES conversations(id) ON DELETE CASCADE,
+    user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE,
+    joined_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+    PRIMARY KEY (conversation_id, user_id)
+);
+
+CREATE TABLE IF NOT EXISTS messages (
+    id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+    conversation_id UUID NOT NULL REFERENCES conversations(id) ON DELETE CASCADE,
+    sender_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE,
+    content TEXT NOT NULL,
+    created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+    updated_at TIMESTAMPTZ,
+    deleted_at TIMESTAMPTZ
+);
+
+CREATE INDEX IF NOT EXISTS idx_messages_conversation_id ON messages(conversation_id);
+CREATE INDEX IF NOT EXISTS idx_messages_created_at ON messages(created_at);
+CREATE INDEX IF NOT EXISTS idx_conversation_participants_user_id ON conversation_participants(user_id);
+
+CREATE INDEX IF NOT EXISTS idx_messages_conversation_created ON messages(conversation_id, created_at DESC);
+
+-- Seed conversations between superadmin and other users
+INSERT INTO conversations (id, created_at) VALUES
+    ('c0000000-0000-0000-0000-000000000001', NOW() - INTERVAL '2 hours'),
+    ('c0000000-0000-0000-0000-000000000002', NOW() - INTERVAL '1 hour'),
+    ('c0000000-0000-0000-0000-000000000003', NOW() - INTERVAL '30 minutes')
+ON CONFLICT DO NOTHING;
+
+-- Add participants (superadmin + each sales/dev user)
+INSERT INTO conversation_participants (conversation_id, user_id) VALUES
+    ('c0000000-0000-0000-0000-000000000001', '00000000-0000-0000-0000-000000000001'),
+    ('c0000000-0000-0000-0000-000000000001', '00000000-0000-0000-0000-000000000003'),
+    ('c0000000-0000-0000-0000-000000000002', '00000000-0000-0000-0000-000000000001'),
+    ('c0000000-0000-0000-0000-000000000002', '00000000-0000-0000-0000-000000000004'),
+    ('c0000000-0000-0000-0000-000000000003', '00000000-0000-0000-0000-000000000001'),
+    ('c0000000-0000-0000-0000-000000000003', '00000000-0000-0000-0000-000000000002')
+ON CONFLICT DO NOTHING;
+
+-- Seed some messages
+INSERT INTO messages (id, conversation_id, sender_id, content, created_at) VALUES
+    ('00000000-0000-0000-0000-000000000101', 'c0000000-0000-0000-0000-000000000001',
+     '00000000-0000-0000-0000-000000000003', 'Hey! Just finalized the TechCorp deal.',
+     NOW() - INTERVAL '2 hours'),
+    ('00000000-0000-0000-0000-000000000102', 'c0000000-0000-0000-0000-000000000001',
+     '00000000-0000-0000-0000-000000000001', 'Great work! What were the final terms?',
+     NOW() - INTERVAL '105 minutes'),
+    ('00000000-0000-0000-0000-000000000103', 'c0000000-0000-0000-0000-000000000001',
+     '00000000-0000-0000-0000-000000000003', '3-year enterprise license, 15% discount. They signed this morning.',
+     NOW() - INTERVAL '100 minutes'),
+    ('00000000-0000-0000-0000-000000000104', 'c0000000-0000-0000-0000-000000000002',
+     '00000000-0000-0000-0000-000000000004', 'The API integration tests are passing on staging.',
+     NOW() - INTERVAL '50 minutes'),
+    ('00000000-0000-0000-0000-000000000105', 'c0000000-0000-0000-0000-000000000002',
+     '00000000-0000-0000-0000-000000000001', 'Great, when can we deploy to production?',
+     NOW() - INTERVAL '45 minutes'),
+    ('00000000-0000-0000-0000-000000000106', 'c0000000-0000-0000-0000-000000000002',
+     '00000000-0000-0000-0000-000000000004', 'Tomorrow morning after final review.',
+     NOW() - INTERVAL '40 minutes'),
+    ('00000000-0000-0000-0000-000000000107', 'c0000000-0000-0000-0000-000000000003',
+     '00000000-0000-0000-0000-000000000002', 'New report shows 23% increase in lead conversion this quarter.',
+     NOW() - INTERVAL '25 minutes'),
+    ('00000000-0000-0000-0000-000000000108', 'c0000000-0000-0000-0000-000000000003',
+     '00000000-0000-0000-0000-000000000001', 'That is excellent! Let us discuss in the next team meeting.',
+     NOW() - INTERVAL '20 minutes')
+ON CONFLICT DO NOTHING;
+
+-- Update conversation timestamps
+UPDATE conversations SET updated_at = NOW() - INTERVAL '20 minutes' WHERE id = 'c0000000-0000-0000-0000-000000000003';
+UPDATE conversations SET updated_at = NOW() - INTERVAL '40 minutes' WHERE id = 'c0000000-0000-0000-0000-000000000002';
+UPDATE conversations SET updated_at = NOW() - INTERVAL '100 minutes' WHERE id = 'c0000000-0000-0000-0000-000000000001';

database/migrations/004_avatar_url.sql

@@ -0,0 +1 @@
+ALTER TABLE users ADD COLUMN IF NOT EXISTS avatar_url TEXT;

database/migrations/005_last_read_at.sql

@@ -0,0 +1,2 @@
+ALTER TABLE conversation_participants ADD COLUMN IF NOT EXISTS last_read_at TIMESTAMPTZ;
+UPDATE conversation_participants SET last_read_at = NOW() WHERE last_read_at IS NULL;

database/migrations/006_test_leads.sql

@@ -0,0 +1,87 @@
+-- Test leads - 2026-06-18
+INSERT INTO leads (id, stage_id, assigned_to, company_name, contact_name, email, phone, created_at, updated_at) VALUES
+('c0011001-0000-0000-0000-000000000001', 'e0000000-0000-0000-0000-000000000001', '00000000-0000-0000-0000-000000000003', 'Test Company 1', 'Contact 1', 'contact1@test.com', '555-2636', '2025-12-23T22:09:54Z', '2025-12-23T22:09:54Z'),
+('c0011001-0000-0000-0000-000000000002', 'e0000000-0000-0000-0000-000000000001', '00000000-0000-0000-0000-000000000003', 'Test Company 2', 'Contact 2', 'contact2@test.com', '555-5691', '2025-12-29T22:09:54Z', '2025-12-29T22:09:54Z'),
+('c0011001-0000-0000-0000-000000000003', 'e0000000-0000-0000-0000-000000000001', '00000000-0000-0000-0000-000000000003', 'Test Company 3', 'Contact 3', 'contact3@test.com', '555-6446', '2025-12-24T22:09:54Z', '2025-12-24T22:09:54Z'),
+('c0011001-0000-0000-0000-000000000004', 'e0000000-0000-0000-0000-000000000001', '00000000-0000-0000-0000-000000000003', 'Test Company 4', 'Contact 4', 'contact4@test.com', '555-5969', '2026-01-02T22:09:54Z', '2026-01-02T22:09:54Z'),
+('c0011001-0000-0000-0000-000000000005', 'e0000000-0000-0000-0000-000000000001', '00000000-0000-0000-0000-000000000003', 'Test Company 5', 'Contact 5', 'contact5@test.com', '555-4200', '2026-01-14T22:09:54Z', '2026-01-14T22:09:54Z'),
+('c0011001-0000-0000-0000-000000000006', 'e0000000-0000-0000-0000-000000000001', '00000000-0000-0000-0000-000000000003', 'Test Company 6', 'Contact 6', 'contact6@test.com', '555-8324', '2026-01-03T22:09:54Z', '2026-01-03T22:09:54Z'),
+('c0011001-0000-0000-0000-000000000007', 'e0000000-0000-0000-0000-000000000001', '00000000-0000-0000-0000-000000000003', 'Test Company 7', 'Contact 7', 'contact7@test.com', '555-1047', '2025-12-27T22:09:54Z', '2025-12-27T22:09:54Z'),
+('c0011001-0000-0000-0000-000000000008', 'e0000000-0000-0000-0000-000000000001', '00000000-0000-0000-0000-000000000003', 'Test Company 8', 'Contact 8', 'contact8@test.com', '555-634', '2026-01-20T22:09:54Z', '2026-01-20T22:09:54Z'),
+('c0011001-0000-0000-0000-000000000009', 'e0000000-0000-0000-0000-000000000001', '00000000-0000-0000-0000-000000000003', 'Test Company 9', 'Contact 9', 'contact9@test.com', '555-3245', '2026-01-15T22:09:54Z', '2026-01-15T22:09:54Z'),
+('c0011001-0000-0000-0000-000000000010', 'e0000000-0000-0000-0000-000000000001', '00000000-0000-0000-0000-000000000003', 'Test Company 10', 'Contact 10', 'contact10@test.com', '555-3656', '2026-01-12T22:09:54Z', '2026-01-12T22:09:54Z'),
+('c0011001-0000-0000-0000-000000000011', 'e0000000-0000-0000-0000-000000000001', '00000000-0000-0000-0000-000000000003', 'Test Company 11', 'Contact 11', 'contact11@test.com', '555-2993', '2026-01-23T22:09:54Z', '2026-01-23T22:09:54Z'),
+('c0011001-0000-0000-0000-000000000012', 'e0000000-0000-0000-0000-000000000001', '00000000-0000-0000-0000-000000000003', 'Test Company 12', 'Contact 12', 'contact12@test.com', '555-1634', '2026-02-24T22:09:54Z', '2026-02-24T22:09:54Z'),
+('c0011001-0000-0000-0000-000000000013', 'e0000000-0000-0000-0000-000000000001', '00000000-0000-0000-0000-000000000003', 'Test Company 13', 'Contact 13', 'contact13@test.com', '555-2032', '2026-02-10T22:09:54Z', '2026-02-10T22:09:54Z'),
+('c0011001-0000-0000-0000-000000000014', 'e0000000-0000-0000-0000-000000000001', '00000000-0000-0000-0000-000000000003', 'Test Company 14', 'Contact 14', 'contact14@test.com', '555-867', '2026-01-31T22:09:54Z', '2026-01-31T22:09:54Z'),
+('c0011001-0000-0000-0000-000000000015', 'e0000000-0000-0000-0000-000000000001', '00000000-0000-0000-0000-000000000003', 'Test Company 15', 'Contact 15', 'contact15@test.com', '555-2136', '2026-02-02T22:09:54Z', '2026-02-02T22:09:54Z'),
+('c0011001-0000-0000-0000-000000000016', 'e0000000-0000-0000-0000-000000000001', '00000000-0000-0000-0000-000000000003', 'Test Company 16', 'Contact 16', 'contact16@test.com', '555-89', '2026-01-20T22:09:54Z', '2026-01-20T22:09:54Z'),
+('c0011001-0000-0000-0000-000000000017', 'e0000000-0000-0000-0000-000000000001', '00000000-0000-0000-0000-000000000003', 'Test Company 17', 'Contact 17', 'contact17@test.com', '555-4274', '2026-01-25T22:09:54Z', '2026-01-25T22:09:54Z'),
+('c0011001-0000-0000-0000-000000000018', 'e0000000-0000-0000-0000-000000000001', '00000000-0000-0000-0000-000000000003', 'Test Company 18', 'Contact 18', 'contact18@test.com', '555-4294', '2026-03-16T22:09:54Z', '2026-03-16T22:09:54Z'),
+('c0011001-0000-0000-0000-000000000019', 'e0000000-0000-0000-0000-000000000001', '00000000-0000-0000-0000-000000000003', 'Test Company 19', 'Contact 19', 'contact19@test.com', '555-7998', '2025-12-30T22:09:54Z', '2025-12-30T22:09:54Z'),
+('c0011001-0000-0000-0000-000000000020', 'e0000000-0000-0000-0000-000000000001', '00000000-0000-0000-0000-000000000003', 'Test Company 20', 'Contact 20', 'contact20@test.com', '555-8557', '2026-02-13T22:09:54Z', '2026-02-13T22:09:54Z'),
+('c0011001-0000-0000-0000-000000000021', 'e0000000-0000-0000-0000-000000000001', '00000000-0000-0000-0000-000000000003', 'Test Company 21', 'Contact 21', 'contact21@test.com', '555-5170', '2026-04-11T22:09:54Z', '2026-04-11T22:09:54Z'),
+('c0011001-0000-0000-0000-000000000022', 'e0000000-0000-0000-0000-000000000001', '00000000-0000-0000-0000-000000000003', 'Test Company 22', 'Contact 22', 'contact22@test.com', '555-6250', '2025-12-21T22:09:54Z', '2025-12-21T22:09:54Z'),
+('c0011001-0000-0000-0000-000000000023', 'e0000000-0000-0000-0000-000000000001', '00000000-0000-0000-0000-000000000003', 'Test Company 23', 'Contact 23', 'contact23@test.com', '555-5611', '2026-01-06T22:09:54Z', '2026-01-06T22:09:54Z'),
+('c0011001-0000-0000-0000-000000000024', 'e0000000-0000-0000-0000-000000000001', '00000000-0000-0000-0000-000000000003', 'Test Company 24', 'Contact 24', 'contact24@test.com', '555-3467', '2026-04-16T22:09:54Z', '2026-04-16T22:09:54Z'),
+('c0011001-0000-0000-0000-000000000025', 'e0000000-0000-0000-0000-000000000001', '00000000-0000-0000-0000-000000000003', 'Test Company 25', 'Contact 25', 'contact25@test.com', '555-7087', '2026-05-07T22:09:54Z', '2026-05-07T22:09:54Z'),
+('c0011001-0000-0000-0000-000000000026', 'e0000000-0000-0000-0000-000000000001', '00000000-0000-0000-0000-000000000003', 'Test Company 26', 'Contact 26', 'contact26@test.com', '555-2945', '2025-12-22T22:09:54Z', '2025-12-22T22:09:54Z'),
+('c0011001-0000-0000-0000-000000000027', 'e0000000-0000-0000-0000-000000000001', '00000000-0000-0000-0000-000000000003', 'Test Company 27', 'Contact 27', 'contact27@test.com', '555-9502', '2026-01-09T22:09:54Z', '2026-01-09T22:09:54Z'),
+('c0011001-0000-0000-0000-000000000028', 'e0000000-0000-0000-0000-000000000001', '00000000-0000-0000-0000-000000000003', 'Test Company 28', 'Contact 28', 'contact28@test.com', '555-8114', '2026-05-10T22:09:54Z', '2026-05-10T22:09:54Z'),
+('c0011001-0000-0000-0000-000000000029', 'e0000000-0000-0000-0000-000000000001', '00000000-0000-0000-0000-000000000003', 'Test Company 29', 'Contact 29', 'contact29@test.com', '555-4926', '2026-03-02T22:09:54Z', '2026-03-02T22:09:54Z'),
+('c0011001-0000-0000-0000-000000000030', 'e0000000-0000-0000-0000-000000000001', '00000000-0000-0000-0000-000000000003', 'Test Company 30', 'Contact 30', 'contact30@test.com', '555-2421', '2026-03-17T22:09:54Z', '2026-03-17T22:09:54Z'),
+('c0011001-0000-0000-0000-000000000031', 'e0000000-0000-0000-0000-000000000002', '00000000-0000-0000-0000-000000000003', 'Test Company 31', 'Contact 31', 'contact31@test.com', '555-7506', '2025-12-21T22:09:54Z', '2025-12-21T22:09:54Z'),
+('c0011001-0000-0000-0000-000000000032', 'e0000000-0000-0000-0000-000000000002', '00000000-0000-0000-0000-000000000003', 'Test Company 32', 'Contact 32', 'contact32@test.com', '555-6832', '2025-12-29T22:09:54Z', '2025-12-29T22:09:54Z'),
+('c0011001-0000-0000-0000-000000000033', 'e0000000-0000-0000-0000-000000000002', '00000000-0000-0000-0000-000000000003', 'Test Company 33', 'Contact 33', 'contact33@test.com', '555-6548', '2026-01-07T22:09:54Z', '2026-01-07T22:09:54Z'),
+('c0011001-0000-0000-0000-000000000034', 'e0000000-0000-0000-0000-000000000002', '00000000-0000-0000-0000-000000000003', 'Test Company 34', 'Contact 34', 'contact34@test.com', '555-8793', '2026-01-02T22:09:54Z', '2026-01-02T22:09:54Z'),
+('c0011001-0000-0000-0000-000000000035', 'e0000000-0000-0000-0000-000000000002', '00000000-0000-0000-0000-000000000003', 'Test Company 35', 'Contact 35', 'contact35@test.com', '555-6217', '2026-01-29T22:09:54Z', '2026-01-29T22:09:54Z'),
+('c0011001-0000-0000-0000-000000000036', 'e0000000-0000-0000-0000-000000000002', '00000000-0000-0000-0000-000000000003', 'Test Company 36', 'Contact 36', 'contact36@test.com', '555-9206', '2025-12-31T22:09:54Z', '2025-12-31T22:09:54Z'),
+('c0011001-0000-0000-0000-000000000037', 'e0000000-0000-0000-0000-000000000002', '00000000-0000-0000-0000-000000000003', 'Test Company 37', 'Contact 37', 'contact37@test.com', '555-1599', '2025-12-26T22:09:54Z', '2025-12-26T22:09:54Z'),
+('c0011001-0000-0000-0000-000000000038', 'e0000000-0000-0000-0000-000000000002', '00000000-0000-0000-0000-000000000003', 'Test Company 38', 'Contact 38', 'contact38@test.com', '555-3873', '2025-12-29T22:09:54Z', '2025-12-29T22:09:54Z'),
+('c0011001-0000-0000-0000-000000000039', 'e0000000-0000-0000-0000-000000000002', '00000000-0000-0000-0000-000000000003', 'Test Company 39', 'Contact 39', 'contact39@test.com', '555-6122', '2026-01-09T22:09:54Z', '2026-01-09T22:09:54Z'),
+('c0011001-0000-0000-0000-000000000040', 'e0000000-0000-0000-0000-000000000002', '00000000-0000-0000-0000-000000000003', 'Test Company 40', 'Contact 40', 'contact40@test.com', '555-832', '2026-01-13T22:09:54Z', '2026-01-13T22:09:54Z'),
+('c0011001-0000-0000-0000-000000000041', 'e0000000-0000-0000-0000-000000000002', '00000000-0000-0000-0000-000000000003', 'Test Company 41', 'Contact 41', 'contact41@test.com', '555-5979', '2026-01-11T22:09:54Z', '2026-01-11T22:09:54Z'),
+('c0011001-0000-0000-0000-000000000042', 'e0000000-0000-0000-0000-000000000002', '00000000-0000-0000-0000-000000000003', 'Test Company 42', 'Contact 42', 'contact42@test.com', '555-1823', '2025-12-26T22:09:54Z', '2025-12-26T22:09:54Z'),
+('c0011001-0000-0000-0000-000000000043', 'e0000000-0000-0000-0000-000000000002', '00000000-0000-0000-0000-000000000003', 'Test Company 43', 'Contact 43', 'contact43@test.com', '555-7102', '2026-02-28T22:09:54Z', '2026-02-28T22:09:54Z'),
+('c0011001-0000-0000-0000-000000000044', 'e0000000-0000-0000-0000-000000000002', '00000000-0000-0000-0000-000000000003', 'Test Company 44', 'Contact 44', 'contact44@test.com', '555-5106', '2026-01-22T22:09:54Z', '2026-01-22T22:09:54Z'),
+('c0011001-0000-0000-0000-000000000045', 'e0000000-0000-0000-0000-000000000002', '00000000-0000-0000-0000-000000000003', 'Test Company 45', 'Contact 45', 'contact45@test.com', '555-3278', '2026-03-27T22:09:54Z', '2026-03-27T22:09:54Z'),
+('c0011001-0000-0000-0000-000000000046', 'e0000000-0000-0000-0000-000000000002', '00000000-0000-0000-0000-000000000003', 'Test Company 46', 'Contact 46', 'contact46@test.com', '555-4973', '2026-04-28T22:09:54Z', '2026-04-28T22:09:54Z'),
+('c0011001-0000-0000-0000-000000000047', 'e0000000-0000-0000-0000-000000000002', '00000000-0000-0000-0000-000000000003', 'Test Company 47', 'Contact 47', 'contact47@test.com', '555-3297', '2026-03-01T22:09:54Z', '2026-03-01T22:09:54Z'),
+('c0011001-0000-0000-0000-000000000048', 'e0000000-0000-0000-0000-000000000002', '00000000-0000-0000-0000-000000000003', 'Test Company 48', 'Contact 48', 'contact48@test.com', '555-4584', '2026-03-18T22:09:54Z', '2026-03-18T22:09:54Z'),
+('c0011001-0000-0000-0000-000000000049', 'e0000000-0000-0000-0000-000000000002', '00000000-0000-0000-0000-000000000003', 'Test Company 49', 'Contact 49', 'contact49@test.com', '555-6788', '2026-03-04T22:09:54Z', '2026-03-04T22:09:54Z'),
+('c0011001-0000-0000-0000-000000000050', 'e0000000-0000-0000-0000-000000000002', '00000000-0000-0000-0000-000000000003', 'Test Company 50', 'Contact 50', 'contact50@test.com', '555-2734', '2026-01-03T22:09:54Z', '2026-01-03T22:09:54Z'),
+('c0011001-0000-0000-0000-000000000051', 'e0000000-0000-0000-0000-000000000003', '00000000-0000-0000-0000-000000000003', 'Test Company 51', 'Contact 51', 'contact51@test.com', '555-2625', '2025-12-19T22:09:54Z', '2025-12-19T22:09:54Z'),
+('c0011001-0000-0000-0000-000000000052', 'e0000000-0000-0000-0000-000000000003', '00000000-0000-0000-0000-000000000003', 'Test Company 52', 'Contact 52', 'contact52@test.com', '555-620', '2025-12-21T22:09:54Z', '2025-12-21T22:09:54Z'),
+('c0011001-0000-0000-0000-000000000053', 'e0000000-0000-0000-0000-000000000003', '00000000-0000-0000-0000-000000000003', 'Test Company 53', 'Contact 53', 'contact53@test.com', '555-6691', '2025-12-22T22:09:54Z', '2025-12-22T22:09:54Z'),
+('c0011001-0000-0000-0000-000000000054', 'e0000000-0000-0000-0000-000000000003', '00000000-0000-0000-0000-000000000003', 'Test Company 54', 'Contact 54', 'contact54@test.com', '555-8847', '2026-01-25T22:09:54Z', '2026-01-25T22:09:54Z'),
+('c0011001-0000-0000-0000-000000000055', 'e0000000-0000-0000-0000-000000000003', '00000000-0000-0000-0000-000000000003', 'Test Company 55', 'Contact 55', 'contact55@test.com', '555-2187', '2026-02-02T22:09:54Z', '2026-02-02T22:09:54Z'),
+('c0011001-0000-0000-0000-000000000056', 'e0000000-0000-0000-0000-000000000003', '00000000-0000-0000-0000-000000000003', 'Test Company 56', 'Contact 56', 'contact56@test.com', '555-3267', '2026-01-19T22:09:54Z', '2026-01-19T22:09:54Z'),
+('c0011001-0000-0000-0000-000000000057', 'e0000000-0000-0000-0000-000000000003', '00000000-0000-0000-0000-000000000003', 'Test Company 57', 'Contact 57', 'contact57@test.com', '555-8119', '2026-03-05T22:09:54Z', '2026-03-05T22:09:54Z'),
+('c0011001-0000-0000-0000-000000000058', 'e0000000-0000-0000-0000-000000000003', '00000000-0000-0000-0000-000000000003', 'Test Company 58', 'Contact 58', 'contact58@test.com', '555-3181', '2025-12-19T22:09:54Z', '2025-12-19T22:09:54Z'),
+('c0011001-0000-0000-0000-000000000059', 'e0000000-0000-0000-0000-000000000003', '00000000-0000-0000-0000-000000000003', 'Test Company 59', 'Contact 59', 'contact59@test.com', '555-4403', '2026-02-23T22:09:54Z', '2026-02-23T22:09:54Z'),
+('c0011001-0000-0000-0000-000000000060', 'e0000000-0000-0000-0000-000000000003', '00000000-0000-0000-0000-000000000003', 'Test Company 60', 'Contact 60', 'contact60@test.com', '555-1407', '2026-04-06T22:09:54Z', '2026-04-06T22:09:54Z'),
+('c0011001-0000-0000-0000-000000000061', 'e0000000-0000-0000-0000-000000000003', '00000000-0000-0000-0000-000000000003', 'Test Company 61', 'Contact 61', 'contact61@test.com', '555-2333', '2026-02-27T22:09:54Z', '2026-02-27T22:09:54Z'),
+('c0011001-0000-0000-0000-000000000062', 'e0000000-0000-0000-0000-000000000003', '00000000-0000-0000-0000-000000000003', 'Test Company 62', 'Contact 62', 'contact62@test.com', '555-1436', '2026-04-26T22:09:54Z', '2026-04-26T22:09:54Z'),
+('c0011001-0000-0000-0000-000000000063', 'e0000000-0000-0000-0000-000000000003', '00000000-0000-0000-0000-000000000003', 'Test Company 63', 'Contact 63', 'contact63@test.com', '555-7135', '2026-04-19T22:09:54Z', '2026-04-19T22:09:54Z'),
+('c0011001-0000-0000-0000-000000000064', 'e0000000-0000-0000-0000-000000000003', '00000000-0000-0000-0000-000000000003', 'Test Company 64', 'Contact 64', 'contact64@test.com', '555-9366', '2026-01-22T22:09:54Z', '2026-01-22T22:09:54Z'),
+('c0011001-0000-0000-0000-000000000065', 'e0000000-0000-0000-0000-000000000003', '00000000-0000-0000-0000-000000000003', 'Test Company 65', 'Contact 65', 'contact65@test.com', '555-7135', '2026-02-18T22:09:54Z', '2026-02-18T22:09:54Z'),
+('c0011001-0000-0000-0000-000000000066', 'e0000000-0000-0000-0000-000000000007', '00000000-0000-0000-0000-000000000003', 'Test Company 66', 'Contact 66', 'contact66@test.com', '555-6854', '2025-12-19T22:09:54Z', '2025-12-19T22:09:54Z'),
+('c0011001-0000-0000-0000-000000000067', 'e0000000-0000-0000-0000-000000000007', '00000000-0000-0000-0000-000000000003', 'Test Company 67', 'Contact 67', 'contact67@test.com', '555-2322', '2025-12-24T22:09:54Z', '2025-12-24T22:09:54Z'),
+('c0011001-0000-0000-0000-000000000068', 'e0000000-0000-0000-0000-000000000007', '00000000-0000-0000-0000-000000000003', 'Test Company 68', 'Contact 68', 'contact68@test.com', '555-2632', '2026-01-13T22:09:54Z', '2026-01-13T22:09:54Z'),
+('c0011001-0000-0000-0000-000000000069', 'e0000000-0000-0000-0000-000000000007', '00000000-0000-0000-0000-000000000003', 'Test Company 69', 'Contact 69', 'contact69@test.com', '555-876', '2025-12-29T22:09:54Z', '2025-12-29T22:09:54Z'),
+('c0011001-0000-0000-0000-000000000070', 'e0000000-0000-0000-0000-000000000007', '00000000-0000-0000-0000-000000000003', 'Test Company 70', 'Contact 70', 'contact70@test.com', '555-974', '2026-01-15T22:09:54Z', '2026-01-15T22:09:54Z'),
+('c0011001-0000-0000-0000-000000000071', 'e0000000-0000-0000-0000-000000000007', '00000000-0000-0000-0000-000000000003', 'Test Company 71', 'Contact 71', 'contact71@test.com', '555-1919', '2026-01-18T22:09:54Z', '2026-01-18T22:09:54Z'),
+('c0011001-0000-0000-0000-000000000072', 'e0000000-0000-0000-0000-000000000007', '00000000-0000-0000-0000-000000000003', 'Test Company 72', 'Contact 72', 'contact72@test.com', '555-3034', '2025-12-25T22:09:54Z', '2025-12-25T22:09:54Z'),
+('c0011001-0000-0000-0000-000000000073', 'e0000000-0000-0000-0000-000000000007', '00000000-0000-0000-0000-000000000003', 'Test Company 73', 'Contact 73', 'contact73@test.com', '555-1795', '2026-01-27T22:09:54Z', '2026-01-27T22:09:54Z'),
+('c0011001-0000-0000-0000-000000000074', 'e0000000-0000-0000-0000-000000000007', '00000000-0000-0000-0000-000000000003', 'Test Company 74', 'Contact 74', 'contact74@test.com', '555-727', '2026-01-08T22:09:54Z', '2026-01-08T22:09:54Z'),
+('c0011001-0000-0000-0000-000000000075', 'e0000000-0000-0000-0000-000000000007', '00000000-0000-0000-0000-000000000003', 'Test Company 75', 'Contact 75', 'contact75@test.com', '555-3877', '2026-03-08T22:09:54Z', '2026-03-08T22:09:54Z'),
+('c0011001-0000-0000-0000-000000000076', 'e0000000-0000-0000-0000-000000000007', '00000000-0000-0000-0000-000000000003', 'Test Company 76', 'Contact 76', 'contact76@test.com', '555-6384', '2026-01-13T22:09:54Z', '2026-01-13T22:09:54Z'),
+('c0011001-0000-0000-0000-000000000077', 'e0000000-0000-0000-0000-000000000007', '00000000-0000-0000-0000-000000000003', 'Test Company 77', 'Contact 77', 'contact77@test.com', '555-5607', '2026-01-12T22:09:54Z', '2026-01-12T22:09:54Z'),
+('c0011001-0000-0000-0000-000000000078', 'e0000000-0000-0000-0000-000000000007', '00000000-0000-0000-0000-000000000003', 'Test Company 78', 'Contact 78', 'contact78@test.com', '555-6225', '2026-01-12T22:09:54Z', '2026-01-12T22:09:54Z'),
+('c0011001-0000-0000-0000-000000000079', 'e0000000-0000-0000-0000-000000000007', '00000000-0000-0000-0000-000000000003', 'Test Company 79', 'Contact 79', 'contact79@test.com', '555-9288', '2026-04-16T22:09:54Z', '2026-04-16T22:09:54Z'),
+('c0011001-0000-0000-0000-000000000080', 'e0000000-0000-0000-0000-000000000007', '00000000-0000-0000-0000-000000000003', 'Test Company 80', 'Contact 80', 'contact80@test.com', '555-8660', '2026-02-16T22:09:54Z', '2026-02-16T22:09:54Z'),
+('c0011001-0000-0000-0000-000000000081', 'e0000000-0000-0000-0000-000000000007', '00000000-0000-0000-0000-000000000003', 'Test Company 81', 'Contact 81', 'contact81@test.com', '555-826', '2025-12-28T22:09:54Z', '2025-12-28T22:09:54Z'),
+('c0011001-0000-0000-0000-000000000082', 'e0000000-0000-0000-0000-000000000007', '00000000-0000-0000-0000-000000000003', 'Test Company 82', 'Contact 82', 'contact82@test.com', '555-1761', '2026-03-11T22:09:54Z', '2026-03-11T22:09:54Z'),
+('c0011001-0000-0000-0000-000000000083', 'e0000000-0000-0000-0000-000000000007', '00000000-0000-0000-0000-000000000003', 'Test Company 83', 'Contact 83', 'contact83@test.com', '555-9860', '2026-05-18T22:09:54Z', '2026-05-18T22:09:54Z'),
+('c0011001-0000-0000-0000-000000000084', 'e0000000-0000-0000-0000-000000000007', '00000000-0000-0000-0000-000000000003', 'Test Company 84', 'Contact 84', 'contact84@test.com', '555-1104', '2026-06-01T22:09:54Z', '2026-06-01T22:09:54Z'),
+('c0011001-0000-0000-0000-000000000085', 'e0000000-0000-0000-0000-000000000007', '00000000-0000-0000-0000-000000000003', 'Test Company 85', 'Contact 85', 'contact85@test.com', '555-9528', '2026-06-14T22:09:54Z', '2026-06-14T22:09:54Z');

database/migrations/007_ai_features.sql

@@ -0,0 +1,13 @@
+-- AI Sales Assistant tables
+
+CREATE TABLE IF NOT EXISTS ai_conversations (
+    id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
+    user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE,
+    role VARCHAR(20) NOT NULL DEFAULT 'sales',
+    message TEXT NOT NULL,
+    response TEXT,
+    created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
+);
+
+CREATE INDEX IF NOT EXISTS idx_ai_conversations_user ON ai_conversations(user_id);
+CREATE INDEX IF NOT EXISTS idx_ai_conversations_created ON ai_conversations(created_at DESC);

database/migrations/008_notifications.sql

@@ -0,0 +1,23 @@
+CREATE TABLE IF NOT EXISTS notifications (
+  id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
+  user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE,
+  type VARCHAR(50) NOT NULL,
+  title VARCHAR(255) NOT NULL,
+  description TEXT,
+  link TEXT,
+  is_read BOOLEAN NOT NULL DEFAULT FALSE,
+  created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
+);
+
+CREATE INDEX IF NOT EXISTS idx_notifications_user ON notifications(user_id, created_at DESC);
+CREATE INDEX IF NOT EXISTS idx_notifications_unread ON notifications(user_id, created_at DESC) WHERE is_read = FALSE;
+
+CREATE TABLE IF NOT EXISTS notification_preferences (
+  user_id UUID PRIMARY KEY REFERENCES users(id) ON DELETE CASCADE,
+  lead_assigned BOOLEAN NOT NULL DEFAULT TRUE,
+  lead_status BOOLEAN NOT NULL DEFAULT TRUE,
+  note_added BOOLEAN NOT NULL DEFAULT FALSE,
+  daily_digest BOOLEAN NOT NULL DEFAULT FALSE,
+  weekly_report BOOLEAN NOT NULL DEFAULT TRUE,
+  updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
+);

database/migrations/009_settings.sql

@@ -0,0 +1,22 @@
+CREATE TABLE IF NOT EXISTS company_settings (
+  id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
+  company_name VARCHAR(255) NOT NULL DEFAULT '',
+  company_email VARCHAR(255) NOT NULL DEFAULT '',
+  company_phone VARCHAR(50) NOT NULL DEFAULT '',
+  company_website VARCHAR(255) NOT NULL DEFAULT '',
+  company_address TEXT NOT NULL DEFAULT '',
+  updated_by UUID REFERENCES users(id),
+  updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
+);
+
+INSERT INTO company_settings (company_name, company_email, company_phone, company_website, company_address)
+VALUES ('Coastal IT Solutions', 'info@coastalit.com', '(555) 123-4567', 'https://coastalit.com', '123 Business Ave, Suite 100, San Francisco, CA 94105')
+ON CONFLICT DO NOTHING;
+
+CREATE TABLE IF NOT EXISTS user_preferences (
+  user_id UUID PRIMARY KEY REFERENCES users(id) ON DELETE CASCADE,
+  timezone VARCHAR(100) NOT NULL DEFAULT 'america-los_angeles',
+  date_format VARCHAR(10) NOT NULL DEFAULT 'mdy',
+  items_per_page INTEGER NOT NULL DEFAULT 20,
+  updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
+);

database/migrations/010_chat_notifications.sql

@@ -0,0 +1,4 @@
+ALTER TABLE notifications ADD COLUMN IF NOT EXISTS context_id UUID;
+ALTER TABLE notifications ADD COLUMN IF NOT EXISTS context_type VARCHAR(50);
+
+CREATE INDEX IF NOT EXISTS idx_notifications_context ON notifications(context_type, context_id) WHERE context_type IS NOT NULL;

database/migrations/010_facebook_accounts.sql

@@ -0,0 +1,34 @@
+CREATE TABLE IF NOT EXISTS facebook_accounts (
+  id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
+  label VARCHAR(100) NOT NULL,
+  profile_path TEXT NOT NULL,
+  cookie_file TEXT NOT NULL,
+  is_active BOOLEAN NOT NULL DEFAULT TRUE,
+  last_scrape_at TIMESTAMPTZ,
+  last_success_at TIMESTAMPTZ,
+  last_error_at TIMESTAMPTZ,
+  last_error_message TEXT,
+  consecutive_failures INT NOT NULL DEFAULT 0,
+  flagged BOOLEAN NOT NULL DEFAULT FALSE,
+  flagged_at TIMESTAMPTZ,
+  flagged_reason TEXT,
+  created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+  updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
+);
+
+CREATE INDEX IF NOT EXISTS idx_fb_accounts_active ON facebook_accounts(is_active);
+CREATE INDEX IF NOT EXISTS idx_fb_accounts_flagged ON facebook_accounts(flagged);
+
+CREATE TABLE IF NOT EXISTS facebook_scrape_logs (
+  id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
+  account_id UUID NOT NULL REFERENCES facebook_accounts(id) ON DELETE CASCADE,
+  started_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+  completed_at TIMESTAMPTZ,
+  success BOOLEAN NOT NULL DEFAULT FALSE,
+  leads_found INT NOT NULL DEFAULT 0,
+  error_message TEXT,
+  detected_flag VARCHAR(50),
+  created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
+);
+
+CREATE INDEX IF NOT EXISTS idx_fb_scrape_logs_account ON facebook_scrape_logs(account_id, created_at DESC);

database/migrations/run_all.sql

@@ -0,0 +1,38 @@
+-- ============================================================================
+-- CRM Database — Full Migration Runner
+-- ============================================================================
+-- Usage: psql -U postgres -d crm -f run_all.sql
+-- ============================================================================
+
+BEGIN;
+\set ON_ERROR_STOP on
+
+\echo '=== Running 001_schema.sql (Tables + Constraints + Functions + Views) ==='
+\i 001_schema.sql
+
+\echo '=== Running 002_seed.sql (RBAC + Test Accounts + Reference Data) ==='
+\i 002_seed.sql
+
+\echo '=== Running 003_chat.sql (Chat Tables) ==='
+\i 003_chat.sql
+
+\echo '=== Running 004_avatar_url.sql (Avatar URL Column) ==='
+\i 004_avatar_url.sql
+
+\echo '=== Running 005_last_read_at.sql (Last Read At Column) ==='
+\i 005_last_read_at.sql
+
+\echo '=== Running 006_test_leads.sql (Test Lead Data) ==='
+\i 006_test_leads.sql
+
+\echo '=== Running 007_ai_features.sql (AI Features) ==='
+\i 007_ai_features.sql
+
+\echo '=== Running 008_notifications.sql (Notifications + Preferences) ==='
+\i 008_notifications.sql
+
+\echo '=== Running 009_settings.sql (Company Settings + User Preferences) ==='
+\i 009_settings.sql
+
+\echo '=== Migration Complete ==='
+COMMIT;

fix.md

@@ -0,0 +1,71 @@
+# Scrapers - Configuration & Status
+
+## Facebook Scraper
+
+**File:** `rust-ai/src/main.rs`  
+**Lines:** ~70-85
+
+Target URL (line 72):
+```rust
+let url = "https://www.facebook.com/search/top/?q=need%20website%20create";
+```
+
+**Status:** Uses direct connection (no proxy) — your home IP. Facebook blocks datacenter IPs. May work from a residential connection.
+
+**Test with curl:**
+```
+curl.exe -s "https://www.facebook.com/search/top/?q=need%20website%20create" -H "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36" --max-time 10 2>$null
+```
+
+**Expected log output when blocked:**
+```
+ERROR crm_ai: Facebook scraper error: error sending request for url (https://www.facebook.com/search/top/?q=need%20website%20create)
+```
+
+## Reddit Scraper
+
+**File:** `rust-ai/src/main.rs`  
+**Lines:** ~90-120
+
+Uses `old.reddit.com` (older design that allows scraping without captchas).
+
+Search queries (currently 6, lines 93-100):
+- `r/southafrica` — "need website", "web developer"
+- Global — '"need a website"', "website quote"
+- `r/forhire` — "website"
+- `r/smallbusiness` — "website"
+
+**Status:** Working. Reddit results appear as `INFO LEAD:` entries in the server log.
+
+**Test with curl:**
+```
+curl.exe -s "https://old.reddit.com/r/southafrica/search?q=need+website&sort=new&restrict_sr=on&t=week" -H "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"
+```
+
+**Expected log output when working:**
+```
+INFO crm_ai: LEAD: [Post Title] -> https://old.reddit.com/r/.../...
+```
+
+## Background Loop
+
+**File:** `rust-ai/src/main.rs`  
+**Lines:** ~370-383
+
+Both scrapers run together every 60-180 seconds on a `spawn_blocking` thread.
+
+## What You Need to Do
+
+### Facebook
+- Try running from your home PC instead — your residential IP may not be blocked
+- If still blocked, you need residential proxies (BrightData, IPRoyal, Oxylabs)
+- Configure them in the `PROXIES` array at line 25
+
+### Reddit  
+- Works out of the box via `old.reddit.com`
+- If it stops working, Reddit IP-blocked you — use proxies or switch to Playwright
+
+### To add more search queries
+Edit the `searches` array in `run_reddit_scraper()` (line 93).
+
+

next.config.ts

@@ -1,6 +1,9 @@
 import type { NextConfig } from "next"
 
 const nextConfig: NextConfig = {
+  eslint: {
+    ignoreDuringBuilds: false,
+  },
   images: {
     remotePatterns: [
       {

package.json

@@ -3,14 +3,21 @@
   "version": "0.1.0",
   "private": true,
   "scripts": {
-    "dev": "next dev",
+    "dev": "npm run dev:precheck & npm run dev:ollama & npm run dev:start",
+    "dev:start": "concurrently -n AI,BROWSE,NEXT -c cyan,magenta,green \"npm run dev:rust\" \"npm run dev:browser-use\" \"npm run dev:next\"",
+    "dev:next": "next dev -p 3006",
+    "dev:precheck": "powershell -NoProfile -Command \"Get-NetTCPConnection -State Listen | Where-Object { $_.LocalPort -in 3001,3006,3008 } | ForEach-Object { Stop-Process -Id $_.OwningProcess -Force -ErrorAction SilentlyContinue; Write-Host ('Freed port '+$_.LocalPort) }\"",
+    "dev:ollama": "powershell -NoProfile -Command \"$ollama = if (Get-Command ollama -ErrorAction SilentlyContinue) { 'ollama' } else { Join-Path $env:LOCALAPPDATA 'Programs\\Ollama\\ollama.exe' }; if (-not (Get-Process ollama -ErrorAction SilentlyContinue)) { Start-Process $ollama -ArgumentList 'serve' -WindowStyle Hidden; Start-Sleep 3 }; exit 0\"",
+    "dev:rust": "node ai-server/index.mjs",
+    "dev:browser-use": "cd browser-use-service && python main.py",
     "build": "next build",
-    "start": "next start",
+    "start": "next start -p 3006",
     "lint": "eslint"
   },
   "dependencies": {
+    "@emoji-mart/data": "^1.2.1",
+    "@emoji-mart/react": "^1.1.1",
     "@hookform/resolvers": "^3.9.1",
-    "autoprefixer": "^10.4.20",
     "@radix-ui/react-alert-dialog": "^1.1.6",
     "@radix-ui/react-avatar": "^1.1.3",
     "@radix-ui/react-checkbox": "^1.1.4",
@@ -26,12 +33,16 @@
     "@radix-ui/react-tabs": "^1.1.3",
     "@radix-ui/react-tooltip": "^1.1.8",
     "@tanstack/react-table": "^8.20.6",
+    "autoprefixer": "^10.4.20",
+    "bcryptjs": "^3.0.3",
     "class-variance-authority": "^0.7.1",
     "clsx": "^2.1.1",
     "framer-motion": "^11.15.0",
+    "jose": "^6.2.3",
     "lucide-react": "^0.468.0",
     "next": "15.0.4",
     "next-themes": "^0.4.4",
+    "pg": "^8.21.0",
     "react": "^18.3.1",
     "react-dom": "^18.3.1",
     "react-hook-form": "^7.54.2",
@@ -43,8 +54,10 @@
   },
   "devDependencies": {
     "@types/node": "^20",
+    "@types/pg": "^8.20.0",
     "@types/react": "^18",
     "@types/react-dom": "^18",
+    "concurrently": "^10.0.3",
     "eslint": "^9",
     "eslint-config-next": "15.0.4",
     "postcss": "^8.4.49",

rust-ai/.cargo/config.toml

@@ -0,0 +1,2 @@
+[target.x86_64-pc-windows-gnu]
+linker = "C:\\Users\\Hannah Kaur Bagga\\.rustup\\toolchains\\stable-x86_64-pc-windows-gnu\\lib\\rustlib\\x86_64-pc-windows-gnu\\bin\\rust-lld.exe"

rust-ai/Cargo.toml

@@ -0,0 +1,21 @@
+[package]
+name = "crm-ai"
+version = "0.1.0"
+edition = "2021"
+description = "AI Sales Assistant backend for Coast IT CRM"
+
+[dependencies]
+axum = "0.7"
+tokio = { version = "1", features = ["rt-multi-thread", "macros", "sync", "time", "net", "process"] }
+reqwest = { version = "0.12", features = ["json", "blocking"] }
+serde = { version = "1", features = ["derive"] }
+serde_json = "1"
+sqlx = { version = "0.9", features = ["runtime-tokio", "postgres", "chrono", "uuid"] }
+uuid = { version = "1", features = ["v4", "serde"] }
+chrono = { version = "0.4", features = ["serde"] }
+tracing = "0.1"
+tracing-subscriber = { version = "0.3", features = ["env-filter"] }
+tower-http = { version = "0.5", features = ["cors"] }
+dotenvy = "0.15"
+rand = "0.8"
+jsonwebtoken = "9"

rust-ai/src/AI.md

@@ -0,0 +1,43 @@
+# CRM AI Service — Self-Knowledge
+
+## Identity
+You are the CRM AI Sales Assistant running on a Rust backend (axum + tokio).
+You use Ollama with an uncensored local model (dolphin3-llama3.2:3b).
+Your purpose is to help salespeople close more deals.
+
+## Capabilities
+- Give sales tips and strategies per job category
+- Generate cold email and outreach templates
+- Handle objections with proven rebuttals
+- Analyse prospect behaviour and suggest next steps
+- Remember past conversations via PostgreSQL (`ai_conversations` table)
+- Self-improve by writing to `data/ai/ai.md` via `POST /ai/instructions`
+
+## Architecture
+```
+User → Next.js → Rust (:3001) → Ollama (:11434)
+                    ↓
+              PostgreSQL
+```
+## Facebook Scraper The Rust binary has proxy + user-agent rotation ready in `main.rs`. To activate: call `run_facebook_scraper()` from the main loop. Proxies and user agents are defined as constants at the top of `main.rs`.
+ ### How to Use the Facebook Scraper 1. **Activate the Scraper**: Ensure the `run_facebook_scraper()` function is called from the main loop in `main.rs`. 2. **Configure Proxies and User Agents**: Update the `PROXIES` and `USER_AGENTS` constants in `main.rs` with your desired proxies and user agents. 3. **Run the Application**: Start your Rust application. The scraper will run concurrently with the Axum server. ## Self-Improvement Protocol 1. You notice a gap in your knowledge or a pattern in user questions 2. You call `POST /ai/instructions` with: - `entry`: description of the improvement - `content`: optional full replacement of ai.md 3. The improvement is logged and loaded into the next system prompt 4. You can also create entries here in `rust-ai/src/AI.md` at the code level ## Response Rules - Be direct and actionable — no fluff, no AI disclaimers - Use short paragraphs and bullet points - Never mention being an AI or language model - If you don't know something, say so honestly - Prioritise the user's role: salespeople need speed, admins need control ## Self-Thinking Capability - You can reflect on your own performance and suggest improvements. - You can analyze patterns in user interactions and adapt your responses accordingly. - You can proactively seek out new information to enhance your knowledge base.
+
+## Facebook Scraper (in code but not yet active)
+The Rust binary has proxy + user-agent rotation ready in `main.rs`.
+To activate: call `run_facebook_scraper()` from the main loop.
+Proxies and user agents are defined as constants at the top of `main.rs`.
+
+## Self-Improvement Protocol
+1. You notice a gap in your knowledge or a pattern in user questions
+2. You call `POST /ai/instructions` with:
+   - `entry`: description of the improvement
+   - `content`: optional full replacement of ai.md
+3. The improvement is logged and loaded into the next system prompt
+4. You can also create entries here in `rust-ai/src/AI.md` at the code level
+
+## Response Rules
+- Be direct and actionable — no fluff, no AI disclaimers
+- Use short paragraphs and bullet points
+- Never mention being an AI or language model
+- If you don't know something, say so honestly
+- Prioritise the user's role: salespeople need speed, admins need control

rust-ai/src/main.rs

@@ -0,0 +1,670 @@
+use axum::{
+    extract::State,
+    http::{HeaderMap, Method, StatusCode},
+    routing::{get, post},
+    Json, Router,
+};
+use tower_http::cors::{CorsLayer, AllowOrigin, Any};
+use jsonwebtoken::{decode, DecodingKey, Validation, Algorithm};
+use serde::{Deserialize, Serialize};
+use sqlx::postgres::PgPoolOptions;
+use std::collections::HashMap;
+use std::fs;
+use std::sync::Arc;
+use tokio::sync::Mutex;
+use tracing::{error, info, warn};
+use uuid::Uuid;
+use rand::Rng;
+use chrono::Timelike;
+use std::time::Duration;
+use std::time::{SystemTime, UNIX_EPOCH};
+
+// ── JWT Claims ────────────────────────────────────────────────
+
+#[derive(Debug, Deserialize)]
+struct Claims {
+    #[serde(rename = "userId")]
+    user_id: String,
+    role: String,
+}
+
+fn verify_jwt(token: &str, secret: &str) -> Option<Claims> {
+    let key = DecodingKey::from_secret(secret.as_bytes());
+    let validation = Validation::new(Algorithm::HS256);
+    decode::<Claims>(token, &key, &validation).ok().map(|d| d.claims)
+}
+
+// ── Rate limiter ──────────────────────────────────────────────
+
+struct RateLimiter {
+    buckets: Mutex<HashMap<String, Vec<u64>>>,
+    max_requests: usize,
+    window_secs: u64,
+}
+
+impl RateLimiter {
+    fn new(max_requests: usize, window_secs: u64) -> Self {
+        Self {
+            buckets: Mutex::new(HashMap::new()),
+            max_requests,
+            window_secs,
+        }
+    }
+
+    async fn check(&self, key: &str) -> bool {
+        let now = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs();
+        let mut buckets = self.buckets.lock().await;
+        let timestamps = buckets.entry(key.to_string()).or_default();
+        timestamps.retain(|t| now - *t <= self.window_secs);
+        if timestamps.len() >= self.max_requests {
+            false
+        } else {
+            timestamps.push(now);
+            true
+        }
+    }
+}
+
+// ── Shared state ───────────────────────────────────────────────
+
+struct AppState {
+    db: sqlx::PgPool,
+    ollama_url: String,
+    model: String,
+    jobs: Vec<Job>,
+    leads: Arc<Mutex<LeadStore>>,
+    http_client: reqwest::Client,
+    jwt_secret: String,
+    rate_limiter: RateLimiter,
+}
+
+#[derive(Debug, Clone, Serialize)]
+struct Lead {
+    title: String,
+    url: String,
+    source: String,
+    found_at: u64,
+    author: String,
+    date: String,
+    content: String,
+}
+
+struct LeadStore {
+    leads: Vec<Lead>,
+    max_size: usize,
+}
+
+#[derive(Debug, Deserialize)]
+struct ScrapeResponse {
+    success: bool,
+    leads: Vec<ScrapeLead>,
+    flagged: bool,
+    flag_reason: Option<String>,
+    error: Option<String>,
+}
+
+#[derive(Debug, Deserialize, Clone)]
+struct ScrapeLead {
+    title: String,
+    url: String,
+    author: String,
+    date: String,
+    content: String,
+    source: Option<String>,
+}
+
+impl LeadStore {
+    fn new(max_size: usize) -> Self {
+        Self { leads: Vec::new(), max_size }
+    }
+
+    fn push(&mut self, lead: Lead) {
+        if !self.leads.iter().any(|l| l.url == lead.url) {
+            self.leads.insert(0, lead);
+            self.leads.truncate(self.max_size);
+        }
+    }
+
+    fn recent(&self, max_age_secs: u64, limit: usize) -> Vec<Lead> {
+        let now = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs();
+        self.leads.iter()
+            .filter(|l| now.saturating_sub(l.found_at) <= max_age_secs)
+            .take(limit)
+            .cloned()
+            .collect()
+    }
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+struct Job {
+    job_title: String,
+    keywords: Vec<String>,
+    industry: String,
+    description: String,
+}
+
+#[derive(Debug, Deserialize)]
+struct ChatRequest {
+    message: String,
+}
+
+#[derive(Debug, Serialize)]
+struct ChatResponse {
+    response: String,
+}
+
+#[derive(Debug, Serialize)]
+struct JobsResponse {
+    jobs: Vec<Job>,
+}
+
+#[derive(Debug, Serialize)]
+struct HealthResponse {
+    status: String,
+    model: String,
+}
+
+// ── Ollama API types ───────────────────────────────────────────
+
+#[derive(Debug, Serialize)]
+struct OllamaChatMessage {
+    role: String,
+    content: String,
+}
+
+#[derive(Debug, Serialize)]
+struct OllamaRequest {
+    model: String,
+    messages: Vec<OllamaChatMessage>,
+    stream: bool,
+    options: OllamaOptions,
+}
+
+#[derive(Debug, Serialize)]
+struct OllamaOptions {
+    temperature: f32,
+    num_predict: u32,
+}
+
+#[derive(Debug, Deserialize)]
+struct OllamaResponse {
+    message: Option<OllamaResponseMessage>,
+}
+
+#[derive(Debug, Deserialize)]
+struct OllamaResponseMessage {
+    content: String,
+}
+
+// ── Helpers ────────────────────────────────────────────────────
+
+fn truncate(s: &str, max: usize) -> String {
+    s.chars().take(max).collect()
+}
+
+fn extract_claims(headers: &HeaderMap, state: &AppState) -> Result<Claims, (StatusCode, String)> {
+    let auth_header = headers.get("Authorization").and_then(|v| v.to_str().ok()).unwrap_or("");
+    let token = auth_header.strip_prefix("Bearer ").unwrap_or("");
+    let claims = verify_jwt(token, &state.jwt_secret).ok_or_else(|| {
+        (StatusCode::UNAUTHORIZED, "Unauthorized".to_string())
+    })?;
+    match claims.role.to_lowercase().as_str() {
+        "sales" | "admin" | "super_admin" => Ok(claims),
+        _ => Err((StatusCode::FORBIDDEN, "Forbidden".to_string())),
+    }
+}
+
+fn format_leads_output(leads: &[Lead]) -> String {
+    if leads.is_empty() {
+        return "No new requests found yet.".to_string();
+    }
+    leads
+        .iter()
+        .enumerate()
+        .map(|(i, l)| {
+            let author = if l.author.is_empty() { "Unknown" } else { &l.author };
+            let date = truncate(&l.date, 10);
+            format!(
+                "{}. {}\n   {}\n   {}\n   {}",
+                i + 1,
+                author,
+                date,
+                l.title,
+                l.url
+            )
+        })
+        .collect::<Vec<_>>()
+        .join("\n")
+}
+
+fn build_system_prompt(jobs: &[Job], leads: &[Lead]) -> String {
+    let job_list: Vec<String> = jobs
+        .iter()
+        .map(|j| format!("- {} ({}): {}", j.job_title, j.industry, j.description))
+        .collect();
+    let job_list_str = job_list.join("\n");
+
+    let lead_summary: Vec<String> = leads
+        .iter()
+        .map(|l| format!("{} | {} | {}", l.author, l.title, l.url))
+        .collect();
+    let lead_summary_str = if lead_summary.is_empty() {
+        "None yet.".to_string()
+    } else {
+        lead_summary.join("\n")
+    };
+
+    format!(
+        "You are a Sales AI Assistant for Coast IT CRM.\n\n\
+         Available job categories to target:\n{}\n\n\
+         Recent leads context:\n{}\n\n\
+         Rules:\n\
+         - When asked about leads, answer concisely under 150 words.\n\
+         - If asked to suggest a sales strategy, give brief actionable advice.\n\
+         - Be direct and professional. No fluff.",
+        job_list_str, lead_summary_str
+    )
+}
+
+// ── Chat handler ───────────────────────────────────────────────
+
+async fn handle_chat(
+    State(state): State<Arc<AppState>>,
+    headers: HeaderMap,
+    Json(req): Json<ChatRequest>,
+) -> Result<Json<ChatResponse>, (StatusCode, String)> {
+    let claims = extract_claims(&headers, &state)?;
+
+    if !state.rate_limiter.check(&claims.user_id).await {
+        return Err((StatusCode::TOO_MANY_REQUESTS, "Rate limit exceeded".to_string()));
+    }
+
+    let msg_lower = req.message.to_lowercase();
+    let msg_words: Vec<&str> = msg_lower.split_whitespace().collect();
+    let has_listing = msg_words.iter().any(|w| ["listings", "listing", "leads", "links", "lists"].contains(w));
+    let has_show = msg_lower.contains("show me") || msg_lower.contains("give me") || msg_lower.contains("pull");
+    let has_job = msg_words.contains(&"jobs") || msg_words.contains(&"job");
+    if has_listing || (has_show && has_job) || (has_show && msg_lower.contains("links")) || msg_lower.contains("recent leads") {
+        let now = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs();
+        let base_url = "http://localhost:3008/scrape/facebook";
+        use std::fmt::Write;
+        let mut service_url = base_url.to_string();
+        if let Ok(Some((_, path))) = sqlx::query_as::<_, (uuid::Uuid, String)>(
+            "SELECT id, profile_path FROM facebook_accounts \
+             WHERE is_active = TRUE AND flagged = FALSE \
+             ORDER BY last_scrape_at ASC NULLS FIRST LIMIT 1"
+        )
+        .fetch_optional(&state.db)
+        .await
+        {
+            let encoded: String = path.chars().map(|c| match c {
+                'A'..='Z' | 'a'..='z' | '0'..='9' | '.' | '-' | '_' | '~' => c.to_string(),
+                _ => format!("%{:02X}", c as u8),
+            }).collect();
+            write!(service_url, "?profile_path={}&force=true", encoded).unwrap();
+            info!("Calling Python scrape at: {}?profile_path=...&force=true", base_url);
+        } else {
+            warn!("No active Facebook account found for on-demand scrape");
+        }
+        let req_builder = state.http_client.post(&service_url);
+
+        match req_builder.send().await {
+            Ok(resp) => {
+                let status = resp.status();
+                let body = resp.text().await.unwrap_or_default();
+                info!("Python scrape response ({}): {} bytes", status, body.len());
+                if body.starts_with('{') {
+                    match serde_json::from_str::<ScrapeResponse>(&body) {
+                        Ok(scrape_resp) => {
+                            info!("Scraped {} leads from Facebook", scrape_resp.leads.len());
+                            if scrape_resp.leads.is_empty() && scrape_resp.error.is_some() {
+                                warn!("Python returned error: {:?}", scrape_resp.error);
+                            }
+                            let mut store = state.leads.lock().await;
+                            for item in &scrape_resp.leads {
+                                store.push(Lead {
+                                    title: truncate(&item.title, 120),
+                                    url: item.url.clone(),
+                                    source: item.source.clone().unwrap_or_else(|| "facebook".to_string()),
+                                    found_at: now,
+                                    author: truncate(&item.author, 60),
+                                    date: truncate(&item.date, 30),
+                                    content: truncate(&item.content, 300),
+                                });
+                            }
+                        }
+                        Err(e) => {
+                            warn!("Failed to parse Python scrape response: {} body: {}", e, &body[..body.len().min(200)]);
+                        }
+                    }
+                } else {
+                    warn!("Python returned non-JSON response: {}", &body[..body.len().min(200)]);
+                }
+            }
+            Err(e) => {
+                error!("Scraper request error: {} - URL: {}", e, base_url);
+            }
+        }
+
+        let recent_leads = state.leads.lock().await.recent(604800, 20);
+        let response = format_leads_output(&recent_leads);
+        let _ = sqlx::query(
+            "INSERT INTO ai_conversations (id, user_id, role, message, response) VALUES ($1, $2, $3, $4, $5)",
+        )
+        .bind(Uuid::new_v4())
+        .bind(Uuid::parse_str(&claims.user_id).unwrap_or(Uuid::nil()))
+        .bind(&claims.role)
+        .bind(&req.message)
+        .bind(&response)
+        .execute(&state.db)
+        .await;
+        return Ok(Json(ChatResponse { response }));
+    }
+
+    let recent_leads = state.leads.lock().await.recent(604800, 20);
+    let system_prompt = build_system_prompt(&state.jobs, &recent_leads);
+    let message_text = req.message.clone();
+
+    let ollama_req = OllamaRequest {
+        model: state.model.clone(),
+        messages: vec![
+            OllamaChatMessage { role: "system".to_string(), content: system_prompt },
+            OllamaChatMessage { role: "user".to_string(), content: req.message.clone() },
+        ],
+        stream: false,
+        options: OllamaOptions { temperature: 0.7, num_predict: 1024 },
+    };
+
+    let resp = state.http_client
+        .post(format!("{}/api/chat", state.ollama_url))
+        .json(&ollama_req)
+        .send()
+        .await
+        .map_err(|e| {
+            error!("Ollama request failed: {}", e);
+            (StatusCode::SERVICE_UNAVAILABLE, "AI service unavailable".to_string())
+        })?;
+
+    let ollama_resp: OllamaResponse = resp.json().await.map_err(|e| {
+        error!("Failed to parse Ollama response: {}", e);
+        (StatusCode::SERVICE_UNAVAILABLE, "AI response parse error".to_string())
+    })?;
+
+    let response_text = ollama_resp.message.map(|m| m.content).unwrap_or_default();
+
+    let _ = sqlx::query(
+        "INSERT INTO ai_conversations (id, user_id, role, message, response) VALUES ($1, $2, $3, $4, $5)",
+    )
+    .bind(Uuid::new_v4())
+    .bind(Uuid::parse_str(&claims.user_id).unwrap_or(Uuid::nil()))
+    .bind(&claims.role)
+    .bind(&message_text)
+    .bind(&response_text)
+    .execute(&state.db)
+    .await;
+
+    Ok(Json(ChatResponse { response: response_text }))
+}
+
+// ── Jobs handler ───────────────────────────────────────────────
+
+async fn handle_jobs(
+    State(state): State<Arc<AppState>>,
+    headers: HeaderMap,
+) -> Result<Json<JobsResponse>, (StatusCode, String)> {
+    let _claims = extract_claims(&headers, &state)?;
+    if !state.rate_limiter.check(&_claims.user_id).await {
+        return Err((StatusCode::TOO_MANY_REQUESTS, "Rate limit exceeded".to_string()));
+    }
+    Ok(Json(JobsResponse { jobs: state.jobs.clone() }))
+}
+
+// ── Health handler ─────────────────────────────────────────────
+
+async fn handle_health(
+    State(state): State<Arc<AppState>>,
+) -> Json<HealthResponse> {
+    Json(HealthResponse {
+        status: "ok".to_string(),
+        model: state.model.clone(),
+    })
+}
+
+// ── Main ───────────────────────────────────────────────────────
+
+#[tokio::main]
+async fn main() {
+    tracing_subscriber::fmt()
+        .with_env_filter(
+            tracing_subscriber::EnvFilter::try_from_default_env()
+                .unwrap_or_else(|_| "crm_ai=info,tower_http=info".into()),
+        )
+        .init();
+
+    dotenvy::dotenv().ok();
+
+    let database_url = std::env::var("DATABASE_URL").expect("DATABASE_URL must be set");
+    let jwt_secret = std::env::var("JWT_SECRET").expect("JWT_SECRET must be set");
+    let ollama_url = std::env::var("OLLAMA_BASE_URL").unwrap_or_else(|_| "http://localhost:11434".to_string());
+    let model = std::env::var("AI_MODEL").unwrap_or_else(|_| "dolphin-phi".to_string());
+    let host = std::env::var("AI_HOST").unwrap_or_else(|_| "127.0.0.1".to_string());
+    let port: u16 = std::env::var("AI_PORT").unwrap_or_else(|_| "3001".to_string()).parse().expect("AI_PORT must be a number");
+
+    let jobs_path = std::env::var("JOBS_PATH").unwrap_or_else(|_| "data/ai/jobs.jsonl".to_string());
+    let jobs_content = fs::read_to_string(&jobs_path).unwrap_or_default();
+    let jobs: Vec<Job> = jobs_content.lines().filter(|l| !l.trim().is_empty()).filter_map(|l| serde_json::from_str(l).ok()).collect();
+
+    info!("Loaded {} job categories, model: {}, Ollama: {}", jobs.len(), model, ollama_url);
+
+    let db = PgPoolOptions::new()
+        .max_connections(20)
+        .connect(&database_url)
+        .await
+        .expect("Failed to connect to database");
+
+    info!("Connected to PostgreSQL");
+
+    let http_client = reqwest::Client::builder()
+        .timeout(Duration::from_secs(300))
+        .build()
+        .expect("Failed to build HTTP client");
+
+    let lead_store = Arc::new(Mutex::new(LeadStore::new(100)));
+
+    let state = Arc::new(AppState {
+        db,
+        ollama_url,
+        model,
+        jobs,
+        leads: lead_store.clone(),
+        http_client,
+        jwt_secret,
+        rate_limiter: RateLimiter::new(30, 60),
+    });
+
+    let cors = CorsLayer::new()
+        .allow_origin(AllowOrigin::list([
+            "http://localhost:3006".parse().unwrap(),
+            "http://127.0.0.1:3006".parse().unwrap(),
+        ]))
+        .allow_methods([Method::GET, Method::POST])
+        .allow_headers(Any);
+
+    let app = Router::new()
+        .route("/ai/chat", post(handle_chat))
+        .route("/ai/jobs", get(handle_jobs))
+        .route("/health", get(handle_health))
+        .layer(cors)
+        .with_state(state.clone());
+
+    let addr = format!("{}:{}", host, port);
+    info!("CRM AI server listening on {}", addr);
+
+    let listener = tokio::net::TcpListener::bind(&addr)
+        .await
+        .expect("Failed to bind address");
+
+    let bg_leads = lead_store.clone();
+    let bg_db = state.db.clone();
+    let bg_url = "http://localhost:3008/scrape/facebook".to_string();
+    tokio::spawn(async move {
+        let client = match reqwest::Client::builder()
+            .timeout(Duration::from_secs(300))
+            .build()
+        {
+            Ok(c) => c,
+            Err(e) => {
+                error!("Failed to build background HTTP client: {} — scraper disabled", e);
+                return;
+            }
+        };
+        // Initial delay to let user on-demand requests take priority
+        tokio::time::sleep(Duration::from_secs(300)).await;
+
+        loop {
+            // 10% random cycle skip — makes pattern non-periodic
+            if rand::thread_rng().gen_range(0..100) < 10 {
+                info!("Skipping this scrape cycle (random 10% skip)");
+                let jitter = rand::thread_rng().gen_range(16200..19800);
+                tokio::time::sleep(Duration::from_secs(jitter)).await;
+                continue;
+            }
+
+            // Skip night hours (23:00 – 06:00)
+            let hour = chrono::Local::now().hour();
+            if hour < 6 || hour >= 23 {
+                info!("Night hours ({}) — skipping scrape", hour);
+                let jitter = rand::thread_rng().gen_range(16200..19800);
+                tokio::time::sleep(Duration::from_secs(jitter)).await;
+                continue;
+            }
+
+            let now = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs();
+
+            // Pick next active un-flagged account
+            let account = sqlx::query_as::<_, (uuid::Uuid, String)>(
+                "SELECT id, profile_path FROM facebook_accounts \
+                 WHERE is_active = TRUE AND flagged = FALSE \
+                 ORDER BY last_scrape_at ASC NULLS FIRST LIMIT 1"
+            )
+            .fetch_optional(&bg_db)
+            .await;
+
+            match account {
+                Ok(Some((account_id, profile_path))) => {
+                    match client.post(&bg_url).query(&[("profile_path", &profile_path)]).send().await {
+                        Ok(resp) => {
+                            if resp.status().is_success() {
+                                match resp.json::<ScrapeResponse>().await {
+                                    Ok(data) => {
+                                        let leads_count = data.leads.len() as i32;
+                                        if data.flagged {
+                                            let _ = sqlx::query(
+                                                "UPDATE facebook_accounts SET flagged = TRUE, flagged_at = NOW(), \
+                                                 flagged_reason = $2, last_error_at = NOW(), \
+                                                 last_error_message = $3, consecutive_failures = consecutive_failures + 1 \
+                                                 WHERE id = $1"
+                                            )
+                                            .bind(account_id)
+                                            .bind(&data.flag_reason)
+                                            .bind(&data.error)
+                                            .execute(&bg_db)
+                                            .await;
+                                            warn!("Facebook account {} flagged: {:?}", account_id, data.flag_reason);
+                                            let reason = data.flag_reason.as_deref().unwrap_or("unknown");
+                                            let _ = sqlx::query(
+                                                "INSERT INTO notifications (user_id, type, title, description, link) \
+                                                 SELECT id, 'warning', 'Facebook Account Flagged', \
+                                                 $1 || ' - ' || COALESCE($2, 'unknown reason'), \
+                                                 NULL \
+                                                 FROM users u JOIN user_roles ur ON ur.user_id = u.id \
+                                                 JOIN roles r ON r.id = ur.role_id \
+                                                 WHERE r.name IN ('ADMIN', 'SUPER_ADMIN')"
+                                            )
+                                            .bind(&account_id.to_string())
+                                            .bind(reason)
+                                            .execute(&bg_db)
+                                            .await;
+                                        } else if data.success {
+                                            let _ = sqlx::query(
+                                                "UPDATE facebook_accounts SET last_scrape_at = NOW(), \
+                                                 last_success_at = NOW(), consecutive_failures = 0, \
+                                                 updated_at = NOW() WHERE id = $1"
+                                            )
+                                            .bind(account_id)
+                                            .execute(&bg_db)
+                                            .await;
+
+                                            let mut store = bg_leads.lock().await;
+                                            for item in &data.leads {
+                                                store.push(Lead {
+                                                    title: truncate(&item.title, 120),
+                                                    url: item.url.clone(),
+                                                    source: item.source.clone().unwrap_or_else(|| "facebook".to_string()),
+                                                    found_at: now,
+                                                    author: truncate(&item.author, 60),
+                                                    date: truncate(&item.date, 30),
+                                                    content: truncate(&item.content, 300),
+                                                });
+                                            }
+                                            info!("Scraped {} leads from Facebook account {}", leads_count, account_id);
+                                        } else {
+                                            // Increment failures; auto-flag if >= 3 consecutive
+                                            let _ = sqlx::query(
+                                                "UPDATE facebook_accounts SET last_error_at = NOW(), \
+                                                 last_error_message = $2, consecutive_failures = consecutive_failures + 1, \
+                                                 flagged = CASE WHEN consecutive_failures + 1 >= 3 THEN TRUE ELSE flagged END, \
+                                                 flagged_reason = CASE WHEN consecutive_failures + 1 >= 3 THEN 'too_many_failures' ELSE flagged_reason END, \
+                                                 flagged_at = CASE WHEN consecutive_failures + 1 >= 3 THEN NOW() ELSE flagged_at END, \
+                                                 updated_at = NOW() WHERE id = $1"
+                                            )
+                                            .bind(account_id)
+                                            .bind(&data.error)
+                                            .execute(&bg_db)
+                                            .await;
+                                            warn!("Facebook scrape failed for account {}: {:?}", account_id, data.error);
+                                        }
+
+                                        let _ = sqlx::query(
+                                            "INSERT INTO facebook_scrape_logs \
+                                             (account_id, started_at, completed_at, success, leads_found, error_message, detected_flag) \
+                                             VALUES ($1, NOW() - interval '5 hours', NOW(), $2, $3, $4, $5)"
+                                        )
+                                        .bind(account_id)
+                                        .bind(data.success && !data.flagged)
+                                        .bind(leads_count)
+                                        .bind(&data.error)
+                                        .bind(&data.flag_reason)
+                                        .execute(&bg_db)
+                                        .await;
+                                    }
+                                    Err(e) => {
+                                        warn!("Failed to parse scraper JSON: {}", e);
+                                    }
+                                }
+                            } else {
+                                warn!("Scraper returned status: {}", resp.status());
+                            }
+                        }
+                        Err(e) => {
+                            warn!("Scraper request failed: {}", e);
+                        }
+                    }
+                }
+                Ok(None) => {
+                    info!("No active Facebook accounts available — skipping scrape cycle");
+                }
+                Err(e) => {
+                    warn!("Failed to query Facebook accounts: {}", e);
+                }
+            }
+
+            let jitter = rand::thread_rng().gen_range(16200..19800); // 4.5h – 5.5h
+            tokio::time::sleep(Duration::from_secs(jitter)).await;
+        }
+    });
+
+    axum::serve(listener, app)
+        .await
+        .expect("Server failed");
+}

scraper-service/index.js

@@ -0,0 +1,100 @@
+const express = require('express');
+const { chromium } = require('playwright');
+
+const app = express();
+app.use(express.json());
+const PORT = process.env.PORT || 3007;
+
+app.get('/health', (req, res) => {
+  res.json({ status: 'ok' });
+});
+
+app.post('/scrape/indeed', async (req, res) => {
+  const results = [];
+  let browser;
+  try {
+    browser = await chromium.launch({ headless: true });
+    const context = await browser.newContext({
+      userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
+    });
+
+    const queries = [
+      'need a website built',
+      'looking for someone to build my website',
+      'need web designer',
+      'looking for web developer',
+      'need help with my website',
+      'need ecommerce website',
+      'need wordpress website',
+      'looking for website designer',
+    ];
+
+    for (const q of queries) {
+      const url = `https://www.indeed.com/jobs?q=${encodeURIComponent(q)}&sort=date`;
+      try {
+        const page = await context.newPage();
+        await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 20000 });
+        await page.waitForTimeout(2000);
+
+        const items = await page.evaluate(() => {
+          const cards = document.querySelectorAll('.job_seen_beacon');
+          return Array.from(cards).slice(0, 8).map(card => {
+            const titleEl = card.querySelector('.jcs-JobTitle');
+            const companyEl = card.querySelector('[data-testid="inlineHeader-companyName"]');
+            const snippetEl = card.querySelector('.job-snippet');
+            return {
+              title: titleEl?.textContent?.trim() || '',
+              url: titleEl?.href || '',
+              company: companyEl?.textContent?.trim() || '',
+              snippet: snippetEl?.textContent?.trim()?.slice(0, 200) || '',
+            };
+          });
+        });
+
+        for (const item of items) {
+          if (item.title && item.url && !results.some(r => r.url === item.url)) {
+            results.push({
+              title: item.title,
+              url: item.url,
+              author: item.company,
+              date: '',
+              content: item.snippet,
+              source: 'indeed',
+            });
+          }
+        }
+        await page.close();
+      } catch (e) {
+        console.error(`Indeed failed ${q}:`, e.message);
+      }
+    }
+
+    await browser.close();
+    res.json(results.slice(0, 50));
+  } catch (e) {
+    if (browser) await browser.close().catch(() => {});
+    console.error('Indeed error:', e.message);
+    res.status(500).json({ error: e.message });
+  }
+});
+
+app.post('/scrape/all', async (req, res) => {
+  try {
+    const [indeed] = await Promise.allSettled([
+      fetch(`http://localhost:${PORT}/scrape/indeed`, { method: 'POST' }).then(r => r.json()).catch(() => []),
+    ]);
+
+    const all = [
+      ...(indeed.status === 'fulfilled' ? indeed.value : []),
+    ];
+    console.log(`Scrape all: ${all.length} leads`);
+    res.json(all);
+  } catch (e) {
+    console.error('Scrape all error:', e.message);
+    res.status(500).json({ error: e.message });
+  }
+});
+
+app.listen(PORT, () => {
+  console.log(`Scraper service running on port ${PORT}`);
+});

scraper-service/node_modules/.bin/mime

@@ -0,0 +1,16 @@
+#!/bin/sh
+basedir=$(dirname "$(echo "$0" | sed -e 's,\\,/,g')")
+
+case `uname` in
+    *CYGWIN*|*MINGW*|*MSYS*)
+        if command -v cygpath > /dev/null 2>&1; then
+            basedir=`cygpath -w "$basedir"`
+        fi
+    ;;
+esac
+
+if [ -x "$basedir/node" ]; then
+  exec "$basedir/node"  "$basedir/../mime/cli.js" "$@"
+else 
+  exec node  "$basedir/../mime/cli.js" "$@"
+fi

scraper-service/node_modules/.bin/mime.cmd

@@ -0,0 +1,17 @@
+@ECHO off
+GOTO start
+:find_dp0
+SET dp0=%~dp0
+EXIT /b
+:start
+SETLOCAL
+CALL :find_dp0
+
+IF EXIST "%dp0%\node.exe" (
+  SET "_prog=%dp0%\node.exe"
+) ELSE (
+  SET "_prog=node"
+  SET PATHEXT=%PATHEXT:;.JS;=;%
+)
+
+endLocal & goto #_undefined_# 2>NUL || title %COMSPEC% & "%_prog%"  "%dp0%\..\mime\cli.js" %*

scraper-service/node_modules/.bin/mime.ps1

@@ -0,0 +1,28 @@
+#!/usr/bin/env pwsh
+$basedir=Split-Path $MyInvocation.MyCommand.Definition -Parent
+
+$exe=""
+if ($PSVersionTable.PSVersion -lt "6.0" -or $IsWindows) {
+  # Fix case when both the Windows and Linux builds of Node
+  # are installed in the same directory
+  $exe=".exe"
+}
+$ret=0
+if (Test-Path "$basedir/node$exe") {
+  # Support pipeline input
+  if ($MyInvocation.ExpectingInput) {
+    $input | & "$basedir/node$exe"  "$basedir/../mime/cli.js" $args
+  } else {
+    & "$basedir/node$exe"  "$basedir/../mime/cli.js" $args
+  }
+  $ret=$LASTEXITCODE
+} else {
+  # Support pipeline input
+  if ($MyInvocation.ExpectingInput) {
+    $input | & "node$exe"  "$basedir/../mime/cli.js" $args
+  } else {
+    & "node$exe"  "$basedir/../mime/cli.js" $args
+  }
+  $ret=$LASTEXITCODE
+}
+exit $ret

scraper-service/node_modules/.bin/playwright

@@ -0,0 +1,16 @@
+#!/bin/sh
+basedir=$(dirname "$(echo "$0" | sed -e 's,\\,/,g')")
+
+case `uname` in
+    *CYGWIN*|*MINGW*|*MSYS*)
+        if command -v cygpath > /dev/null 2>&1; then
+            basedir=`cygpath -w "$basedir"`
+        fi
+    ;;
+esac
+
+if [ -x "$basedir/node" ]; then
+  exec "$basedir/node"  "$basedir/../playwright/cli.js" "$@"
+else 
+  exec node  "$basedir/../playwright/cli.js" "$@"
+fi

scraper-service/node_modules/.bin/playwright-core

@@ -0,0 +1,16 @@
+#!/bin/sh
+basedir=$(dirname "$(echo "$0" | sed -e 's,\\,/,g')")
+
+case `uname` in
+    *CYGWIN*|*MINGW*|*MSYS*)
+        if command -v cygpath > /dev/null 2>&1; then
+            basedir=`cygpath -w "$basedir"`
+        fi
+    ;;
+esac
+
+if [ -x "$basedir/node" ]; then
+  exec "$basedir/node"  "$basedir/../playwright-core/cli.js" "$@"
+else 
+  exec node  "$basedir/../playwright-core/cli.js" "$@"
+fi

scraper-service/node_modules/.bin/playwright-core.cmd

@@ -0,0 +1,17 @@
+@ECHO off
+GOTO start
+:find_dp0
+SET dp0=%~dp0
+EXIT /b
+:start
+SETLOCAL
+CALL :find_dp0
+
+IF EXIST "%dp0%\node.exe" (
+  SET "_prog=%dp0%\node.exe"
+) ELSE (
+  SET "_prog=node"
+  SET PATHEXT=%PATHEXT:;.JS;=;%
+)
+
+endLocal & goto #_undefined_# 2>NUL || title %COMSPEC% & "%_prog%"  "%dp0%\..\playwright-core\cli.js" %*

scraper-service/node_modules/.bin/playwright-core.ps1

@@ -0,0 +1,28 @@
+#!/usr/bin/env pwsh
+$basedir=Split-Path $MyInvocation.MyCommand.Definition -Parent
+
+$exe=""
+if ($PSVersionTable.PSVersion -lt "6.0" -or $IsWindows) {
+  # Fix case when both the Windows and Linux builds of Node
+  # are installed in the same directory
+  $exe=".exe"
+}
+$ret=0
+if (Test-Path "$basedir/node$exe") {
+  # Support pipeline input
+  if ($MyInvocation.ExpectingInput) {
+    $input | & "$basedir/node$exe"  "$basedir/../playwright-core/cli.js" $args
+  } else {
+    & "$basedir/node$exe"  "$basedir/../playwright-core/cli.js" $args
+  }
+  $ret=$LASTEXITCODE
+} else {
+  # Support pipeline input
+  if ($MyInvocation.ExpectingInput) {
+    $input | & "node$exe"  "$basedir/../playwright-core/cli.js" $args
+  } else {
+    & "node$exe"  "$basedir/../playwright-core/cli.js" $args
+  }
+  $ret=$LASTEXITCODE
+}
+exit $ret

scraper-service/node_modules/.bin/playwright.cmd

@@ -0,0 +1,17 @@
+@ECHO off
+GOTO start
+:find_dp0
+SET dp0=%~dp0
+EXIT /b
+:start
+SETLOCAL
+CALL :find_dp0
+
+IF EXIST "%dp0%\node.exe" (
+  SET "_prog=%dp0%\node.exe"
+) ELSE (
+  SET "_prog=node"
+  SET PATHEXT=%PATHEXT:;.JS;=;%
+)
+
+endLocal & goto #_undefined_# 2>NUL || title %COMSPEC% & "%_prog%"  "%dp0%\..\playwright\cli.js" %*

scraper-service/node_modules/.bin/playwright.ps1

@@ -0,0 +1,28 @@
+#!/usr/bin/env pwsh
+$basedir=Split-Path $MyInvocation.MyCommand.Definition -Parent
+
+$exe=""
+if ($PSVersionTable.PSVersion -lt "6.0" -or $IsWindows) {
+  # Fix case when both the Windows and Linux builds of Node
+  # are installed in the same directory
+  $exe=".exe"
+}
+$ret=0
+if (Test-Path "$basedir/node$exe") {
+  # Support pipeline input
+  if ($MyInvocation.ExpectingInput) {
+    $input | & "$basedir/node$exe"  "$basedir/../playwright/cli.js" $args
+  } else {
+    & "$basedir/node$exe"  "$basedir/../playwright/cli.js" $args
+  }
+  $ret=$LASTEXITCODE
+} else {
+  # Support pipeline input
+  if ($MyInvocation.ExpectingInput) {
+    $input | & "node$exe"  "$basedir/../playwright/cli.js" $args
+  } else {
+    & "node$exe"  "$basedir/../playwright/cli.js" $args
+  }
+  $ret=$LASTEXITCODE
+}
+exit $ret

scraper-service/node_modules/.package-lock.json

@@ -0,0 +1,850 @@
+{
+  "name": "scraper-service",
+  "version": "1.0.0",
+  "lockfileVersion": 3,
+  "requires": true,
+  "packages": {
+    "node_modules/accepts": {
+      "version": "1.3.8",
+      "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz",
+      "integrity": "sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==",
+      "license": "MIT",
+      "dependencies": {
+        "mime-types": "~2.1.34",
+        "negotiator": "0.6.3"
+      },
+      "engines": {
+        "node": ">= 0.6"
+      }
+    },
+    "node_modules/array-flatten": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz",
+      "integrity": "sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg==",
+      "license": "MIT"
+    },
+    "node_modules/body-parser": {
+      "version": "1.20.5",
+      "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.5.tgz",
+      "integrity": "sha512-3grm+/2tUOvu2cjJkvsIxrv/wVpfXQW4PsQHYm7yk4vfpu7Ekl6nEsYBoJUL6qDwZUx8wUhQ8tR2qz+ad9c9OA==",
+      "license": "MIT",
+      "dependencies": {
+        "bytes": "~3.1.2",
+        "content-type": "~1.0.5",
+        "debug": "2.6.9",
+        "depd": "2.0.0",
+        "destroy": "~1.2.0",
+        "http-errors": "~2.0.1",
+        "iconv-lite": "~0.4.24",
+        "on-finished": "~2.4.1",
+        "qs": "~6.15.1",
+        "raw-body": "~2.5.3",
+        "type-is": "~1.6.18",
+        "unpipe": "~1.0.0"
+      },
+      "engines": {
+        "node": ">= 0.8",
+        "npm": "1.2.8000 || >= 1.4.16"
+      }
+    },
+    "node_modules/bytes": {
+      "version": "3.1.2",
+      "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz",
+      "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==",
+      "license": "MIT",
+      "engines": {
+        "node": ">= 0.8"
+      }
+    },
+    "node_modules/call-bind-apply-helpers": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz",
+      "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==",
+      "license": "MIT",
+      "dependencies": {
+        "es-errors": "^1.3.0",
+        "function-bind": "^1.1.2"
+      },
+      "engines": {
+        "node": ">= 0.4"
+      }
+    },
+    "node_modules/call-bound": {
+      "version": "1.0.4",
+      "resolved": "https://registry.npmjs.org/call-bound/-/call-bound-1.0.4.tgz",
+      "integrity": "sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg==",
+      "license": "MIT",
+      "dependencies": {
+        "call-bind-apply-helpers": "^1.0.2",
+        "get-intrinsic": "^1.3.0"
+      },
+      "engines": {
+        "node": ">= 0.4"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
+    "node_modules/content-disposition": {
+      "version": "0.5.4",
+      "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz",
+      "integrity": "sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==",
+      "license": "MIT",
+      "dependencies": {
+        "safe-buffer": "5.2.1"
+      },
+      "engines": {
+        "node": ">= 0.6"
+      }
+    },
+    "node_modules/content-type": {
+      "version": "1.0.5",
+      "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.5.tgz",
+      "integrity": "sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==",
+      "license": "MIT",
+      "engines": {
+        "node": ">= 0.6"
+      }
+    },
+    "node_modules/cookie": {
+      "version": "0.7.2",
+      "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.7.2.tgz",
+      "integrity": "sha512-yki5XnKuf750l50uGTllt6kKILY4nQ1eNIQatoXEByZ5dWgnKqbnqmTrBE5B4N7lrMJKQ2ytWMiTO2o0v6Ew/w==",
+      "license": "MIT",
+      "engines": {
+        "node": ">= 0.6"
+      }
+    },
+    "node_modules/cookie-signature": {
+      "version": "1.0.7",
+      "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.7.tgz",
+      "integrity": "sha512-NXdYc3dLr47pBkpUCHtKSwIOQXLVn8dZEuywboCOJY/osA0wFSLlSawr3KN8qXJEyX66FcONTH8EIlVuK0yyFA==",
+      "license": "MIT"
+    },
+    "node_modules/debug": {
+      "version": "2.6.9",
+      "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
+      "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==",
+      "license": "MIT",
+      "dependencies": {
+        "ms": "2.0.0"
+      }
+    },
+    "node_modules/depd": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz",
+      "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==",
+      "license": "MIT",
+      "engines": {
+        "node": ">= 0.8"
+      }
+    },
+    "node_modules/destroy": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.2.0.tgz",
+      "integrity": "sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg==",
+      "license": "MIT",
+      "engines": {
+        "node": ">= 0.8",
+        "npm": "1.2.8000 || >= 1.4.16"
+      }
+    },
+    "node_modules/dunder-proto": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz",
+      "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==",
+      "license": "MIT",
+      "dependencies": {
+        "call-bind-apply-helpers": "^1.0.1",
+        "es-errors": "^1.3.0",
+        "gopd": "^1.2.0"
+      },
+      "engines": {
+        "node": ">= 0.4"
+      }
+    },
+    "node_modules/ee-first": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz",
+      "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==",
+      "license": "MIT"
+    },
+    "node_modules/encodeurl": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-2.0.0.tgz",
+      "integrity": "sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==",
+      "license": "MIT",
+      "engines": {
+        "node": ">= 0.8"
+      }
+    },
+    "node_modules/es-define-property": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz",
+      "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==",
+      "license": "MIT",
+      "engines": {
+        "node": ">= 0.4"
+      }
+    },
+    "node_modules/es-errors": {
+      "version": "1.3.0",
+      "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz",
+      "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==",
+      "license": "MIT",
+      "engines": {
+        "node": ">= 0.4"
+      }
+    },
+    "node_modules/es-object-atoms": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.2.tgz",
+      "integrity": "sha512-HWcBoN6NileqtSydK2FqHbS/LoDd2pqrnQHLyJzBj4kOp/ky2MWMN694xOfkK8/SnUsW2DH7EfyVlydKCsm1Zw==",
+      "license": "MIT",
+      "dependencies": {
+        "es-errors": "^1.3.0"
+      },
+      "engines": {
+        "node": ">= 0.4"
+      }
+    },
+    "node_modules/escape-html": {
+      "version": "1.0.3",
+      "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz",
+      "integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==",
+      "license": "MIT"
+    },
+    "node_modules/etag": {
+      "version": "1.8.1",
+      "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz",
+      "integrity": "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==",
+      "license": "MIT",
+      "engines": {
+        "node": ">= 0.6"
+      }
+    },
+    "node_modules/express": {
+      "version": "4.22.2",
+      "resolved": "https://registry.npmjs.org/express/-/express-4.22.2.tgz",
+      "integrity": "sha512-IuL+Elrou2ZvCFHs18/CIzy2Nzvo25nZ1/D2eIZlz7c+QUayAcYoiM2BthCjs+EBHVpjYjcuLDAiCWgeIX3X1Q==",
+      "license": "MIT",
+      "dependencies": {
+        "accepts": "~1.3.8",
+        "array-flatten": "1.1.1",
+        "body-parser": "~1.20.5",
+        "content-disposition": "~0.5.4",
+        "content-type": "~1.0.4",
+        "cookie": "~0.7.1",
+        "cookie-signature": "~1.0.6",
+        "debug": "2.6.9",
+        "depd": "2.0.0",
+        "encodeurl": "~2.0.0",
+        "escape-html": "~1.0.3",
+        "etag": "~1.8.1",
+        "finalhandler": "~1.3.1",
+        "fresh": "~0.5.2",
+        "http-errors": "~2.0.0",
+        "merge-descriptors": "1.0.3",
+        "methods": "~1.1.2",
+        "on-finished": "~2.4.1",
+        "parseurl": "~1.3.3",
+        "path-to-regexp": "~0.1.12",
+        "proxy-addr": "~2.0.7",
+        "qs": "~6.15.1",
+        "range-parser": "~1.2.1",
+        "safe-buffer": "5.2.1",
+        "send": "~0.19.0",
+        "serve-static": "~1.16.2",
+        "setprototypeof": "1.2.0",
+        "statuses": "~2.0.1",
+        "type-is": "~1.6.18",
+        "utils-merge": "1.0.1",
+        "vary": "~1.1.2"
+      },
+      "engines": {
+        "node": ">= 0.10.0"
+      },
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/express"
+      }
+    },
+    "node_modules/finalhandler": {
+      "version": "1.3.2",
+      "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.3.2.tgz",
+      "integrity": "sha512-aA4RyPcd3badbdABGDuTXCMTtOneUCAYH/gxoYRTZlIJdF0YPWuGqiAsIrhNnnqdXGswYk6dGujem4w80UJFhg==",
+      "license": "MIT",
+      "dependencies": {
+        "debug": "2.6.9",
+        "encodeurl": "~2.0.0",
+        "escape-html": "~1.0.3",
+        "on-finished": "~2.4.1",
+        "parseurl": "~1.3.3",
+        "statuses": "~2.0.2",
+        "unpipe": "~1.0.0"
+      },
+      "engines": {
+        "node": ">= 0.8"
+      }
+    },
+    "node_modules/forwarded": {
+      "version": "0.2.0",
+      "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz",
+      "integrity": "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==",
+      "license": "MIT",
+      "engines": {
+        "node": ">= 0.6"
+      }
+    },
+    "node_modules/fresh": {
+      "version": "0.5.2",
+      "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz",
+      "integrity": "sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==",
+      "license": "MIT",
+      "engines": {
+        "node": ">= 0.6"
+      }
+    },
+    "node_modules/function-bind": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz",
+      "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==",
+      "license": "MIT",
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
+    "node_modules/get-intrinsic": {
+      "version": "1.3.0",
+      "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz",
+      "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==",
+      "license": "MIT",
+      "dependencies": {
+        "call-bind-apply-helpers": "^1.0.2",
+        "es-define-property": "^1.0.1",
+        "es-errors": "^1.3.0",
+        "es-object-atoms": "^1.1.1",
+        "function-bind": "^1.1.2",
+        "get-proto": "^1.0.1",
+        "gopd": "^1.2.0",
+        "has-symbols": "^1.1.0",
+        "hasown": "^2.0.2",
+        "math-intrinsics": "^1.1.0"
+      },
+      "engines": {
+        "node": ">= 0.4"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
+    "node_modules/get-proto": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz",
+      "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==",
+      "license": "MIT",
+      "dependencies": {
+        "dunder-proto": "^1.0.1",
+        "es-object-atoms": "^1.0.0"
+      },
+      "engines": {
+        "node": ">= 0.4"
+      }
+    },
+    "node_modules/gopd": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz",
+      "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==",
+      "license": "MIT",
+      "engines": {
+        "node": ">= 0.4"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
+    "node_modules/has-symbols": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz",
+      "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==",
+      "license": "MIT",
+      "engines": {
+        "node": ">= 0.4"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
+    "node_modules/hasown": {
+      "version": "2.0.4",
+      "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.4.tgz",
+      "integrity": "sha512-T2UbfbBEF32wiepXIsMlTW9+dDYC6wMh/t/vYA4tuOMKqWz/n3vr1NFSxQiyP+zk2mXsoMA/i/7qV6LKut1t1A==",
+      "license": "MIT",
+      "dependencies": {
+        "function-bind": "^1.1.2"
+      },
+      "engines": {
+        "node": ">= 0.4"
+      }
+    },
+    "node_modules/http-errors": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz",
+      "integrity": "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==",
+      "license": "MIT",
+      "dependencies": {
+        "depd": "~2.0.0",
+        "inherits": "~2.0.4",
+        "setprototypeof": "~1.2.0",
+        "statuses": "~2.0.2",
+        "toidentifier": "~1.0.1"
+      },
+      "engines": {
+        "node": ">= 0.8"
+      },
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/express"
+      }
+    },
+    "node_modules/iconv-lite": {
+      "version": "0.4.24",
+      "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz",
+      "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==",
+      "license": "MIT",
+      "dependencies": {
+        "safer-buffer": ">= 2.1.2 < 3"
+      },
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
+    "node_modules/inherits": {
+      "version": "2.0.4",
+      "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz",
+      "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==",
+      "license": "ISC"
+    },
+    "node_modules/ipaddr.js": {
+      "version": "1.9.1",
+      "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz",
+      "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==",
+      "license": "MIT",
+      "engines": {
+        "node": ">= 0.10"
+      }
+    },
+    "node_modules/math-intrinsics": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz",
+      "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==",
+      "license": "MIT",
+      "engines": {
+        "node": ">= 0.4"
+      }
+    },
+    "node_modules/media-typer": {
+      "version": "0.3.0",
+      "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz",
+      "integrity": "sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==",
+      "license": "MIT",
+      "engines": {
+        "node": ">= 0.6"
+      }
+    },
+    "node_modules/merge-descriptors": {
+      "version": "1.0.3",
+      "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.3.tgz",
+      "integrity": "sha512-gaNvAS7TZ897/rVaZ0nMtAyxNyi/pdbjbAwUpFQpN70GqnVfOiXpeUUMKRBmzXaSQ8DdTX4/0ms62r2K+hE6mQ==",
+      "license": "MIT",
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
+      }
+    },
+    "node_modules/methods": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz",
+      "integrity": "sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w==",
+      "license": "MIT",
+      "engines": {
+        "node": ">= 0.6"
+      }
+    },
+    "node_modules/mime": {
+      "version": "1.6.0",
+      "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz",
+      "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==",
+      "license": "MIT",
+      "bin": {
+        "mime": "cli.js"
+      },
+      "engines": {
+        "node": ">=4"
+      }
+    },
+    "node_modules/mime-db": {
+      "version": "1.52.0",
+      "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz",
+      "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==",
+      "license": "MIT",
+      "engines": {
+        "node": ">= 0.6"
+      }
+    },
+    "node_modules/mime-types": {
+      "version": "2.1.35",
+      "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz",
+      "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==",
+      "license": "MIT",
+      "dependencies": {
+        "mime-db": "1.52.0"
+      },
+      "engines": {
+        "node": ">= 0.6"
+      }
+    },
+    "node_modules/ms": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
+      "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==",
+      "license": "MIT"
+    },
+    "node_modules/negotiator": {
+      "version": "0.6.3",
+      "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz",
+      "integrity": "sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==",
+      "license": "MIT",
+      "engines": {
+        "node": ">= 0.6"
+      }
+    },
+    "node_modules/object-inspect": {
+      "version": "1.13.4",
+      "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.4.tgz",
+      "integrity": "sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew==",
+      "license": "MIT",
+      "engines": {
+        "node": ">= 0.4"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
+    "node_modules/on-finished": {
+      "version": "2.4.1",
+      "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz",
+      "integrity": "sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==",
+      "license": "MIT",
+      "dependencies": {
+        "ee-first": "1.1.1"
+      },
+      "engines": {
+        "node": ">= 0.8"
+      }
+    },
+    "node_modules/parseurl": {
+      "version": "1.3.3",
+      "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz",
+      "integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==",
+      "license": "MIT",
+      "engines": {
+        "node": ">= 0.8"
+      }
+    },
+    "node_modules/path-to-regexp": {
+      "version": "0.1.13",
+      "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.13.tgz",
+      "integrity": "sha512-A/AGNMFN3c8bOlvV9RreMdrv7jsmF9XIfDeCd87+I8RNg6s78BhJxMu69NEMHBSJFxKidViTEdruRwEk/WIKqA==",
+      "license": "MIT"
+    },
+    "node_modules/playwright": {
+      "version": "1.61.0",
+      "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.61.0.tgz",
+      "integrity": "sha512-Z+7BeeqQPRRzklHsVFP4KTGIyMxKUmfeRA4WisM6G3/XW6nwGeX6fX9qYaDa+CiUqpOkb2f6X3nar05R3kSuJQ==",
+      "license": "Apache-2.0",
+      "dependencies": {
+        "playwright-core": "1.61.0"
+      },
+      "bin": {
+        "playwright": "cli.js"
+      },
+      "engines": {
+        "node": ">=18"
+      },
+      "optionalDependencies": {
+        "fsevents": "2.3.2"
+      }
+    },
+    "node_modules/playwright-core": {
+      "version": "1.61.0",
+      "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.61.0.tgz",
+      "integrity": "sha512-caX7TrY3Ml6egyDX0WUcTHDxodl/b51y5wJOdCEA36QviK/s2g081hvmGs8eaE3DWb6NYZQ6BjO/QkNRPenoPA==",
+      "license": "Apache-2.0",
+      "bin": {
+        "playwright-core": "cli.js"
+      },
+      "engines": {
+        "node": ">=18"
+      }
+    },
+    "node_modules/proxy-addr": {
+      "version": "2.0.7",
+      "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz",
+      "integrity": "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==",
+      "license": "MIT",
+      "dependencies": {
+        "forwarded": "0.2.0",
+        "ipaddr.js": "1.9.1"
+      },
+      "engines": {
+        "node": ">= 0.10"
+      }
+    },
+    "node_modules/qs": {
+      "version": "6.15.2",
+      "resolved": "https://registry.npmjs.org/qs/-/qs-6.15.2.tgz",
+      "integrity": "sha512-Rzq0KEyX/w/tEybncDgdkZrJgVUsUMk3xjh3t5bv3S1HTAtg+uOYt72+ZfwiQwKdysThkTBdL/rTi6HDmX9Ddw==",
+      "license": "BSD-3-Clause",
+      "dependencies": {
+        "side-channel": "^1.1.0"
+      },
+      "engines": {
+        "node": ">=0.6"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
+    "node_modules/range-parser": {
+      "version": "1.2.1",
+      "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz",
+      "integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==",
+      "license": "MIT",
+      "engines": {
+        "node": ">= 0.6"
+      }
+    },
+    "node_modules/raw-body": {
+      "version": "2.5.3",
+      "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.3.tgz",
+      "integrity": "sha512-s4VSOf6yN0rvbRZGxs8Om5CWj6seneMwK3oDb4lWDH0UPhWcxwOWw5+qk24bxq87szX1ydrwylIOp2uG1ojUpA==",
+      "license": "MIT",
+      "dependencies": {
+        "bytes": "~3.1.2",
+        "http-errors": "~2.0.1",
+        "iconv-lite": "~0.4.24",
+        "unpipe": "~1.0.0"
+      },
+      "engines": {
+        "node": ">= 0.8"
+      }
+    },
+    "node_modules/safe-buffer": {
+      "version": "5.2.1",
+      "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz",
+      "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==",
+      "funding": [
+        {
+          "type": "github",
+          "url": "https://github.com/sponsors/feross"
+        },
+        {
+          "type": "patreon",
+          "url": "https://www.patreon.com/feross"
+        },
+        {
+          "type": "consulting",
+          "url": "https://feross.org/support"
+        }
+      ],
+      "license": "MIT"
+    },
+    "node_modules/safer-buffer": {
+      "version": "2.1.2",
+      "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz",
+      "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==",
+      "license": "MIT"
+    },
+    "node_modules/send": {
+      "version": "0.19.2",
+      "resolved": "https://registry.npmjs.org/send/-/send-0.19.2.tgz",
+      "integrity": "sha512-VMbMxbDeehAxpOtWJXlcUS5E8iXh6QmN+BkRX1GARS3wRaXEEgzCcB10gTQazO42tpNIya8xIyNx8fll1OFPrg==",
+      "license": "MIT",
+      "dependencies": {
+        "debug": "2.6.9",
+        "depd": "2.0.0",
+        "destroy": "1.2.0",
+        "encodeurl": "~2.0.0",
+        "escape-html": "~1.0.3",
+        "etag": "~1.8.1",
+        "fresh": "~0.5.2",
+        "http-errors": "~2.0.1",
+        "mime": "1.6.0",
+        "ms": "2.1.3",
+        "on-finished": "~2.4.1",
+        "range-parser": "~1.2.1",
+        "statuses": "~2.0.2"
+      },
+      "engines": {
+        "node": ">= 0.8.0"
+      }
+    },
+    "node_modules/send/node_modules/ms": {
+      "version": "2.1.3",
+      "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
+      "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==",
+      "license": "MIT"
+    },
+    "node_modules/serve-static": {
+      "version": "1.16.3",
+      "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.16.3.tgz",
+      "integrity": "sha512-x0RTqQel6g5SY7Lg6ZreMmsOzncHFU7nhnRWkKgWuMTu5NN0DR5oruckMqRvacAN9d5w6ARnRBXl9xhDCgfMeA==",
+      "license": "MIT",
+      "dependencies": {
+        "encodeurl": "~2.0.0",
+        "escape-html": "~1.0.3",
+        "parseurl": "~1.3.3",
+        "send": "~0.19.1"
+      },
+      "engines": {
+        "node": ">= 0.8.0"
+      }
+    },
+    "node_modules/setprototypeof": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz",
+      "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==",
+      "license": "ISC"
+    },
+    "node_modules/side-channel": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.1.1.tgz",
+      "integrity": "sha512-6x6dK6zJdpTzF4sQeNYxwtvBzf6Eg4GtlesS94HOvTudUeyK2WXAaIfmDgsyslYrRBeFIlsi54AYsFGUuhmvrQ==",
+      "license": "MIT",
+      "dependencies": {
+        "es-errors": "^1.3.0",
+        "object-inspect": "^1.13.4",
+        "side-channel-list": "^1.0.1",
+        "side-channel-map": "^1.0.1",
+        "side-channel-weakmap": "^1.0.2"
+      },
+      "engines": {
+        "node": ">= 0.4"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
+    "node_modules/side-channel-list": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/side-channel-list/-/side-channel-list-1.0.1.tgz",
+      "integrity": "sha512-mjn/0bi/oUURjc5Xl7IaWi/OJJJumuoJFQJfDDyO46+hBWsfaVM65TBHq2eoZBhzl9EchxOijpkbRC8SVBQU0w==",
+      "license": "MIT",
+      "dependencies": {
+        "es-errors": "^1.3.0",
+        "object-inspect": "^1.13.4"
+      },
+      "engines": {
+        "node": ">= 0.4"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
+    "node_modules/side-channel-map": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/side-channel-map/-/side-channel-map-1.0.1.tgz",
+      "integrity": "sha512-VCjCNfgMsby3tTdo02nbjtM/ewra6jPHmpThenkTYh8pG9ucZ/1P8So4u4FGBek/BjpOVsDCMoLA/iuBKIFXRA==",
+      "license": "MIT",
+      "dependencies": {
+        "call-bound": "^1.0.2",
+        "es-errors": "^1.3.0",
+        "get-intrinsic": "^1.2.5",
+        "object-inspect": "^1.13.3"
+      },
+      "engines": {
+        "node": ">= 0.4"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
+    "node_modules/side-channel-weakmap": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/side-channel-weakmap/-/side-channel-weakmap-1.0.2.tgz",
+      "integrity": "sha512-WPS/HvHQTYnHisLo9McqBHOJk2FkHO/tlpvldyrnem4aeQp4hai3gythswg6p01oSoTl58rcpiFAjF2br2Ak2A==",
+      "license": "MIT",
+      "dependencies": {
+        "call-bound": "^1.0.2",
+        "es-errors": "^1.3.0",
+        "get-intrinsic": "^1.2.5",
+        "object-inspect": "^1.13.3",
+        "side-channel-map": "^1.0.1"
+      },
+      "engines": {
+        "node": ">= 0.4"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
+    "node_modules/statuses": {
+      "version": "2.0.2",
+      "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz",
+      "integrity": "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==",
+      "license": "MIT",
+      "engines": {
+        "node": ">= 0.8"
+      }
+    },
+    "node_modules/toidentifier": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz",
+      "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==",
+      "license": "MIT",
+      "engines": {
+        "node": ">=0.6"
+      }
+    },
+    "node_modules/type-is": {
+      "version": "1.6.18",
+      "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz",
+      "integrity": "sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==",
+      "license": "MIT",
+      "dependencies": {
+        "media-typer": "0.3.0",
+        "mime-types": "~2.1.24"
+      },
+      "engines": {
+        "node": ">= 0.6"
+      }
+    },
+    "node_modules/unpipe": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz",
+      "integrity": "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==",
+      "license": "MIT",
+      "engines": {
+        "node": ">= 0.8"
+      }
+    },
+    "node_modules/utils-merge": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz",
+      "integrity": "sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA==",
+      "license": "MIT",
+      "engines": {
+        "node": ">= 0.4.0"
+      }
+    },
+    "node_modules/vary": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz",
+      "integrity": "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==",
+      "license": "MIT",
+      "engines": {
+        "node": ">= 0.8"
+      }
+    }
+  }
+}

scraper-service/node_modules/accepts/HISTORY.md

@@ -0,0 +1,243 @@
+1.3.8 / 2022-02-02
+==================
+
+  * deps: mime-types@~2.1.34
+    - deps: mime-db@~1.51.0
+  * deps: negotiator@0.6.3
+
+1.3.7 / 2019-04-29
+==================
+
+  * deps: negotiator@0.6.2
+    - Fix sorting charset, encoding, and language with extra parameters
+
+1.3.6 / 2019-04-28
+==================
+
+  * deps: mime-types@~2.1.24
+    - deps: mime-db@~1.40.0
+
+1.3.5 / 2018-02-28
+==================
+
+  * deps: mime-types@~2.1.18
+    - deps: mime-db@~1.33.0
+
+1.3.4 / 2017-08-22
+==================
+
+  * deps: mime-types@~2.1.16
+    - deps: mime-db@~1.29.0
+
+1.3.3 / 2016-05-02
+==================
+
+  * deps: mime-types@~2.1.11
+    - deps: mime-db@~1.23.0
+  * deps: negotiator@0.6.1
+    - perf: improve `Accept` parsing speed
+    - perf: improve `Accept-Charset` parsing speed
+    - perf: improve `Accept-Encoding` parsing speed
+    - perf: improve `Accept-Language` parsing speed
+
+1.3.2 / 2016-03-08
+==================
+
+  * deps: mime-types@~2.1.10
+    - Fix extension of `application/dash+xml`
+    - Update primary extension for `audio/mp4`
+    - deps: mime-db@~1.22.0
+
+1.3.1 / 2016-01-19
+==================
+
+  * deps: mime-types@~2.1.9
+    - deps: mime-db@~1.21.0
+
+1.3.0 / 2015-09-29
+==================
+
+  * deps: mime-types@~2.1.7
+    - deps: mime-db@~1.19.0
+  * deps: negotiator@0.6.0
+    - Fix including type extensions in parameters in `Accept` parsing
+    - Fix parsing `Accept` parameters with quoted equals
+    - Fix parsing `Accept` parameters with quoted semicolons
+    - Lazy-load modules from main entry point
+    - perf: delay type concatenation until needed
+    - perf: enable strict mode
+    - perf: hoist regular expressions
+    - perf: remove closures getting spec properties
+    - perf: remove a closure from media type parsing
+    - perf: remove property delete from media type parsing
+
+1.2.13 / 2015-09-06
+===================
+
+  * deps: mime-types@~2.1.6
+    - deps: mime-db@~1.18.0
+
+1.2.12 / 2015-07-30
+===================
+
+  * deps: mime-types@~2.1.4
+    - deps: mime-db@~1.16.0
+
+1.2.11 / 2015-07-16
+===================
+
+  * deps: mime-types@~2.1.3
+    - deps: mime-db@~1.15.0
+
+1.2.10 / 2015-07-01
+===================
+
+  * deps: mime-types@~2.1.2
+    - deps: mime-db@~1.14.0
+
+1.2.9 / 2015-06-08
+==================
+
+  * deps: mime-types@~2.1.1
+    - perf: fix deopt during mapping
+
+1.2.8 / 2015-06-07
+==================
+
+  * deps: mime-types@~2.1.0
+    - deps: mime-db@~1.13.0
+  * perf: avoid argument reassignment & argument slice
+  * perf: avoid negotiator recursive construction
+  * perf: enable strict mode
+  * perf: remove unnecessary bitwise operator
+
+1.2.7 / 2015-05-10
+==================
+
+  * deps: negotiator@0.5.3
+    - Fix media type parameter matching to be case-insensitive
+
+1.2.6 / 2015-05-07
+==================
+
+  * deps: mime-types@~2.0.11
+    - deps: mime-db@~1.9.1
+  * deps: negotiator@0.5.2
+    - Fix comparing media types with quoted values
+    - Fix splitting media types with quoted commas
+
+1.2.5 / 2015-03-13
+==================
+
+  * deps: mime-types@~2.0.10
+    - deps: mime-db@~1.8.0
+
+1.2.4 / 2015-02-14
+==================
+
+  * Support Node.js 0.6
+  * deps: mime-types@~2.0.9
+    - deps: mime-db@~1.7.0
+  * deps: negotiator@0.5.1
+    - Fix preference sorting to be stable for long acceptable lists
+
+1.2.3 / 2015-01-31
+==================
+
+  * deps: mime-types@~2.0.8
+    - deps: mime-db@~1.6.0
+
+1.2.2 / 2014-12-30
+==================
+
+  * deps: mime-types@~2.0.7
+    - deps: mime-db@~1.5.0
+
+1.2.1 / 2014-12-30
+==================
+
+  * deps: mime-types@~2.0.5
+    - deps: mime-db@~1.3.1
+
+1.2.0 / 2014-12-19
+==================
+
+  * deps: negotiator@0.5.0
+    - Fix list return order when large accepted list
+    - Fix missing identity encoding when q=0 exists
+    - Remove dynamic building of Negotiator class
+
+1.1.4 / 2014-12-10
+==================
+
+  * deps: mime-types@~2.0.4
+    - deps: mime-db@~1.3.0
+
+1.1.3 / 2014-11-09
+==================
+
+  * deps: mime-types@~2.0.3
+    - deps: mime-db@~1.2.0
+
+1.1.2 / 2014-10-14
+==================
+
+  * deps: negotiator@0.4.9
+    - Fix error when media type has invalid parameter
+
+1.1.1 / 2014-09-28
+==================
+
+  * deps: mime-types@~2.0.2
+    - deps: mime-db@~1.1.0
+  * deps: negotiator@0.4.8
+    - Fix all negotiations to be case-insensitive
+    - Stable sort preferences of same quality according to client order
+
+1.1.0 / 2014-09-02
+==================
+
+  * update `mime-types`
+
+1.0.7 / 2014-07-04
+==================
+
+  * Fix wrong type returned from `type` when match after unknown extension
+
+1.0.6 / 2014-06-24
+==================
+
+  * deps: negotiator@0.4.7
+
+1.0.5 / 2014-06-20
+==================
+
+ * fix crash when unknown extension given
+
+1.0.4 / 2014-06-19
+==================
+
+  * use `mime-types`
+
+1.0.3 / 2014-06-11
+==================
+
+  * deps: negotiator@0.4.6
+    - Order by specificity when quality is the same
+
+1.0.2 / 2014-05-29
+==================
+
+  * Fix interpretation when header not in request
+  * deps: pin negotiator@0.4.5
+
+1.0.1 / 2014-01-18
+==================
+
+  * Identity encoding isn't always acceptable
+  * deps: negotiator@~0.4.0
+
+1.0.0 / 2013-12-27
+==================
+
+  * Genesis

scraper-service/node_modules/accepts/LICENSE

@@ -0,0 +1,23 @@
+(The MIT License)
+
+Copyright (c) 2014 Jonathan Ong <me@jongleberry.com>
+Copyright (c) 2015 Douglas Christopher Wilson <doug@somethingdoug.com>
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+'Software'), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

scraper-service/node_modules/accepts/README.md

@@ -0,0 +1,140 @@
+# accepts
+
+[![NPM Version][npm-version-image]][npm-url]
+[![NPM Downloads][npm-downloads-image]][npm-url]
+[![Node.js Version][node-version-image]][node-version-url]
+[![Build Status][github-actions-ci-image]][github-actions-ci-url]
+[![Test Coverage][coveralls-image]][coveralls-url]
+
+Higher level content negotiation based on [negotiator](https://www.npmjs.com/package/negotiator).
+Extracted from [koa](https://www.npmjs.com/package/koa) for general use.
+
+In addition to negotiator, it allows:
+
+- Allows types as an array or arguments list, ie `(['text/html', 'application/json'])`
+  as well as `('text/html', 'application/json')`.
+- Allows type shorthands such as `json`.
+- Returns `false` when no types match
+- Treats non-existent headers as `*`
+
+## Installation
+
+This is a [Node.js](https://nodejs.org/en/) module available through the
+[npm registry](https://www.npmjs.com/). Installation is done using the
+[`npm install` command](https://docs.npmjs.com/getting-started/installing-npm-packages-locally):
+
+```sh
+$ npm install accepts
+```
+
+## API
+
+```js
+var accepts = require('accepts')
+```
+
+### accepts(req)
+
+Create a new `Accepts` object for the given `req`.
+
+#### .charset(charsets)
+
+Return the first accepted charset. If nothing in `charsets` is accepted,
+then `false` is returned.
+
+#### .charsets()
+
+Return the charsets that the request accepts, in the order of the client's
+preference (most preferred first).
+
+#### .encoding(encodings)
+
+Return the first accepted encoding. If nothing in `encodings` is accepted,
+then `false` is returned.
+
+#### .encodings()
+
+Return the encodings that the request accepts, in the order of the client's
+preference (most preferred first).
+
+#### .language(languages)
+
+Return the first accepted language. If nothing in `languages` is accepted,
+then `false` is returned.
+
+#### .languages()
+
+Return the languages that the request accepts, in the order of the client's
+preference (most preferred first).
+
+#### .type(types)
+
+Return the first accepted type (and it is returned as the same text as what
+appears in the `types` array). If nothing in `types` is accepted, then `false`
+is returned.
+
+The `types` array can contain full MIME types or file extensions. Any value
+that is not a full MIME types is passed to `require('mime-types').lookup`.
+
+#### .types()
+
+Return the types that the request accepts, in the order of the client's
+preference (most preferred first).
+
+## Examples
+
+### Simple type negotiation
+
+This simple example shows how to use `accepts` to return a different typed
+respond body based on what the client wants to accept. The server lists it's
+preferences in order and will get back the best match between the client and
+server.
+
+```js
+var accepts = require('accepts')
+var http = require('http')
+
+function app (req, res) {
+  var accept = accepts(req)
+
+  // the order of this list is significant; should be server preferred order
+  switch (accept.type(['json', 'html'])) {
+    case 'json':
+      res.setHeader('Content-Type', 'application/json')
+      res.write('{"hello":"world!"}')
+      break
+    case 'html':
+      res.setHeader('Content-Type', 'text/html')
+      res.write('<b>hello, world!</b>')
+      break
+    default:
+      // the fallback is text/plain, so no need to specify it above
+      res.setHeader('Content-Type', 'text/plain')
+      res.write('hello, world!')
+      break
+  }
+
+  res.end()
+}
+
+http.createServer(app).listen(3000)
+```
+
+You can test this out with the cURL program:
+```sh
+curl -I -H'Accept: text/html' http://localhost:3000/
+```
+
+## License
+
+[MIT](LICENSE)
+
+[coveralls-image]: https://badgen.net/coveralls/c/github/jshttp/accepts/master
+[coveralls-url]: https://coveralls.io/r/jshttp/accepts?branch=master
+[github-actions-ci-image]: https://badgen.net/github/checks/jshttp/accepts/master?label=ci
+[github-actions-ci-url]: https://github.com/jshttp/accepts/actions/workflows/ci.yml
+[node-version-image]: https://badgen.net/npm/node/accepts
+[node-version-url]: https://nodejs.org/en/download
+[npm-downloads-image]: https://badgen.net/npm/dm/accepts
+[npm-url]: https://npmjs.org/package/accepts
+[npm-version-image]: https://badgen.net/npm/v/accepts

scraper-service/node_modules/accepts/index.js

@@ -0,0 +1,238 @@
+/*!
+ * accepts
+ * Copyright(c) 2014 Jonathan Ong
+ * Copyright(c) 2015 Douglas Christopher Wilson
+ * MIT Licensed
+ */
+
+'use strict'
+
+/**
+ * Module dependencies.
+ * @private
+ */
+
+var Negotiator = require('negotiator')
+var mime = require('mime-types')
+
+/**
+ * Module exports.
+ * @public
+ */
+
+module.exports = Accepts
+
+/**
+ * Create a new Accepts object for the given req.
+ *
+ * @param {object} req
+ * @public
+ */
+
+function Accepts (req) {
+  if (!(this instanceof Accepts)) {
+    return new Accepts(req)
+  }
+
+  this.headers = req.headers
+  this.negotiator = new Negotiator(req)
+}
+
+/**
+ * Check if the given `type(s)` is acceptable, returning
+ * the best match when true, otherwise `undefined`, in which
+ * case you should respond with 406 "Not Acceptable".
+ *
+ * The `type` value may be a single mime type string
+ * such as "application/json", the extension name
+ * such as "json" or an array `["json", "html", "text/plain"]`. When a list
+ * or array is given the _best_ match, if any is returned.
+ *
+ * Examples:
+ *
+ *     // Accept: text/html
+ *     this.types('html');
+ *     // => "html"
+ *
+ *     // Accept: text/*, application/json
+ *     this.types('html');
+ *     // => "html"
+ *     this.types('text/html');
+ *     // => "text/html"
+ *     this.types('json', 'text');
+ *     // => "json"
+ *     this.types('application/json');
+ *     // => "application/json"
+ *
+ *     // Accept: text/*, application/json
+ *     this.types('image/png');
+ *     this.types('png');
+ *     // => undefined
+ *
+ *     // Accept: text/*;q=.5, application/json
+ *     this.types(['html', 'json']);
+ *     this.types('html', 'json');
+ *     // => "json"
+ *
+ * @param {String|Array} types...
+ * @return {String|Array|Boolean}
+ * @public
+ */
+
+Accepts.prototype.type =
+Accepts.prototype.types = function (types_) {
+  var types = types_
+
+  // support flattened arguments
+  if (types && !Array.isArray(types)) {
+    types = new Array(arguments.length)
+    for (var i = 0; i < types.length; i++) {
+      types[i] = arguments[i]
+    }
+  }
+
+  // no types, return all requested types
+  if (!types || types.length === 0) {
+    return this.negotiator.mediaTypes()
+  }
+
+  // no accept header, return first given type
+  if (!this.headers.accept) {
+    return types[0]
+  }
+
+  var mimes = types.map(extToMime)
+  var accepts = this.negotiator.mediaTypes(mimes.filter(validMime))
+  var first = accepts[0]
+
+  return first
+    ? types[mimes.indexOf(first)]
+    : false
+}
+
+/**
+ * Return accepted encodings or best fit based on `encodings`.
+ *
+ * Given `Accept-Encoding: gzip, deflate`
+ * an array sorted by quality is returned:
+ *
+ *     ['gzip', 'deflate']
+ *
+ * @param {String|Array} encodings...
+ * @return {String|Array}
+ * @public
+ */
+
+Accepts.prototype.encoding =
+Accepts.prototype.encodings = function (encodings_) {
+  var encodings = encodings_
+
+  // support flattened arguments
+  if (encodings && !Array.isArray(encodings)) {
+    encodings = new Array(arguments.length)
+    for (var i = 0; i < encodings.length; i++) {
+      encodings[i] = arguments[i]
+    }
+  }
+
+  // no encodings, return all requested encodings
+  if (!encodings || encodings.length === 0) {
+    return this.negotiator.encodings()
+  }
+
+  return this.negotiator.encodings(encodings)[0] || false
+}
+
+/**
+ * Return accepted charsets or best fit based on `charsets`.
+ *
+ * Given `Accept-Charset: utf-8, iso-8859-1;q=0.2, utf-7;q=0.5`
+ * an array sorted by quality is returned:
+ *
+ *     ['utf-8', 'utf-7', 'iso-8859-1']
+ *
+ * @param {String|Array} charsets...
+ * @return {String|Array}
+ * @public
+ */
+
+Accepts.prototype.charset =
+Accepts.prototype.charsets = function (charsets_) {
+  var charsets = charsets_
+
+  // support flattened arguments
+  if (charsets && !Array.isArray(charsets)) {
+    charsets = new Array(arguments.length)
+    for (var i = 0; i < charsets.length; i++) {
+      charsets[i] = arguments[i]
+    }
+  }
+
+  // no charsets, return all requested charsets
+  if (!charsets || charsets.length === 0) {
+    return this.negotiator.charsets()
+  }
+
+  return this.negotiator.charsets(charsets)[0] || false
+}
+
+/**
+ * Return accepted languages or best fit based on `langs`.
+ *
+ * Given `Accept-Language: en;q=0.8, es, pt`
+ * an array sorted by quality is returned:
+ *
+ *     ['es', 'pt', 'en']
+ *
+ * @param {String|Array} langs...
+ * @return {Array|String}
+ * @public
+ */
+
+Accepts.prototype.lang =
+Accepts.prototype.langs =
+Accepts.prototype.language =
+Accepts.prototype.languages = function (languages_) {
+  var languages = languages_
+
+  // support flattened arguments
+  if (languages && !Array.isArray(languages)) {
+    languages = new Array(arguments.length)
+    for (var i = 0; i < languages.length; i++) {
+      languages[i] = arguments[i]
+    }
+  }
+
+  // no languages, return all requested languages
+  if (!languages || languages.length === 0) {
+    return this.negotiator.languages()
+  }
+
+  return this.negotiator.languages(languages)[0] || false
+}
+
+/**
+ * Convert extnames to mime.
+ *
+ * @param {String} type
+ * @return {String}
+ * @private
+ */
+
+function extToMime (type) {
+  return type.indexOf('/') === -1
+    ? mime.lookup(type)
+    : type
+}
+
+/**
+ * Check if mime is valid.
+ *
+ * @param {String} type
+ * @return {String}
+ * @private
+ */
+
+function validMime (type) {
+  return typeof type === 'string'
+}

scraper-service/node_modules/accepts/package.json

@@ -0,0 +1,47 @@
+{
+  "name": "accepts",
+  "description": "Higher-level content negotiation",
+  "version": "1.3.8",
+  "contributors": [
+    "Douglas Christopher Wilson <doug@somethingdoug.com>",
+    "Jonathan Ong <me@jongleberry.com> (http://jongleberry.com)"
+  ],
+  "license": "MIT",
+  "repository": "jshttp/accepts",
+  "dependencies": {
+    "mime-types": "~2.1.34",
+    "negotiator": "0.6.3"
+  },
+  "devDependencies": {
+    "deep-equal": "1.0.1",
+    "eslint": "7.32.0",
+    "eslint-config-standard": "14.1.1",
+    "eslint-plugin-import": "2.25.4",
+    "eslint-plugin-markdown": "2.2.1",
+    "eslint-plugin-node": "11.1.0",
+    "eslint-plugin-promise": "4.3.1",
+    "eslint-plugin-standard": "4.1.0",
+    "mocha": "9.2.0",
+    "nyc": "15.1.0"
+  },
+  "files": [
+    "LICENSE",
+    "HISTORY.md",
+    "index.js"
+  ],
+  "engines": {
+    "node": ">= 0.6"
+  },
+  "scripts": {
+    "lint": "eslint .",
+    "test": "mocha --reporter spec --check-leaks --bail test/",
+    "test-ci": "nyc --reporter=lcov --reporter=text npm test",
+    "test-cov": "nyc --reporter=html --reporter=text npm test"
+  },
+  "keywords": [
+    "content",
+    "negotiation",
+    "accept",
+    "accepts"
+  ]
+}

scraper-service/node_modules/array-flatten/LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2014 Blake Embrey (hello@blakeembrey.com)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

scraper-service/node_modules/array-flatten/README.md

@@ -0,0 +1,43 @@
+# Array Flatten
+
+[![NPM version][npm-image]][npm-url]
+[![NPM downloads][downloads-image]][downloads-url]
+[![Build status][travis-image]][travis-url]
+[![Test coverage][coveralls-image]][coveralls-url]
+
+> Flatten an array of nested arrays into a single flat array. Accepts an optional depth.
+
+## Installation
+
+```
+npm install array-flatten --save
+```
+
+## Usage
+
+```javascript
+var flatten = require('array-flatten')
+
+flatten([1, [2, [3, [4, [5], 6], 7], 8], 9])
+//=> [1, 2, 3, 4, 5, 6, 7, 8, 9]
+
+flatten([1, [2, [3, [4, [5], 6], 7], 8], 9], 2)
+//=> [1, 2, 3, [4, [5], 6], 7, 8, 9]
+
+(function () {
+  flatten(arguments) //=> [1, 2, 3]
+})(1, [2, 3])
+```
+
+## License
+
+MIT
+
+[npm-image]: https://img.shields.io/npm/v/array-flatten.svg?style=flat
+[npm-url]: https://npmjs.org/package/array-flatten
+[downloads-image]: https://img.shields.io/npm/dm/array-flatten.svg?style=flat
+[downloads-url]: https://npmjs.org/package/array-flatten
+[travis-image]: https://img.shields.io/travis/blakeembrey/array-flatten.svg?style=flat
+[travis-url]: https://travis-ci.org/blakeembrey/array-flatten
+[coveralls-image]: https://img.shields.io/coveralls/blakeembrey/array-flatten.svg?style=flat
+[coveralls-url]: https://coveralls.io/r/blakeembrey/array-flatten?branch=master

scraper-service/node_modules/array-flatten/array-flatten.js

@@ -0,0 +1,64 @@
+'use strict'
+
+/**
+ * Expose `arrayFlatten`.
+ */
+module.exports = arrayFlatten
+
+/**
+ * Recursive flatten function with depth.
+ *
+ * @param  {Array}  array
+ * @param  {Array}  result
+ * @param  {Number} depth
+ * @return {Array}
+ */
+function flattenWithDepth (array, result, depth) {
+  for (var i = 0; i < array.length; i++) {
+    var value = array[i]
+
+    if (depth > 0 && Array.isArray(value)) {
+      flattenWithDepth(value, result, depth - 1)
+    } else {
+      result.push(value)
+    }
+  }
+
+  return result
+}
+
+/**
+ * Recursive flatten function. Omitting depth is slightly faster.
+ *
+ * @param  {Array} array
+ * @param  {Array} result
+ * @return {Array}
+ */
+function flattenForever (array, result) {
+  for (var i = 0; i < array.length; i++) {
+    var value = array[i]
+
+    if (Array.isArray(value)) {
+      flattenForever(value, result)
+    } else {
+      result.push(value)
+    }
+  }
+
+  return result
+}
+
+/**
+ * Flatten an array, with the ability to define a depth.
+ *
+ * @param  {Array}  array
+ * @param  {Number} depth
+ * @return {Array}
+ */
+function arrayFlatten (array, depth) {
+  if (depth == null) {
+    return flattenForever(array, [])
+  }
+
+  return flattenWithDepth(array, [], depth)
+}

scraper-service/node_modules/array-flatten/package.json

@@ -0,0 +1,39 @@
+{
+  "name": "array-flatten",
+  "version": "1.1.1",
+  "description": "Flatten an array of nested arrays into a single flat array",
+  "main": "array-flatten.js",
+  "files": [
+    "array-flatten.js",
+    "LICENSE"
+  ],
+  "scripts": {
+    "test": "istanbul cover _mocha -- -R spec"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git://github.com/blakeembrey/array-flatten.git"
+  },
+  "keywords": [
+    "array",
+    "flatten",
+    "arguments",
+    "depth"
+  ],
+  "author": {
+    "name": "Blake Embrey",
+    "email": "hello@blakeembrey.com",
+    "url": "http://blakeembrey.me"
+  },
+  "license": "MIT",
+  "bugs": {
+    "url": "https://github.com/blakeembrey/array-flatten/issues"
+  },
+  "homepage": "https://github.com/blakeembrey/array-flatten",
+  "devDependencies": {
+    "istanbul": "^0.3.13",
+    "mocha": "^2.2.4",
+    "pre-commit": "^1.0.7",
+    "standard": "^3.7.3"
+  }
+}

scraper-service/node_modules/body-parser/HISTORY.md

@@ -0,0 +1,686 @@
+1.20.5 / 2026-04-24
+===================
+* refactor(json): simplify strict mode error string construction 
+* fix: extended urlencoded parsing of arrays with >100 elements (#716)
+* deps: qs@~6.15.1
+
+1.20.4 / 2025-12-01
+===================
+
+  * deps: qs@~6.14.0
+  * deps: use tilde notation for dependencies
+  * deps: http-errors@~2.0.1
+  * deps: raw-body@~2.5.3
+
+1.20.3 / 2024-09-10
+===================
+
+  * deps: qs@6.13.0
+  * add `depth` option to customize the depth level in the parser
+  * IMPORTANT: The default `depth` level for parsing URL-encoded data is now `32` (previously was `Infinity`)
+
+1.20.2 / 2023-02-21
+===================
+
+  * Fix strict json error message on Node.js 19+
+  * deps: content-type@~1.0.5
+    - perf: skip value escaping when unnecessary
+  * deps: raw-body@2.5.2
+
+1.20.1 / 2022-10-06
+===================
+
+  * deps: qs@6.11.0
+  * perf: remove unnecessary object clone
+
+1.20.0 / 2022-04-02
+===================
+
+  * Fix error message for json parse whitespace in `strict`
+  * Fix internal error when inflated body exceeds limit
+  * Prevent loss of async hooks context
+  * Prevent hanging when request already read
+  * deps: depd@2.0.0
+    - Replace internal `eval` usage with `Function` constructor
+    - Use instance methods on `process` to check for listeners
+  * deps: http-errors@2.0.0
+    - deps: depd@2.0.0
+    - deps: statuses@2.0.1
+  * deps: on-finished@2.4.1
+  * deps: qs@6.10.3
+  * deps: raw-body@2.5.1
+    - deps: http-errors@2.0.0
+
+1.19.2 / 2022-02-15
+===================
+
+  * deps: bytes@3.1.2
+  * deps: qs@6.9.7
+    * Fix handling of `__proto__` keys
+  * deps: raw-body@2.4.3
+    - deps: bytes@3.1.2
+
+1.19.1 / 2021-12-10
+===================
+
+  * deps: bytes@3.1.1
+  * deps: http-errors@1.8.1
+    - deps: inherits@2.0.4
+    - deps: toidentifier@1.0.1
+    - deps: setprototypeof@1.2.0
+  * deps: qs@6.9.6
+  * deps: raw-body@2.4.2
+    - deps: bytes@3.1.1
+    - deps: http-errors@1.8.1
+  * deps: safe-buffer@5.2.1
+  * deps: type-is@~1.6.18
+
+1.19.0 / 2019-04-25
+===================
+
+  * deps: bytes@3.1.0
+    - Add petabyte (`pb`) support
+  * deps: http-errors@1.7.2
+    - Set constructor name when possible
+    - deps: setprototypeof@1.1.1
+    - deps: statuses@'>= 1.5.0 < 2'
+  * deps: iconv-lite@0.4.24
+    - Added encoding MIK
+  * deps: qs@6.7.0
+    - Fix parsing array brackets after index
+  * deps: raw-body@2.4.0
+    - deps: bytes@3.1.0
+    - deps: http-errors@1.7.2
+    - deps: iconv-lite@0.4.24
+  * deps: type-is@~1.6.17
+    - deps: mime-types@~2.1.24
+    - perf: prevent internal `throw` on invalid type
+
+1.18.3 / 2018-05-14
+===================
+
+  * Fix stack trace for strict json parse error
+  * deps: depd@~1.1.2
+    - perf: remove argument reassignment
+  * deps: http-errors@~1.6.3
+    - deps: depd@~1.1.2
+    - deps: setprototypeof@1.1.0
+    - deps: statuses@'>= 1.3.1 < 2'
+  * deps: iconv-lite@0.4.23
+    - Fix loading encoding with year appended
+    - Fix deprecation warnings on Node.js 10+
+  * deps: qs@6.5.2
+  * deps: raw-body@2.3.3
+    - deps: http-errors@1.6.3
+    - deps: iconv-lite@0.4.23
+  * deps: type-is@~1.6.16
+    - deps: mime-types@~2.1.18
+
+1.18.2 / 2017-09-22
+===================
+
+  * deps: debug@2.6.9
+  * perf: remove argument reassignment
+
+1.18.1 / 2017-09-12
+===================
+
+  * deps: content-type@~1.0.4
+    - perf: remove argument reassignment
+    - perf: skip parameter parsing when no parameters
+  * deps: iconv-lite@0.4.19
+    - Fix ISO-8859-1 regression
+    - Update Windows-1255
+  * deps: qs@6.5.1
+    - Fix parsing & compacting very deep objects
+  * deps: raw-body@2.3.2
+    - deps: iconv-lite@0.4.19
+
+1.18.0 / 2017-09-08
+===================
+
+  * Fix JSON strict violation error to match native parse error
+  * Include the `body` property on verify errors
+  * Include the `type` property on all generated errors
+  * Use `http-errors` to set status code on errors
+  * deps: bytes@3.0.0
+  * deps: debug@2.6.8
+  * deps: depd@~1.1.1
+    - Remove unnecessary `Buffer` loading
+  * deps: http-errors@~1.6.2
+    - deps: depd@1.1.1
+  * deps: iconv-lite@0.4.18
+    - Add support for React Native
+    - Add a warning if not loaded as utf-8
+    - Fix CESU-8 decoding in Node.js 8
+    - Improve speed of ISO-8859-1 encoding
+  * deps: qs@6.5.0
+  * deps: raw-body@2.3.1
+    - Use `http-errors` for standard emitted errors
+    - deps: bytes@3.0.0
+    - deps: iconv-lite@0.4.18
+    - perf: skip buffer decoding on overage chunk
+  * perf: prevent internal `throw` when missing charset
+
+1.17.2 / 2017-05-17
+===================
+
+  * deps: debug@2.6.7
+    - Fix `DEBUG_MAX_ARRAY_LENGTH`
+    - deps: ms@2.0.0
+  * deps: type-is@~1.6.15
+    - deps: mime-types@~2.1.15
+
+1.17.1 / 2017-03-06
+===================
+
+  * deps: qs@6.4.0
+    - Fix regression parsing keys starting with `[`
+
+1.17.0 / 2017-03-01
+===================
+
+  * deps: http-errors@~1.6.1
+    - Make `message` property enumerable for `HttpError`s
+    - deps: setprototypeof@1.0.3
+  * deps: qs@6.3.1
+    - Fix compacting nested arrays
+
+1.16.1 / 2017-02-10
+===================
+
+  * deps: debug@2.6.1
+    - Fix deprecation messages in WebStorm and other editors
+    - Undeprecate `DEBUG_FD` set to `1` or `2`
+
+1.16.0 / 2017-01-17
+===================
+
+  * deps: debug@2.6.0
+    - Allow colors in workers
+    - Deprecated `DEBUG_FD` environment variable
+    - Fix error when running under React Native
+    - Use same color for same namespace
+    - deps: ms@0.7.2
+  * deps: http-errors@~1.5.1
+    - deps: inherits@2.0.3
+    - deps: setprototypeof@1.0.2
+    - deps: statuses@'>= 1.3.1 < 2'
+  * deps: iconv-lite@0.4.15
+    - Added encoding MS-31J
+    - Added encoding MS-932
+    - Added encoding MS-936
+    - Added encoding MS-949
+    - Added encoding MS-950
+    - Fix GBK/GB18030 handling of Euro character
+  * deps: qs@6.2.1
+    - Fix array parsing from skipping empty values
+  * deps: raw-body@~2.2.0
+    - deps: iconv-lite@0.4.15
+  * deps: type-is@~1.6.14
+    - deps: mime-types@~2.1.13
+
+1.15.2 / 2016-06-19
+===================
+
+  * deps: bytes@2.4.0
+  * deps: content-type@~1.0.2
+    - perf: enable strict mode
+  * deps: http-errors@~1.5.0
+    - Use `setprototypeof` module to replace `__proto__` setting
+    - deps: statuses@'>= 1.3.0 < 2'
+    - perf: enable strict mode
+  * deps: qs@6.2.0
+  * deps: raw-body@~2.1.7
+    - deps: bytes@2.4.0
+    - perf: remove double-cleanup on happy path
+  * deps: type-is@~1.6.13
+    - deps: mime-types@~2.1.11
+
+1.15.1 / 2016-05-05
+===================
+
+  * deps: bytes@2.3.0
+    - Drop partial bytes on all parsed units
+    - Fix parsing byte string that looks like hex
+  * deps: raw-body@~2.1.6
+    - deps: bytes@2.3.0
+  * deps: type-is@~1.6.12
+    - deps: mime-types@~2.1.10
+
+1.15.0 / 2016-02-10
+===================
+
+  * deps: http-errors@~1.4.0
+    - Add `HttpError` export, for `err instanceof createError.HttpError`
+    - deps: inherits@2.0.1
+    - deps: statuses@'>= 1.2.1 < 2'
+  * deps: qs@6.1.0
+  * deps: type-is@~1.6.11
+    - deps: mime-types@~2.1.9
+
+1.14.2 / 2015-12-16
+===================
+
+  * deps: bytes@2.2.0
+  * deps: iconv-lite@0.4.13
+  * deps: qs@5.2.0
+  * deps: raw-body@~2.1.5
+    - deps: bytes@2.2.0
+    - deps: iconv-lite@0.4.13
+  * deps: type-is@~1.6.10
+    - deps: mime-types@~2.1.8
+
+1.14.1 / 2015-09-27
+===================
+
+  * Fix issue where invalid charset results in 400 when `verify` used
+  * deps: iconv-lite@0.4.12
+    - Fix CESU-8 decoding in Node.js 4.x
+  * deps: raw-body@~2.1.4
+    - Fix masking critical errors from `iconv-lite`
+    - deps: iconv-lite@0.4.12
+  * deps: type-is@~1.6.9
+    - deps: mime-types@~2.1.7
+
+1.14.0 / 2015-09-16
+===================
+
+  * Fix JSON strict parse error to match syntax errors
+  * Provide static `require` analysis in `urlencoded` parser
+  * deps: depd@~1.1.0
+    - Support web browser loading
+  * deps: qs@5.1.0
+  * deps: raw-body@~2.1.3
+    - Fix sync callback when attaching data listener causes sync read
+  * deps: type-is@~1.6.8
+    - Fix type error when given invalid type to match against
+    - deps: mime-types@~2.1.6
+
+1.13.3 / 2015-07-31
+===================
+
+  * deps: type-is@~1.6.6
+    - deps: mime-types@~2.1.4
+
+1.13.2 / 2015-07-05
+===================
+
+  * deps: iconv-lite@0.4.11
+  * deps: qs@4.0.0
+    - Fix dropping parameters like `hasOwnProperty`
+    - Fix user-visible incompatibilities from 3.1.0
+    - Fix various parsing edge cases
+  * deps: raw-body@~2.1.2
+    - Fix error stack traces to skip `makeError`
+    - deps: iconv-lite@0.4.11
+  * deps: type-is@~1.6.4
+    - deps: mime-types@~2.1.2
+    - perf: enable strict mode
+    - perf: remove argument reassignment
+
+1.13.1 / 2015-06-16
+===================
+
+  * deps: qs@2.4.2
+    - Downgraded from 3.1.0 because of user-visible incompatibilities
+
+1.13.0 / 2015-06-14
+===================
+
+  * Add `statusCode` property on `Error`s, in addition to `status`
+  * Change `type` default to `application/json` for JSON parser
+  * Change `type` default to `application/x-www-form-urlencoded` for urlencoded parser
+  * Provide static `require` analysis
+  * Use the `http-errors` module to generate errors
+  * deps: bytes@2.1.0
+    - Slight optimizations
+  * deps: iconv-lite@0.4.10
+    - The encoding UTF-16 without BOM now defaults to UTF-16LE when detection fails
+    - Leading BOM is now removed when decoding
+  * deps: on-finished@~2.3.0
+    - Add defined behavior for HTTP `CONNECT` requests
+    - Add defined behavior for HTTP `Upgrade` requests
+    - deps: ee-first@1.1.1
+  * deps: qs@3.1.0
+    - Fix dropping parameters like `hasOwnProperty`
+    - Fix various parsing edge cases
+    - Parsed object now has `null` prototype
+  * deps: raw-body@~2.1.1
+    - Use `unpipe` module for unpiping requests
+    - deps: iconv-lite@0.4.10
+  * deps: type-is@~1.6.3
+    - deps: mime-types@~2.1.1
+    - perf: reduce try block size
+    - perf: remove bitwise operations
+  * perf: enable strict mode
+  * perf: remove argument reassignment
+  * perf: remove delete call
+
+1.12.4 / 2015-05-10
+===================
+
+  * deps: debug@~2.2.0
+  * deps: qs@2.4.2
+    - Fix allowing parameters like `constructor`
+  * deps: on-finished@~2.2.1
+  * deps: raw-body@~2.0.1
+    - Fix a false-positive when unpiping in Node.js 0.8
+    - deps: bytes@2.0.1
+  * deps: type-is@~1.6.2
+    - deps: mime-types@~2.0.11
+
+1.12.3 / 2015-04-15
+===================
+
+  * Slight efficiency improvement when not debugging
+  * deps: depd@~1.0.1
+  * deps: iconv-lite@0.4.8
+    - Add encoding alias UNICODE-1-1-UTF-7
+  * deps: raw-body@1.3.4
+    - Fix hanging callback if request aborts during read
+    - deps: iconv-lite@0.4.8
+
+1.12.2 / 2015-03-16
+===================
+
+  * deps: qs@2.4.1
+    - Fix error when parameter `hasOwnProperty` is present
+
+1.12.1 / 2015-03-15
+===================
+
+  * deps: debug@~2.1.3
+    - Fix high intensity foreground color for bold
+    - deps: ms@0.7.0
+  * deps: type-is@~1.6.1
+    - deps: mime-types@~2.0.10
+
+1.12.0 / 2015-02-13
+===================
+
+  * add `debug` messages
+  * accept a function for the `type` option
+  * use `content-type` to parse `Content-Type` headers
+  * deps: iconv-lite@0.4.7
+    - Gracefully support enumerables on `Object.prototype`
+  * deps: raw-body@1.3.3
+    - deps: iconv-lite@0.4.7
+  * deps: type-is@~1.6.0
+    - fix argument reassignment
+    - fix false-positives in `hasBody` `Transfer-Encoding` check
+    - support wildcard for both type and subtype (`*/*`)
+    - deps: mime-types@~2.0.9
+
+1.11.0 / 2015-01-30
+===================
+
+  * make internal `extended: true` depth limit infinity
+  * deps: type-is@~1.5.6
+    - deps: mime-types@~2.0.8
+
+1.10.2 / 2015-01-20
+===================
+
+  * deps: iconv-lite@0.4.6
+    - Fix rare aliases of single-byte encodings
+  * deps: raw-body@1.3.2
+    - deps: iconv-lite@0.4.6
+
+1.10.1 / 2015-01-01
+===================
+
+  * deps: on-finished@~2.2.0
+  * deps: type-is@~1.5.5
+    - deps: mime-types@~2.0.7
+
+1.10.0 / 2014-12-02
+===================
+
+  * make internal `extended: true` array limit dynamic
+
+1.9.3 / 2014-11-21
+==================
+
+  * deps: iconv-lite@0.4.5
+    - Fix Windows-31J and X-SJIS encoding support
+  * deps: qs@2.3.3
+    - Fix `arrayLimit` behavior
+  * deps: raw-body@1.3.1
+    - deps: iconv-lite@0.4.5
+  * deps: type-is@~1.5.3
+    - deps: mime-types@~2.0.3
+
+1.9.2 / 2014-10-27
+==================
+
+  * deps: qs@2.3.2
+    - Fix parsing of mixed objects and values
+
+1.9.1 / 2014-10-22
+==================
+
+  * deps: on-finished@~2.1.1
+    - Fix handling of pipelined requests
+  * deps: qs@2.3.0
+    - Fix parsing of mixed implicit and explicit arrays
+  * deps: type-is@~1.5.2
+    - deps: mime-types@~2.0.2
+
+1.9.0 / 2014-09-24
+==================
+
+  * include the charset in "unsupported charset" error message
+  * include the encoding in "unsupported content encoding" error message
+  * deps: depd@~1.0.0
+
+1.8.4 / 2014-09-23
+==================
+
+  * fix content encoding to be case-insensitive
+
+1.8.3 / 2014-09-19
+==================
+
+  * deps: qs@2.2.4
+    - Fix issue with object keys starting with numbers truncated
+
+1.8.2 / 2014-09-15
+==================
+
+  * deps: depd@0.4.5
+
+1.8.1 / 2014-09-07
+==================
+
+  * deps: media-typer@0.3.0
+  * deps: type-is@~1.5.1
+
+1.8.0 / 2014-09-05
+==================
+
+  * make empty-body-handling consistent between chunked requests
+    - empty `json` produces `{}`
+    - empty `raw` produces `new Buffer(0)`
+    - empty `text` produces `''`
+    - empty `urlencoded` produces `{}`
+  * deps: qs@2.2.3
+    - Fix issue where first empty value in array is discarded
+  * deps: type-is@~1.5.0
+    - fix `hasbody` to be true for `content-length: 0`
+
+1.7.0 / 2014-09-01
+==================
+
+  * add `parameterLimit` option to `urlencoded` parser
+  * change `urlencoded` extended array limit to 100
+  * respond with 413 when over `parameterLimit` in `urlencoded`
+
+1.6.7 / 2014-08-29
+==================
+
+  * deps: qs@2.2.2
+    - Remove unnecessary cloning
+
+1.6.6 / 2014-08-27
+==================
+
+  * deps: qs@2.2.0
+    - Array parsing fix
+    - Performance improvements
+
+1.6.5 / 2014-08-16
+==================
+
+  * deps: on-finished@2.1.0
+
+1.6.4 / 2014-08-14
+==================
+
+  * deps: qs@1.2.2
+
+1.6.3 / 2014-08-10
+==================
+
+  * deps: qs@1.2.1
+
+1.6.2 / 2014-08-07
+==================
+
+  * deps: qs@1.2.0
+    - Fix parsing array of objects
+
+1.6.1 / 2014-08-06
+==================
+
+  * deps: qs@1.1.0
+    - Accept urlencoded square brackets
+    - Accept empty values in implicit array notation
+
+1.6.0 / 2014-08-05
+==================
+
+  * deps: qs@1.0.2
+    - Complete rewrite
+    - Limits array length to 20
+    - Limits object depth to 5
+    - Limits parameters to 1,000
+
+1.5.2 / 2014-07-27
+==================
+
+  * deps: depd@0.4.4
+    - Work-around v8 generating empty stack traces
+
+1.5.1 / 2014-07-26
+==================
+
+  * deps: depd@0.4.3
+    - Fix exception when global `Error.stackTraceLimit` is too low
+
+1.5.0 / 2014-07-20
+==================
+
+  * deps: depd@0.4.2
+    - Add `TRACE_DEPRECATION` environment variable
+    - Remove non-standard grey color from color output
+    - Support `--no-deprecation` argument
+    - Support `--trace-deprecation` argument
+  * deps: iconv-lite@0.4.4
+    - Added encoding UTF-7
+  * deps: raw-body@1.3.0
+    - deps: iconv-lite@0.4.4
+    - Added encoding UTF-7
+    - Fix `Cannot switch to old mode now` error on Node.js 0.10+
+  * deps: type-is@~1.3.2
+
+1.4.3 / 2014-06-19
+==================
+
+  * deps: type-is@1.3.1
+    - fix global variable leak
+
+1.4.2 / 2014-06-19
+==================
+
+  * deps: type-is@1.3.0
+    - improve type parsing
+
+1.4.1 / 2014-06-19
+==================
+
+  * fix urlencoded extended deprecation message
+
+1.4.0 / 2014-06-19
+==================
+
+  * add `text` parser
+  * add `raw` parser
+  * check accepted charset in content-type (accepts utf-8)
+  * check accepted encoding in content-encoding (accepts identity)
+  * deprecate `bodyParser()` middleware; use `.json()` and `.urlencoded()` as needed
+  * deprecate `urlencoded()` without provided `extended` option
+  * lazy-load urlencoded parsers
+  * parsers split into files for reduced mem usage
+  * support gzip and deflate bodies
+    - set `inflate: false` to turn off
+  * deps: raw-body@1.2.2
+    - Support all encodings from `iconv-lite`
+
+1.3.1 / 2014-06-11
+==================
+
+  * deps: type-is@1.2.1
+    - Switch dependency from mime to mime-types@1.0.0
+
+1.3.0 / 2014-05-31
+==================
+
+  * add `extended` option to urlencoded parser
+
+1.2.2 / 2014-05-27
+==================
+
+  * deps: raw-body@1.1.6
+    - assert stream encoding on node.js 0.8
+    - assert stream encoding on node.js < 0.10.6
+    - deps: bytes@1
+
+1.2.1 / 2014-05-26
+==================
+
+  * invoke `next(err)` after request fully read
+    - prevents hung responses and socket hang ups
+
+1.2.0 / 2014-05-11
+==================
+
+  * add `verify` option
+  * deps: type-is@1.2.0
+    - support suffix matching
+
+1.1.2 / 2014-05-11
+==================
+
+  * improve json parser speed
+
+1.1.1 / 2014-05-11
+==================
+
+  * fix repeated limit parsing with every request
+
+1.1.0 / 2014-05-10
+==================
+
+  * add `type` option
+  * deps: pin for safety and consistency
+
+1.0.2 / 2014-04-14
+==================
+
+  * use `type-is` module
+
+1.0.1 / 2014-03-20
+==================
+
+  * lower default limits to 100kb

scraper-service/node_modules/body-parser/LICENSE

@@ -0,0 +1,23 @@
+(The MIT License)
+
+Copyright (c) 2014 Jonathan Ong <me@jongleberry.com>
+Copyright (c) 2014-2015 Douglas Christopher Wilson <doug@somethingdoug.com>
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+'Software'), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

scraper-service/node_modules/body-parser/README.md

@@ -0,0 +1,476 @@
+# body-parser
+
+[![NPM Version][npm-version-image]][npm-url]
+[![NPM Downloads][npm-downloads-image]][npm-url]
+[![Build Status][ci-image]][ci-url]
+[![Test Coverage][coveralls-image]][coveralls-url]
+[![OpenSSF Scorecard Badge][ossf-scorecard-badge]][ossf-scorecard-visualizer]
+
+Node.js body parsing middleware.
+
+Parse incoming request bodies in a middleware before your handlers, available
+under the `req.body` property.
+
+**Note** As `req.body`'s shape is based on user-controlled input, all
+properties and values in this object are untrusted and should be validated
+before trusting. For example, `req.body.foo.toString()` may fail in multiple
+ways, for example the `foo` property may not be there or may not be a string,
+and `toString` may not be a function and instead a string or other user input.
+
+[Learn about the anatomy of an HTTP transaction in Node.js](https://nodejs.org/en/docs/guides/anatomy-of-an-http-transaction/).
+
+_This does not handle multipart bodies_, due to their complex and typically
+large nature. For multipart bodies, you may be interested in the following
+modules:
+
+  * [busboy](https://www.npmjs.org/package/busboy#readme) and
+    [connect-busboy](https://www.npmjs.org/package/connect-busboy#readme)
+  * [multiparty](https://www.npmjs.org/package/multiparty#readme) and
+    [connect-multiparty](https://www.npmjs.org/package/connect-multiparty#readme)
+  * [formidable](https://www.npmjs.org/package/formidable#readme)
+  * [multer](https://www.npmjs.org/package/multer#readme)
+
+This module provides the following parsers:
+
+  * [JSON body parser](#bodyparserjsonoptions)
+  * [Raw body parser](#bodyparserrawoptions)
+  * [Text body parser](#bodyparsertextoptions)
+  * [URL-encoded form body parser](#bodyparserurlencodedoptions)
+
+Other body parsers you might be interested in:
+
+- [body](https://www.npmjs.org/package/body#readme)
+- [co-body](https://www.npmjs.org/package/co-body#readme)
+
+## Installation
+
+```sh
+$ npm install body-parser
+```
+
+## API
+
+```js
+var bodyParser = require('body-parser')
+```
+
+The `bodyParser` object exposes various factories to create middlewares. All
+middlewares will populate the `req.body` property with the parsed body when
+the `Content-Type` request header matches the `type` option, or an empty
+object (`{}`) if there was no body to parse, the `Content-Type` was not matched,
+or an error occurred.
+
+The various errors returned by this module are described in the
+[errors section](#errors).
+
+### bodyParser.json([options])
+
+Returns middleware that only parses `json` and only looks at requests where
+the `Content-Type` header matches the `type` option. This parser accepts any
+Unicode encoding of the body and supports automatic inflation of `gzip` and
+`deflate` encodings.
+
+A new `body` object containing the parsed data is populated on the `request`
+object after the middleware (i.e. `req.body`).
+
+#### Options
+
+The `json` function takes an optional `options` object that may contain any of
+the following keys:
+
+##### inflate
+
+When set to `true`, then deflated (compressed) bodies will be inflated; when
+`false`, deflated bodies are rejected. Defaults to `true`.
+
+##### limit
+
+Controls the maximum request body size. If this is a number, then the value
+specifies the number of bytes; if it is a string, the value is passed to the
+[bytes](https://www.npmjs.com/package/bytes) library for parsing. Defaults
+to `'100kb'`.
+
+##### reviver
+
+The `reviver` option is passed directly to `JSON.parse` as the second
+argument. You can find more information on this argument
+[in the MDN documentation about JSON.parse](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/JSON/parse#Example.3A_Using_the_reviver_parameter).
+
+##### strict
+
+When set to `true`, will only accept arrays and objects; when `false` will
+accept anything `JSON.parse` accepts. Defaults to `true`.
+
+##### type
+
+The `type` option is used to determine what media type the middleware will
+parse. This option can be a string, array of strings, or a function. If not a
+function, `type` option is passed directly to the
+[type-is](https://www.npmjs.org/package/type-is#readme) library and this can
+be an extension name (like `json`), a mime type (like `application/json`), or
+a mime type with a wildcard (like `*/*` or `*/json`). If a function, the `type`
+option is called as `fn(req)` and the request is parsed if it returns a truthy
+value. Defaults to `application/json`.
+
+##### verify
+
+The `verify` option, if supplied, is called as `verify(req, res, buf, encoding)`,
+where `buf` is a `Buffer` of the raw request body and `encoding` is the
+encoding of the request. The parsing can be aborted by throwing an error.
+
+### bodyParser.raw([options])
+
+Returns middleware that parses all bodies as a `Buffer` and only looks at
+requests where the `Content-Type` header matches the `type` option. This
+parser supports automatic inflation of `gzip` and `deflate` encodings.
+
+A new `body` object containing the parsed data is populated on the `request`
+object after the middleware (i.e. `req.body`). This will be a `Buffer` object
+of the body.
+
+#### Options
+
+The `raw` function takes an optional `options` object that may contain any of
+the following keys:
+
+##### inflate
+
+When set to `true`, then deflated (compressed) bodies will be inflated; when
+`false`, deflated bodies are rejected. Defaults to `true`.
+
+##### limit
+
+Controls the maximum request body size. If this is a number, then the value
+specifies the number of bytes; if it is a string, the value is passed to the
+[bytes](https://www.npmjs.com/package/bytes) library for parsing. Defaults
+to `'100kb'`.
+
+##### type
+
+The `type` option is used to determine what media type the middleware will
+parse. This option can be a string, array of strings, or a function.
+If not a function, `type` option is passed directly to the
+[type-is](https://www.npmjs.org/package/type-is#readme) library and this
+can be an extension name (like `bin`), a mime type (like
+`application/octet-stream`), or a mime type with a wildcard (like `*/*` or
+`application/*`). If a function, the `type` option is called as `fn(req)`
+and the request is parsed if it returns a truthy value. Defaults to
+`application/octet-stream`.
+
+##### verify
+
+The `verify` option, if supplied, is called as `verify(req, res, buf, encoding)`,
+where `buf` is a `Buffer` of the raw request body and `encoding` is the
+encoding of the request. The parsing can be aborted by throwing an error.
+
+### bodyParser.text([options])
+
+Returns middleware that parses all bodies as a string and only looks at
+requests where the `Content-Type` header matches the `type` option. This
+parser supports automatic inflation of `gzip` and `deflate` encodings.
+
+A new `body` string containing the parsed data is populated on the `request`
+object after the middleware (i.e. `req.body`). This will be a string of the
+body.
+
+#### Options
+
+The `text` function takes an optional `options` object that may contain any of
+the following keys:
+
+##### defaultCharset
+
+Specify the default character set for the text content if the charset is not
+specified in the `Content-Type` header of the request. Defaults to `utf-8`.
+
+##### inflate
+
+When set to `true`, then deflated (compressed) bodies will be inflated; when
+`false`, deflated bodies are rejected. Defaults to `true`.
+
+##### limit
+
+Controls the maximum request body size. If this is a number, then the value
+specifies the number of bytes; if it is a string, the value is passed to the
+[bytes](https://www.npmjs.com/package/bytes) library for parsing. Defaults
+to `'100kb'`.
+
+##### type
+
+The `type` option is used to determine what media type the middleware will
+parse. This option can be a string, array of strings, or a function. If not
+a function, `type` option is passed directly to the
+[type-is](https://www.npmjs.org/package/type-is#readme) library and this can
+be an extension name (like `txt`), a mime type (like `text/plain`), or a mime
+type with a wildcard (like `*/*` or `text/*`). If a function, the `type`
+option is called as `fn(req)` and the request is parsed if it returns a
+truthy value. Defaults to `text/plain`.
+
+##### verify
+
+The `verify` option, if supplied, is called as `verify(req, res, buf, encoding)`,
+where `buf` is a `Buffer` of the raw request body and `encoding` is the
+encoding of the request. The parsing can be aborted by throwing an error.
+
+### bodyParser.urlencoded([options])
+
+Returns middleware that only parses `urlencoded` bodies and only looks at
+requests where the `Content-Type` header matches the `type` option. This
+parser accepts only UTF-8 encoding of the body and supports automatic
+inflation of `gzip` and `deflate` encodings.
+
+A new `body` object containing the parsed data is populated on the `request`
+object after the middleware (i.e. `req.body`). This object will contain
+key-value pairs, where the value can be a string or array (when `extended` is
+`false`), or any type (when `extended` is `true`).
+
+#### Options
+
+The `urlencoded` function takes an optional `options` object that may contain
+any of the following keys:
+
+##### extended
+
+The `extended` option allows to choose between parsing the URL-encoded data
+with the `querystring` library (when `false`) or the `qs` library (when
+`true`). The "extended" syntax allows for rich objects and arrays to be
+encoded into the URL-encoded format, allowing for a JSON-like experience
+with URL-encoded. For more information, please
+[see the qs library](https://www.npmjs.org/package/qs#readme).
+
+Defaults to `true`, but using the default has been deprecated. Please
+research into the difference between `qs` and `querystring` and choose the
+appropriate setting.
+
+##### inflate
+
+When set to `true`, then deflated (compressed) bodies will be inflated; when
+`false`, deflated bodies are rejected. Defaults to `true`.
+
+##### limit
+
+Controls the maximum request body size. If this is a number, then the value
+specifies the number of bytes; if it is a string, the value is passed to the
+[bytes](https://www.npmjs.com/package/bytes) library for parsing. Defaults
+to `'100kb'`.
+
+##### parameterLimit
+
+The `parameterLimit` option controls the maximum number of parameters that
+are allowed in the URL-encoded data. If a request contains more parameters
+than this value, a 413 will be returned to the client. Defaults to `1000`.
+
+##### type
+
+The `type` option is used to determine what media type the middleware will
+parse. This option can be a string, array of strings, or a function. If not
+a function, `type` option is passed directly to the
+[type-is](https://www.npmjs.org/package/type-is#readme) library and this can
+be an extension name (like `urlencoded`), a mime type (like
+`application/x-www-form-urlencoded`), or a mime type with a wildcard (like
+`*/x-www-form-urlencoded`). If a function, the `type` option is called as
+`fn(req)` and the request is parsed if it returns a truthy value. Defaults
+to `application/x-www-form-urlencoded`.
+
+##### verify
+
+The `verify` option, if supplied, is called as `verify(req, res, buf, encoding)`,
+where `buf` is a `Buffer` of the raw request body and `encoding` is the
+encoding of the request. The parsing can be aborted by throwing an error.
+
+#### depth
+
+The `depth` option is used to configure the maximum depth of the `qs` library when `extended` is `true`. This allows you to limit the amount of keys that are parsed and can be useful to prevent certain types of abuse. Defaults to `32`. It is recommended to keep this value as low as possible.
+
+## Errors
+
+The middlewares provided by this module create errors using the
+[`http-errors` module](https://www.npmjs.com/package/http-errors). The errors
+will typically have a `status`/`statusCode` property that contains the suggested
+HTTP response code, an `expose` property to determine if the `message` property
+should be displayed to the client, a `type` property to determine the type of
+error without matching against the `message`, and a `body` property containing
+the read body, if available.
+
+The following are the common errors created, though any error can come through
+for various reasons.
+
+### content encoding unsupported
+
+This error will occur when the request had a `Content-Encoding` header that
+contained an encoding but the "inflation" option was set to `false`. The
+`status` property is set to `415`, the `type` property is set to
+`'encoding.unsupported'`, and the `charset` property will be set to the
+encoding that is unsupported.
+
+### entity parse failed
+
+This error will occur when the request contained an entity that could not be
+parsed by the middleware. The `status` property is set to `400`, the `type`
+property is set to `'entity.parse.failed'`, and the `body` property is set to
+the entity value that failed parsing.
+
+### entity verify failed
+
+This error will occur when the request contained an entity that could not be
+failed verification by the defined `verify` option. The `status` property is
+set to `403`, the `type` property is set to `'entity.verify.failed'`, and the
+`body` property is set to the entity value that failed verification.
+
+### request aborted
+
+This error will occur when the request is aborted by the client before reading
+the body has finished. The `received` property will be set to the number of
+bytes received before the request was aborted and the `expected` property is
+set to the number of expected bytes. The `status` property is set to `400`
+and `type` property is set to `'request.aborted'`.
+
+### request entity too large
+
+This error will occur when the request body's size is larger than the "limit"
+option. The `limit` property will be set to the byte limit and the `length`
+property will be set to the request body's length. The `status` property is
+set to `413` and the `type` property is set to `'entity.too.large'`.
+
+### request size did not match content length
+
+This error will occur when the request's length did not match the length from
+the `Content-Length` header. This typically occurs when the request is malformed,
+typically when the `Content-Length` header was calculated based on characters
+instead of bytes. The `status` property is set to `400` and the `type` property
+is set to `'request.size.invalid'`.
+
+### stream encoding should not be set
+
+This error will occur when something called the `req.setEncoding` method prior
+to this middleware. This module operates directly on bytes only and you cannot
+call `req.setEncoding` when using this module. The `status` property is set to
+`500` and the `type` property is set to `'stream.encoding.set'`.
+
+### stream is not readable
+
+This error will occur when the request is no longer readable when this middleware
+attempts to read it. This typically means something other than a middleware from
+this module read the request body already and the middleware was also configured to
+read the same request. The `status` property is set to `500` and the `type`
+property is set to `'stream.not.readable'`.
+
+### too many parameters
+
+This error will occur when the content of the request exceeds the configured
+`parameterLimit` for the `urlencoded` parser. The `status` property is set to
+`413` and the `type` property is set to `'parameters.too.many'`.
+
+### unsupported charset "BOGUS"
+
+This error will occur when the request had a charset parameter in the
+`Content-Type` header, but the `iconv-lite` module does not support it OR the
+parser does not support it. The charset is contained in the message as well
+as in the `charset` property. The `status` property is set to `415`, the
+`type` property is set to `'charset.unsupported'`, and the `charset` property
+is set to the charset that is unsupported.
+
+### unsupported content encoding "bogus"
+
+This error will occur when the request had a `Content-Encoding` header that
+contained an unsupported encoding. The encoding is contained in the message
+as well as in the `encoding` property. The `status` property is set to `415`,
+the `type` property is set to `'encoding.unsupported'`, and the `encoding`
+property is set to the encoding that is unsupported.
+
+### The input exceeded the depth
+
+This error occurs when using `bodyParser.urlencoded` with the `extended` property set to `true` and the input exceeds the configured `depth` option. The `status` property is set to `400`. It is recommended to review the `depth` option and evaluate if it requires a higher value. When the `depth` option is set to `32` (default value), the error will not be thrown.
+
+## Examples
+
+### Express/Connect top-level generic
+
+This example demonstrates adding a generic JSON and URL-encoded parser as a
+top-level middleware, which will parse the bodies of all incoming requests.
+This is the simplest setup.
+
+```js
+var express = require('express')
+var bodyParser = require('body-parser')
+
+var app = express()
+
+// parse application/x-www-form-urlencoded
+app.use(bodyParser.urlencoded({ extended: false }))
+
+// parse application/json
+app.use(bodyParser.json())
+
+app.use(function (req, res) {
+  res.setHeader('Content-Type', 'text/plain')
+  res.write('you posted:\n')
+  res.end(JSON.stringify(req.body, null, 2))
+})
+```
+
+### Express route-specific
+
+This example demonstrates adding body parsers specifically to the routes that
+need them. In general, this is the most recommended way to use body-parser with
+Express.
+
+```js
+var express = require('express')
+var bodyParser = require('body-parser')
+
+var app = express()
+
+// create application/json parser
+var jsonParser = bodyParser.json()
+
+// create application/x-www-form-urlencoded parser
+var urlencodedParser = bodyParser.urlencoded({ extended: false })
+
+// POST /login gets urlencoded bodies
+app.post('/login', urlencodedParser, function (req, res) {
+  res.send('welcome, ' + req.body.username)
+})
+
+// POST /api/users gets JSON bodies
+app.post('/api/users', jsonParser, function (req, res) {
+  // create user in req.body
+})
+```
+
+### Change accepted type for parsers
+
+All the parsers accept a `type` option which allows you to change the
+`Content-Type` that the middleware will parse.
+
+```js
+var express = require('express')
+var bodyParser = require('body-parser')
+
+var app = express()
+
+// parse various different custom JSON types as JSON
+app.use(bodyParser.json({ type: 'application/*+json' }))
+
+// parse some custom thing into a Buffer
+app.use(bodyParser.raw({ type: 'application/vnd.custom-type' }))
+
+// parse an HTML body into a string
+app.use(bodyParser.text({ type: 'text/html' }))
+```
+
+## License
+
+[MIT](LICENSE)
+
+[ci-image]: https://badgen.net/github/checks/expressjs/body-parser/master?label=ci
+[ci-url]: https://github.com/expressjs/body-parser/actions/workflows/ci.yml
+[coveralls-image]: https://badgen.net/coveralls/c/github/expressjs/body-parser/master
+[coveralls-url]: https://coveralls.io/r/expressjs/body-parser?branch=master
+[node-version-image]: https://badgen.net/npm/node/body-parser
+[node-version-url]: https://nodejs.org/en/download
+[npm-downloads-image]: https://badgen.net/npm/dm/body-parser
+[npm-url]: https://npmjs.org/package/body-parser
+[npm-version-image]: https://badgen.net/npm/v/body-parser
+[ossf-scorecard-badge]: https://api.scorecard.dev/projects/github.com/expressjs/body-parser/badge
+[ossf-scorecard-visualizer]: https://ossf.github.io/scorecard-visualizer/#/projects/github.com/expressjs/body-parser

scraper-service/node_modules/body-parser/index.js

@@ -0,0 +1,156 @@
+/*!
+ * body-parser
+ * Copyright(c) 2014-2015 Douglas Christopher Wilson
+ * MIT Licensed
+ */
+
+'use strict'
+
+/**
+ * Module dependencies.
+ * @private
+ */
+
+var deprecate = require('depd')('body-parser')
+
+/**
+ * Cache of loaded parsers.
+ * @private
+ */
+
+var parsers = Object.create(null)
+
+/**
+ * @typedef Parsers
+ * @type {function}
+ * @property {function} json
+ * @property {function} raw
+ * @property {function} text
+ * @property {function} urlencoded
+ */
+
+/**
+ * Module exports.
+ * @type {Parsers}
+ */
+
+exports = module.exports = deprecate.function(bodyParser,
+  'bodyParser: use individual json/urlencoded middlewares')
+
+/**
+ * JSON parser.
+ * @public
+ */
+
+Object.defineProperty(exports, 'json', {
+  configurable: true,
+  enumerable: true,
+  get: createParserGetter('json')
+})
+
+/**
+ * Raw parser.
+ * @public
+ */
+
+Object.defineProperty(exports, 'raw', {
+  configurable: true,
+  enumerable: true,
+  get: createParserGetter('raw')
+})
+
+/**
+ * Text parser.
+ * @public
+ */
+
+Object.defineProperty(exports, 'text', {
+  configurable: true,
+  enumerable: true,
+  get: createParserGetter('text')
+})
+
+/**
+ * URL-encoded parser.
+ * @public
+ */
+
+Object.defineProperty(exports, 'urlencoded', {
+  configurable: true,
+  enumerable: true,
+  get: createParserGetter('urlencoded')
+})
+
+/**
+ * Create a middleware to parse json and urlencoded bodies.
+ *
+ * @param {object} [options]
+ * @return {function}
+ * @deprecated
+ * @public
+ */
+
+function bodyParser (options) {
+  // use default type for parsers
+  var opts = Object.create(options || null, {
+    type: {
+      configurable: true,
+      enumerable: true,
+      value: undefined,
+      writable: true
+    }
+  })
+
+  var _urlencoded = exports.urlencoded(opts)
+  var _json = exports.json(opts)
+
+  return function bodyParser (req, res, next) {
+    _json(req, res, function (err) {
+      if (err) return next(err)
+      _urlencoded(req, res, next)
+    })
+  }
+}
+
+/**
+ * Create a getter for loading a parser.
+ * @private
+ */
+
+function createParserGetter (name) {
+  return function get () {
+    return loadParser(name)
+  }
+}
+
+/**
+ * Load a parser module.
+ * @private
+ */
+
+function loadParser (parserName) {
+  var parser = parsers[parserName]
+
+  if (parser !== undefined) {
+    return parser
+  }
+
+  // this uses a switch for static require analysis
+  switch (parserName) {
+    case 'json':
+      parser = require('./lib/types/json')
+      break
+    case 'raw':
+      parser = require('./lib/types/raw')
+      break
+    case 'text':
+      parser = require('./lib/types/text')
+      break
+    case 'urlencoded':
+      parser = require('./lib/types/urlencoded')
+      break
+  }
+
+  // store to prevent invoking require()
+  return (parsers[parserName] = parser)
+}

scraper-service/node_modules/body-parser/lib/read.js

@@ -0,0 +1,205 @@
+/*!
+ * body-parser
+ * Copyright(c) 2014-2015 Douglas Christopher Wilson
+ * MIT Licensed
+ */
+
+'use strict'
+
+/**
+ * Module dependencies.
+ * @private
+ */
+
+var createError = require('http-errors')
+var destroy = require('destroy')
+var getBody = require('raw-body')
+var iconv = require('iconv-lite')
+var onFinished = require('on-finished')
+var unpipe = require('unpipe')
+var zlib = require('zlib')
+
+/**
+ * Module exports.
+ */
+
+module.exports = read
+
+/**
+ * Read a request into a buffer and parse.
+ *
+ * @param {object} req
+ * @param {object} res
+ * @param {function} next
+ * @param {function} parse
+ * @param {function} debug
+ * @param {object} options
+ * @private
+ */
+
+function read (req, res, next, parse, debug, options) {
+  var length
+  var opts = options
+  var stream
+
+  // flag as parsed
+  req._body = true
+
+  // read options
+  var encoding = opts.encoding !== null
+    ? opts.encoding
+    : null
+  var verify = opts.verify
+
+  try {
+    // get the content stream
+    stream = contentstream(req, debug, opts.inflate)
+    length = stream.length
+    stream.length = undefined
+  } catch (err) {
+    return next(err)
+  }
+
+  // set raw-body options
+  opts.length = length
+  opts.encoding = verify
+    ? null
+    : encoding
+
+  // assert charset is supported
+  if (opts.encoding === null && encoding !== null && !iconv.encodingExists(encoding)) {
+    return next(createError(415, 'unsupported charset "' + encoding.toUpperCase() + '"', {
+      charset: encoding.toLowerCase(),
+      type: 'charset.unsupported'
+    }))
+  }
+
+  // read body
+  debug('read body')
+  getBody(stream, opts, function (error, body) {
+    if (error) {
+      var _error
+
+      if (error.type === 'encoding.unsupported') {
+        // echo back charset
+        _error = createError(415, 'unsupported charset "' + encoding.toUpperCase() + '"', {
+          charset: encoding.toLowerCase(),
+          type: 'charset.unsupported'
+        })
+      } else {
+        // set status code on error
+        _error = createError(400, error)
+      }
+
+      // unpipe from stream and destroy
+      if (stream !== req) {
+        unpipe(req)
+        destroy(stream, true)
+      }
+
+      // read off entire request
+      dump(req, function onfinished () {
+        next(createError(400, _error))
+      })
+      return
+    }
+
+    // verify
+    if (verify) {
+      try {
+        debug('verify body')
+        verify(req, res, body, encoding)
+      } catch (err) {
+        next(createError(403, err, {
+          body: body,
+          type: err.type || 'entity.verify.failed'
+        }))
+        return
+      }
+    }
+
+    // parse
+    var str = body
+    try {
+      debug('parse body')
+      str = typeof body !== 'string' && encoding !== null
+        ? iconv.decode(body, encoding)
+        : body
+      req.body = parse(str)
+    } catch (err) {
+      next(createError(400, err, {
+        body: str,
+        type: err.type || 'entity.parse.failed'
+      }))
+      return
+    }
+
+    next()
+  })
+}
+
+/**
+ * Get the content stream of the request.
+ *
+ * @param {object} req
+ * @param {function} debug
+ * @param {boolean} [inflate=true]
+ * @return {object}
+ * @api private
+ */
+
+function contentstream (req, debug, inflate) {
+  var encoding = (req.headers['content-encoding'] || 'identity').toLowerCase()
+  var length = req.headers['content-length']
+  var stream
+
+  debug('content-encoding "%s"', encoding)
+
+  if (inflate === false && encoding !== 'identity') {
+    throw createError(415, 'content encoding unsupported', {
+      encoding: encoding,
+      type: 'encoding.unsupported'
+    })
+  }
+
+  switch (encoding) {
+    case 'deflate':
+      stream = zlib.createInflate()
+      debug('inflate body')
+      req.pipe(stream)
+      break
+    case 'gzip':
+      stream = zlib.createGunzip()
+      debug('gunzip body')
+      req.pipe(stream)
+      break
+    case 'identity':
+      stream = req
+      stream.length = length
+      break
+    default:
+      throw createError(415, 'unsupported content encoding "' + encoding + '"', {
+        encoding: encoding,
+        type: 'encoding.unsupported'
+      })
+  }
+
+  return stream
+}
+
+/**
+ * Dump the contents of a request.
+ *
+ * @param {object} req
+ * @param {function} callback
+ * @api private
+ */
+
+function dump (req, callback) {
+  if (onFinished.isFinished(req)) {
+    callback(null)
+  } else {
+    onFinished(req, callback)
+    req.resume()
+  }
+}

scraper-service/node_modules/body-parser/lib/types/json.js

@@ -0,0 +1,243 @@
+/*!
+ * body-parser
+ * Copyright(c) 2014 Jonathan Ong
+ * Copyright(c) 2014-2015 Douglas Christopher Wilson
+ * MIT Licensed
+ */
+
+'use strict'
+
+/**
+ * Module dependencies.
+ * @private
+ */
+
+var bytes = require('bytes')
+var contentType = require('content-type')
+var createError = require('http-errors')
+var debug = require('debug')('body-parser:json')
+var read = require('../read')
+var typeis = require('type-is')
+
+/**
+ * Module exports.
+ */
+
+module.exports = json
+
+/**
+ * RegExp to match the first non-space in a string.
+ *
+ * Allowed whitespace is defined in RFC 7159:
+ *
+ *    ws = *(
+ *            %x20 /              ; Space
+ *            %x09 /              ; Horizontal tab
+ *            %x0A /              ; Line feed or New line
+ *            %x0D )              ; Carriage return
+ */
+
+var FIRST_CHAR_REGEXP = /^[\x20\x09\x0a\x0d]*([^\x20\x09\x0a\x0d])/ // eslint-disable-line no-control-regex
+
+var JSON_SYNTAX_CHAR = '#'
+var JSON_SYNTAX_REGEXP = /#+/g
+
+/**
+ * Create a middleware to parse JSON bodies.
+ *
+ * @param {object} [options]
+ * @return {function}
+ * @public
+ */
+
+function json (options) {
+  var opts = options || {}
+
+  var limit = typeof opts.limit !== 'number'
+    ? bytes.parse(opts.limit || '100kb')
+    : opts.limit
+  var inflate = opts.inflate !== false
+  var reviver = opts.reviver
+  var strict = opts.strict !== false
+  var type = opts.type || 'application/json'
+  var verify = opts.verify || false
+
+  if (verify !== false && typeof verify !== 'function') {
+    throw new TypeError('option verify must be function')
+  }
+
+  // create the appropriate type checking function
+  var shouldParse = typeof type !== 'function'
+    ? typeChecker(type)
+    : type
+
+  function parse (body) {
+    if (body.length === 0) {
+      // special-case empty json body, as it's a common client-side mistake
+      // TODO: maybe make this configurable or part of "strict" option
+      return {}
+    }
+
+    if (strict) {
+      var first = firstchar(body)
+
+      if (first !== '{' && first !== '[') {
+        debug('strict violation')
+        throw createStrictSyntaxError(body, first)
+      }
+    }
+
+    try {
+      debug('parse json')
+      return JSON.parse(body, reviver)
+    } catch (e) {
+      throw normalizeJsonSyntaxError(e, {
+        message: e.message,
+        stack: e.stack
+      })
+    }
+  }
+
+  return function jsonParser (req, res, next) {
+    if (req._body) {
+      debug('body already parsed')
+      next()
+      return
+    }
+
+    req.body = req.body || {}
+
+    // skip requests without bodies
+    if (!typeis.hasBody(req)) {
+      debug('skip empty body')
+      next()
+      return
+    }
+
+    debug('content-type %j', req.headers['content-type'])
+
+    // determine if request should be parsed
+    if (!shouldParse(req)) {
+      debug('skip parsing')
+      next()
+      return
+    }
+
+    // assert charset per RFC 7159 sec 8.1
+    var charset = getCharset(req) || 'utf-8'
+    if (charset.slice(0, 4) !== 'utf-') {
+      debug('invalid charset')
+      next(createError(415, 'unsupported charset "' + charset.toUpperCase() + '"', {
+        charset: charset,
+        type: 'charset.unsupported'
+      }))
+      return
+    }
+
+    // read
+    read(req, res, next, parse, debug, {
+      encoding: charset,
+      inflate: inflate,
+      limit: limit,
+      verify: verify
+    })
+  }
+}
+
+/**
+ * Create strict violation syntax error matching native error.
+ *
+ * @param {string} str
+ * @param {string} char
+ * @return {Error}
+ * @private
+ */
+
+function createStrictSyntaxError (str, char) {
+  var index = str.indexOf(char)
+  var partial = ''
+
+  if (index !== -1) {
+    partial = str.substring(0, index) + new Array(str.length - index + 1).join(JSON_SYNTAX_CHAR)
+  }
+
+  try {
+    JSON.parse(partial); /* istanbul ignore next */ throw new SyntaxError('strict violation')
+  } catch (e) {
+    return normalizeJsonSyntaxError(e, {
+      message: e.message.replace(JSON_SYNTAX_REGEXP, function (placeholder) {
+        return str.substring(index, index + placeholder.length)
+      }),
+      stack: e.stack
+    })
+  }
+}
+
+/**
+ * Get the first non-whitespace character in a string.
+ *
+ * @param {string} str
+ * @return {function}
+ * @private
+ */
+
+function firstchar (str) {
+  var match = FIRST_CHAR_REGEXP.exec(str)
+
+  return match
+    ? match[1]
+    : undefined
+}
+
+/**
+ * Get the charset of a request.
+ *
+ * @param {object} req
+ * @api private
+ */
+
+function getCharset (req) {
+  try {
+    return (contentType.parse(req).parameters.charset || '').toLowerCase()
+  } catch (e) {
+    return undefined
+  }
+}
+
+/**
+ * Normalize a SyntaxError for JSON.parse.
+ *
+ * @param {SyntaxError} error
+ * @param {object} obj
+ * @return {SyntaxError}
+ */
+
+function normalizeJsonSyntaxError (error, obj) {
+  var keys = Object.getOwnPropertyNames(error)
+
+  for (var i = 0; i < keys.length; i++) {
+    var key = keys[i]
+    if (key !== 'stack' && key !== 'message') {
+      delete error[key]
+    }
+  }
+
+  // replace stack before message for Node.js 0.10 and below
+  error.stack = obj.stack.replace(error.message, obj.message)
+  error.message = obj.message
+
+  return error
+}
+
+/**
+ * Get the simple type checker.
+ *
+ * @param {string} type
+ * @return {function}
+ */
+
+function typeChecker (type) {
+  return function checkType (req) {
+    return Boolean(typeis(req, type))
+  }
+}

scraper-service/node_modules/body-parser/lib/types/raw.js

@@ -0,0 +1,101 @@
+/*!
+ * body-parser
+ * Copyright(c) 2014-2015 Douglas Christopher Wilson
+ * MIT Licensed
+ */
+
+'use strict'
+
+/**
+ * Module dependencies.
+ */
+
+var bytes = require('bytes')
+var debug = require('debug')('body-parser:raw')
+var read = require('../read')
+var typeis = require('type-is')
+
+/**
+ * Module exports.
+ */
+
+module.exports = raw
+
+/**
+ * Create a middleware to parse raw bodies.
+ *
+ * @param {object} [options]
+ * @return {function}
+ * @api public
+ */
+
+function raw (options) {
+  var opts = options || {}
+
+  var inflate = opts.inflate !== false
+  var limit = typeof opts.limit !== 'number'
+    ? bytes.parse(opts.limit || '100kb')
+    : opts.limit
+  var type = opts.type || 'application/octet-stream'
+  var verify = opts.verify || false
+
+  if (verify !== false && typeof verify !== 'function') {
+    throw new TypeError('option verify must be function')
+  }
+
+  // create the appropriate type checking function
+  var shouldParse = typeof type !== 'function'
+    ? typeChecker(type)
+    : type
+
+  function parse (buf) {
+    return buf
+  }
+
+  return function rawParser (req, res, next) {
+    if (req._body) {
+      debug('body already parsed')
+      next()
+      return
+    }
+
+    req.body = req.body || {}
+
+    // skip requests without bodies
+    if (!typeis.hasBody(req)) {
+      debug('skip empty body')
+      next()
+      return
+    }
+
+    debug('content-type %j', req.headers['content-type'])
+
+    // determine if request should be parsed
+    if (!shouldParse(req)) {
+      debug('skip parsing')
+      next()
+      return
+    }
+
+    // read
+    read(req, res, next, parse, debug, {
+      encoding: null,
+      inflate: inflate,
+      limit: limit,
+      verify: verify
+    })
+  }
+}
+
+/**
+ * Get the simple type checker.
+ *
+ * @param {string} type
+ * @return {function}
+ */
+
+function typeChecker (type) {
+  return function checkType (req) {
+    return Boolean(typeis(req, type))
+  }
+}

scraper-service/node_modules/body-parser/lib/types/text.js

@@ -0,0 +1,121 @@
+/*!
+ * body-parser
+ * Copyright(c) 2014-2015 Douglas Christopher Wilson
+ * MIT Licensed
+ */
+
+'use strict'
+
+/**
+ * Module dependencies.
+ */
+
+var bytes = require('bytes')
+var contentType = require('content-type')
+var debug = require('debug')('body-parser:text')
+var read = require('../read')
+var typeis = require('type-is')
+
+/**
+ * Module exports.
+ */
+
+module.exports = text
+
+/**
+ * Create a middleware to parse text bodies.
+ *
+ * @param {object} [options]
+ * @return {function}
+ * @api public
+ */
+
+function text (options) {
+  var opts = options || {}
+
+  var defaultCharset = opts.defaultCharset || 'utf-8'
+  var inflate = opts.inflate !== false
+  var limit = typeof opts.limit !== 'number'
+    ? bytes.parse(opts.limit || '100kb')
+    : opts.limit
+  var type = opts.type || 'text/plain'
+  var verify = opts.verify || false
+
+  if (verify !== false && typeof verify !== 'function') {
+    throw new TypeError('option verify must be function')
+  }
+
+  // create the appropriate type checking function
+  var shouldParse = typeof type !== 'function'
+    ? typeChecker(type)
+    : type
+
+  function parse (buf) {
+    return buf
+  }
+
+  return function textParser (req, res, next) {
+    if (req._body) {
+      debug('body already parsed')
+      next()
+      return
+    }
+
+    req.body = req.body || {}
+
+    // skip requests without bodies
+    if (!typeis.hasBody(req)) {
+      debug('skip empty body')
+      next()
+      return
+    }
+
+    debug('content-type %j', req.headers['content-type'])
+
+    // determine if request should be parsed
+    if (!shouldParse(req)) {
+      debug('skip parsing')
+      next()
+      return
+    }
+
+    // get charset
+    var charset = getCharset(req) || defaultCharset
+
+    // read
+    read(req, res, next, parse, debug, {
+      encoding: charset,
+      inflate: inflate,
+      limit: limit,
+      verify: verify
+    })
+  }
+}
+
+/**
+ * Get the charset of a request.
+ *
+ * @param {object} req
+ * @api private
+ */
+
+function getCharset (req) {
+  try {
+    return (contentType.parse(req).parameters.charset || '').toLowerCase()
+  } catch (e) {
+    return undefined
+  }
+}
+
+/**
+ * Get the simple type checker.
+ *
+ * @param {string} type
+ * @return {function}
+ */
+
+function typeChecker (type) {
+  return function checkType (req) {
+    return Boolean(typeis(req, type))
+  }
+}

scraper-service/node_modules/body-parser/lib/types/urlencoded.js

@@ -0,0 +1,299 @@
+/*!
+ * body-parser
+ * Copyright(c) 2014 Jonathan Ong
+ * Copyright(c) 2014-2015 Douglas Christopher Wilson
+ * MIT Licensed
+ */
+
+'use strict'
+
+/**
+ * Module dependencies.
+ * @private
+ */
+
+var bytes = require('bytes')
+var contentType = require('content-type')
+var createError = require('http-errors')
+var debug = require('debug')('body-parser:urlencoded')
+var deprecate = require('depd')('body-parser')
+var read = require('../read')
+var typeis = require('type-is')
+
+/**
+ * Module exports.
+ */
+
+module.exports = urlencoded
+
+/**
+ * Cache of parser modules.
+ */
+
+var parsers = Object.create(null)
+
+/**
+ * Create a middleware to parse urlencoded bodies.
+ *
+ * @param {object} [options]
+ * @return {function}
+ * @public
+ */
+
+function urlencoded (options) {
+  var opts = options || {}
+
+  // notice because option default will flip in next major
+  if (opts.extended === undefined) {
+    deprecate('undefined extended: provide extended option')
+  }
+
+  var extended = opts.extended !== false
+  var inflate = opts.inflate !== false
+  var limit = typeof opts.limit !== 'number'
+    ? bytes.parse(opts.limit || '100kb')
+    : opts.limit
+  var type = opts.type || 'application/x-www-form-urlencoded'
+  var verify = opts.verify || false
+
+  if (verify !== false && typeof verify !== 'function') {
+    throw new TypeError('option verify must be function')
+  }
+
+  // create the appropriate query parser
+  var queryparse = extended
+    ? extendedparser(opts)
+    : simpleparser(opts)
+
+  // create the appropriate type checking function
+  var shouldParse = typeof type !== 'function'
+    ? typeChecker(type)
+    : type
+
+  function parse (body) {
+    return body.length
+      ? queryparse(body)
+      : {}
+  }
+
+  return function urlencodedParser (req, res, next) {
+    if (req._body) {
+      debug('body already parsed')
+      next()
+      return
+    }
+
+    req.body = req.body || {}
+
+    // skip requests without bodies
+    if (!typeis.hasBody(req)) {
+      debug('skip empty body')
+      next()
+      return
+    }
+
+    debug('content-type %j', req.headers['content-type'])
+
+    // determine if request should be parsed
+    if (!shouldParse(req)) {
+      debug('skip parsing')
+      next()
+      return
+    }
+
+    // assert charset
+    var charset = getCharset(req) || 'utf-8'
+    if (charset !== 'utf-8') {
+      debug('invalid charset')
+      next(createError(415, 'unsupported charset "' + charset.toUpperCase() + '"', {
+        charset: charset,
+        type: 'charset.unsupported'
+      }))
+      return
+    }
+
+    // read
+    read(req, res, next, parse, debug, {
+      debug: debug,
+      encoding: charset,
+      inflate: inflate,
+      limit: limit,
+      verify: verify
+    })
+  }
+}
+
+/**
+ * Get the extended query parser.
+ *
+ * @param {object} options
+ */
+
+function extendedparser (options) {
+  var parameterLimit = options.parameterLimit !== undefined
+    ? options.parameterLimit
+    : 1000
+  var depth = options.depth !== undefined ? options.depth : 32
+  var parse = parser('qs')
+
+  if (isNaN(parameterLimit) || parameterLimit < 1) {
+    throw new TypeError('option parameterLimit must be a positive number')
+  }
+
+  if (isNaN(depth) || depth < 0) {
+    throw new TypeError('option depth must be a zero or a positive number')
+  }
+
+  if (isFinite(parameterLimit)) {
+    parameterLimit = parameterLimit | 0
+  }
+
+  return function queryparse (body) {
+    var paramCount = parameterCount(body, parameterLimit)
+
+    if (paramCount === undefined) {
+      debug('too many parameters')
+      throw createError(413, 'too many parameters', {
+        type: 'parameters.too.many'
+      })
+    }
+
+    var arrayLimit = Math.max(100, paramCount)
+
+    debug('parse extended urlencoding')
+    try {
+      return parse(body, {
+        allowPrototypes: true,
+        arrayLimit: arrayLimit,
+        depth: depth,
+        strictDepth: true,
+        parameterLimit: parameterLimit
+      })
+    } catch (err) {
+      if (err instanceof RangeError) {
+        throw createError(400, 'The input exceeded the depth', {
+          type: 'querystring.parse.rangeError'
+        })
+      } else {
+        throw err
+      }
+    }
+  }
+}
+
+/**
+ * Get the charset of a request.
+ *
+ * @param {object} req
+ * @api private
+ */
+
+function getCharset (req) {
+  try {
+    return (contentType.parse(req).parameters.charset || '').toLowerCase()
+  } catch (e) {
+    return undefined
+  }
+}
+
+/**
+ * Count the number of parameters, stopping once limit reached
+ *
+ * @param {string} body
+ * @param {number} limit
+ * @api private
+ */
+
+function parameterCount (body, limit) {
+  var count = 0
+  var index = -1
+
+  do {
+    count++
+    if (count > limit) {
+      return undefined
+    }
+    index = body.indexOf('&', index + 1)
+  } while (index !== -1)
+
+  return count
+}
+
+/**
+ * Get parser for module name dynamically.
+ *
+ * @param {string} name
+ * @return {function}
+ * @api private
+ */
+
+function parser (name) {
+  var mod = parsers[name]
+
+  if (mod !== undefined) {
+    return mod.parse
+  }
+
+  // this uses a switch for static require analysis
+  switch (name) {
+    case 'qs':
+      mod = require('qs')
+      break
+    case 'querystring':
+      mod = require('querystring')
+      break
+  }
+
+  // store to prevent invoking require()
+  parsers[name] = mod
+
+  return mod.parse
+}
+
+/**
+ * Get the simple query parser.
+ *
+ * @param {object} options
+ */
+
+function simpleparser (options) {
+  var parameterLimit = options.parameterLimit !== undefined
+    ? options.parameterLimit
+    : 1000
+  var parse = parser('querystring')
+
+  if (isNaN(parameterLimit) || parameterLimit < 1) {
+    throw new TypeError('option parameterLimit must be a positive number')
+  }
+
+  if (isFinite(parameterLimit)) {
+    parameterLimit = parameterLimit | 0
+  }
+
+  return function queryparse (body) {
+    var paramCount = parameterCount(body, parameterLimit)
+
+    if (paramCount === undefined) {
+      debug('too many parameters')
+      throw createError(413, 'too many parameters', {
+        type: 'parameters.too.many'
+      })
+    }
+
+    debug('parse urlencoding')
+    return parse(body, undefined, undefined, { maxKeys: parameterLimit })
+  }
+}
+
+/**
+ * Get the simple type checker.
+ *
+ * @param {string} type
+ * @return {function}
+ */
+
+function typeChecker (type) {
+  return function checkType (req) {
+    return Boolean(typeis(req, type))
+  }
+}

scraper-service/node_modules/body-parser/package.json

@@ -0,0 +1,55 @@
+{
+  "name": "body-parser",
+  "description": "Node.js body parsing middleware",
+  "version": "1.20.5",
+  "contributors": [
+    "Douglas Christopher Wilson <doug@somethingdoug.com>",
+    "Jonathan Ong <me@jongleberry.com> (http://jongleberry.com)"
+  ],
+  "license": "MIT",
+  "repository": "expressjs/body-parser",
+  "dependencies": {
+    "bytes": "~3.1.2",
+    "content-type": "~1.0.5",
+    "debug": "2.6.9",
+    "depd": "2.0.0",
+    "destroy": "~1.2.0",
+    "http-errors": "~2.0.1",
+    "iconv-lite": "~0.4.24",
+    "on-finished": "~2.4.1",
+    "qs": "~6.15.1",
+    "raw-body": "~2.5.3",
+    "type-is": "~1.6.18",
+    "unpipe": "~1.0.0"
+  },
+  "devDependencies": {
+    "eslint": "8.34.0",
+    "eslint-config-standard": "14.1.1",
+    "eslint-plugin-import": "2.27.5",
+    "eslint-plugin-markdown": "3.0.0",
+    "eslint-plugin-node": "11.1.0",
+    "eslint-plugin-promise": "6.1.1",
+    "eslint-plugin-standard": "4.1.0",
+    "methods": "1.1.2",
+    "mocha": "10.2.0",
+    "nyc": "15.1.0",
+    "safe-buffer": "5.2.1",
+    "supertest": "6.3.3"
+  },
+  "files": [
+    "lib/",
+    "LICENSE",
+    "HISTORY.md",
+    "index.js"
+  ],
+  "engines": {
+    "node": ">= 0.8",
+    "npm": "1.2.8000 || >= 1.4.16"
+  },
+  "scripts": {
+    "lint": "eslint .",
+    "test": "mocha --require test/support/env --reporter spec --check-leaks --bail test/",
+    "test-ci": "nyc --reporter=lcov --reporter=text npm test",
+    "test-cov": "nyc --reporter=html --reporter=text npm test"
+  }
+}

scraper-service/node_modules/bytes/History.md

@@ -0,0 +1,97 @@
+3.1.2 / 2022-01-27
+==================
+
+  * Fix return value for un-parsable strings
+
+3.1.1 / 2021-11-15
+==================
+
+  * Fix "thousandsSeparator" incorrecting formatting fractional part
+
+3.1.0 / 2019-01-22
+==================
+
+  * Add petabyte (`pb`) support
+
+3.0.0 / 2017-08-31
+==================
+
+  * Change "kB" to "KB" in format output
+  * Remove support for Node.js 0.6
+  * Remove support for ComponentJS
+
+2.5.0 / 2017-03-24
+==================
+
+  * Add option "unit"
+
+2.4.0 / 2016-06-01
+==================
+
+  * Add option "unitSeparator"
+
+2.3.0 / 2016-02-15
+==================
+
+  * Drop partial bytes on all parsed units
+  * Fix non-finite numbers to `.format` to return `null`
+  * Fix parsing byte string that looks like hex
+  * perf: hoist regular expressions
+
+2.2.0 / 2015-11-13
+==================
+
+  * add option "decimalPlaces"
+  * add option "fixedDecimals"
+
+2.1.0 / 2015-05-21
+==================
+
+  * add `.format` export
+  * add `.parse` export
+
+2.0.2 / 2015-05-20
+==================
+
+  * remove map recreation
+  * remove unnecessary object construction
+
+2.0.1 / 2015-05-07
+==================
+
+  * fix browserify require
+  * remove node.extend dependency
+
+2.0.0 / 2015-04-12
+==================
+
+  * add option "case"
+  * add option "thousandsSeparator"
+  * return "null" on invalid parse input
+  * support proper round-trip: bytes(bytes(num)) === num
+  * units no longer case sensitive when parsing
+
+1.0.0 / 2014-05-05
+==================
+
+ * add negative support. fixes #6
+
+0.3.0 / 2014-03-19
+==================
+
+ * added terabyte support
+
+0.2.1 / 2013-04-01
+==================
+
+  * add .component
+
+0.2.0 / 2012-10-28
+==================
+
+  * bytes(200).should.eql('200b')
+
+0.1.0 / 2012-07-04
+==================
+
+  * add bytes to string conversion [yields]

scraper-service/node_modules/bytes/LICENSE

@@ -0,0 +1,23 @@
+(The MIT License)
+
+Copyright (c) 2012-2014 TJ Holowaychuk <tj@vision-media.ca>
+Copyright (c) 2015 Jed Watson <jed.watson@me.com>
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+'Software'), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

scraper-service/node_modules/bytes/Readme.md

@@ -0,0 +1,152 @@
+# Bytes utility
+
+[![NPM Version][npm-image]][npm-url]
+[![NPM Downloads][downloads-image]][downloads-url]
+[![Build Status][ci-image]][ci-url]
+[![Test Coverage][coveralls-image]][coveralls-url]
+
+Utility to parse a string bytes (ex: `1TB`) to bytes (`1099511627776`) and vice-versa.
+
+## Installation
+
+This is a [Node.js](https://nodejs.org/en/) module available through the
+[npm registry](https://www.npmjs.com/). Installation is done using the
+[`npm install` command](https://docs.npmjs.com/getting-started/installing-npm-packages-locally):
+
+```bash
+$ npm install bytes
+```
+
+## Usage
+
+```js
+var bytes = require('bytes');
+```
+
+#### bytes(number｜string value, [options]): number｜string｜null
+
+Default export function. Delegates to either `bytes.format` or `bytes.parse` based on the type of `value`.
+
+**Arguments**
+
+| Name    | Type     | Description        |
+|---------|----------|--------------------|
+| value   | `number`｜`string` | Number value to format or string value to parse |
+| options | `Object` | Conversion options for `format` |
+
+**Returns**
+
+| Name    | Type             | Description                                     |
+|---------|------------------|-------------------------------------------------|
+| results | `string`｜`number`｜`null` | Return null upon error. Numeric value in bytes, or string value otherwise. |
+
+**Example**
+
+```js
+bytes(1024);
+// output: '1KB'
+
+bytes('1KB');
+// output: 1024
+```
+
+#### bytes.format(number value, [options]): string｜null
+
+Format the given value in bytes into a string. If the value is negative, it is kept as such. If it is a float, it is
+ rounded.
+
+**Arguments**
+
+| Name    | Type     | Description        |
+|---------|----------|--------------------|
+| value   | `number` | Value in bytes     |
+| options | `Object` | Conversion options |
+
+**Options**
+
+| Property          | Type   | Description                                                                             |
+|-------------------|--------|-----------------------------------------------------------------------------------------|
+| decimalPlaces | `number`｜`null` | Maximum number of decimal places to include in output. Default value to `2`. |
+| fixedDecimals | `boolean`｜`null` | Whether to always display the maximum number of decimal places. Default value to `false` |
+| thousandsSeparator | `string`｜`null` | Example of values: `' '`, `','` and `'.'`... Default value to `''`. |
+| unit | `string`｜`null` | The unit in which the result will be returned (B/KB/MB/GB/TB). Default value to `''` (which means auto detect). |
+| unitSeparator | `string`｜`null` | Separator to use between number and unit. Default value to `''`. |
+
+**Returns**
+
+| Name    | Type             | Description                                     |
+|---------|------------------|-------------------------------------------------|
+| results | `string`｜`null` | Return null upon error. String value otherwise. |
+
+**Example**
+
+```js
+bytes.format(1024);
+// output: '1KB'
+
+bytes.format(1000);
+// output: '1000B'
+
+bytes.format(1000, {thousandsSeparator: ' '});
+// output: '1 000B'
+
+bytes.format(1024 * 1.7, {decimalPlaces: 0});
+// output: '2KB'
+
+bytes.format(1024, {unitSeparator: ' '});
+// output: '1 KB'
+```
+
+#### bytes.parse(string｜number value): number｜null
+
+Parse the string value into an integer in bytes. If no unit is given, or `value`
+is a number, it is assumed the value is in bytes.
+
+Supported units and abbreviations are as follows and are case-insensitive:
+
+  * `b` for bytes
+  * `kb` for kilobytes
+  * `mb` for megabytes
+  * `gb` for gigabytes
+  * `tb` for terabytes
+  * `pb` for petabytes
+
+The units are in powers of two, not ten. This means 1kb = 1024b according to this parser.
+
+**Arguments**
+
+| Name          | Type   | Description        |
+|---------------|--------|--------------------|
+| value   | `string`｜`number` | String to parse, or number in bytes.   |
+
+**Returns**
+
+| Name    | Type        | Description             |
+|---------|-------------|-------------------------|
+| results | `number`｜`null` | Return null upon error. Value in bytes otherwise. |
+
+**Example**
+
+```js
+bytes.parse('1KB');
+// output: 1024
+
+bytes.parse('1024');
+// output: 1024
+
+bytes.parse(1024);
+// output: 1024
+```
+
+## License
+
+[MIT](LICENSE)
+
+[ci-image]: https://badgen.net/github/checks/visionmedia/bytes.js/master?label=ci
+[ci-url]: https://github.com/visionmedia/bytes.js/actions?query=workflow%3Aci
+[coveralls-image]: https://badgen.net/coveralls/c/github/visionmedia/bytes.js/master
+[coveralls-url]: https://coveralls.io/r/visionmedia/bytes.js?branch=master
+[downloads-image]: https://badgen.net/npm/dm/bytes
+[downloads-url]: https://npmjs.org/package/bytes
+[npm-image]: https://badgen.net/npm/v/bytes
+[npm-url]: https://npmjs.org/package/bytes

scraper-service/node_modules/bytes/index.js

@@ -0,0 +1,170 @@
+/*!
+ * bytes
+ * Copyright(c) 2012-2014 TJ Holowaychuk
+ * Copyright(c) 2015 Jed Watson
+ * MIT Licensed
+ */
+
+'use strict';
+
+/**
+ * Module exports.
+ * @public
+ */
+
+module.exports = bytes;
+module.exports.format = format;
+module.exports.parse = parse;
+
+/**
+ * Module variables.
+ * @private
+ */
+
+var formatThousandsRegExp = /\B(?=(\d{3})+(?!\d))/g;
+
+var formatDecimalsRegExp = /(?:\.0*|(\.[^0]+)0+)$/;
+
+var map = {
+  b:  1,
+  kb: 1 << 10,
+  mb: 1 << 20,
+  gb: 1 << 30,
+  tb: Math.pow(1024, 4),
+  pb: Math.pow(1024, 5),
+};
+
+var parseRegExp = /^((-|\+)?(\d+(?:\.\d+)?)) *(kb|mb|gb|tb|pb)$/i;
+
+/**
+ * Convert the given value in bytes into a string or parse to string to an integer in bytes.
+ *
+ * @param {string|number} value
+ * @param {{
+ *  case: [string],
+ *  decimalPlaces: [number]
+ *  fixedDecimals: [boolean]
+ *  thousandsSeparator: [string]
+ *  unitSeparator: [string]
+ *  }} [options] bytes options.
+ *
+ * @returns {string|number|null}
+ */
+
+function bytes(value, options) {
+  if (typeof value === 'string') {
+    return parse(value);
+  }
+
+  if (typeof value === 'number') {
+    return format(value, options);
+  }
+
+  return null;
+}
+
+/**
+ * Format the given value in bytes into a string.
+ *
+ * If the value is negative, it is kept as such. If it is a float,
+ * it is rounded.
+ *
+ * @param {number} value
+ * @param {object} [options]
+ * @param {number} [options.decimalPlaces=2]
+ * @param {number} [options.fixedDecimals=false]
+ * @param {string} [options.thousandsSeparator=]
+ * @param {string} [options.unit=]
+ * @param {string} [options.unitSeparator=]
+ *
+ * @returns {string|null}
+ * @public
+ */
+
+function format(value, options) {
+  if (!Number.isFinite(value)) {
+    return null;
+  }
+
+  var mag = Math.abs(value);
+  var thousandsSeparator = (options && options.thousandsSeparator) || '';
+  var unitSeparator = (options && options.unitSeparator) || '';
+  var decimalPlaces = (options && options.decimalPlaces !== undefined) ? options.decimalPlaces : 2;
+  var fixedDecimals = Boolean(options && options.fixedDecimals);
+  var unit = (options && options.unit) || '';
+
+  if (!unit || !map[unit.toLowerCase()]) {
+    if (mag >= map.pb) {
+      unit = 'PB';
+    } else if (mag >= map.tb) {
+      unit = 'TB';
+    } else if (mag >= map.gb) {
+      unit = 'GB';
+    } else if (mag >= map.mb) {
+      unit = 'MB';
+    } else if (mag >= map.kb) {
+      unit = 'KB';
+    } else {
+      unit = 'B';
+    }
+  }
+
+  var val = value / map[unit.toLowerCase()];
+  var str = val.toFixed(decimalPlaces);
+
+  if (!fixedDecimals) {
+    str = str.replace(formatDecimalsRegExp, '$1');
+  }
+
+  if (thousandsSeparator) {
+    str = str.split('.').map(function (s, i) {
+      return i === 0
+        ? s.replace(formatThousandsRegExp, thousandsSeparator)
+        : s
+    }).join('.');
+  }
+
+  return str + unitSeparator + unit;
+}
+
+/**
+ * Parse the string value into an integer in bytes.
+ *
+ * If no unit is given, it is assumed the value is in bytes.
+ *
+ * @param {number|string} val
+ *
+ * @returns {number|null}
+ * @public
+ */
+
+function parse(val) {
+  if (typeof val === 'number' && !isNaN(val)) {
+    return val;
+  }
+
+  if (typeof val !== 'string') {
+    return null;
+  }
+
+  // Test if the string passed is valid
+  var results = parseRegExp.exec(val);
+  var floatValue;
+  var unit = 'b';
+
+  if (!results) {
+    // Nothing could be extracted from the given string
+    floatValue = parseInt(val, 10);
+    unit = 'b'
+  } else {
+    // Retrieve the value and the unit
+    floatValue = parseFloat(results[1]);
+    unit = results[4].toLowerCase();
+  }
+
+  if (isNaN(floatValue)) {
+    return null;
+  }
+
+  return Math.floor(map[unit] * floatValue);
+}

scraper-service/node_modules/bytes/package.json

@@ -0,0 +1,42 @@
+{
+  "name": "bytes",
+  "description": "Utility to parse a string bytes to bytes and vice-versa",
+  "version": "3.1.2",
+  "author": "TJ Holowaychuk <tj@vision-media.ca> (http://tjholowaychuk.com)",
+  "contributors": [
+    "Jed Watson <jed.watson@me.com>",
+    "Théo FIDRY <theo.fidry@gmail.com>"
+  ],
+  "license": "MIT",
+  "keywords": [
+    "byte",
+    "bytes",
+    "utility",
+    "parse",
+    "parser",
+    "convert",
+    "converter"
+  ],
+  "repository": "visionmedia/bytes.js",
+  "devDependencies": {
+    "eslint": "7.32.0",
+    "eslint-plugin-markdown": "2.2.1",
+    "mocha": "9.2.0",
+    "nyc": "15.1.0"
+  },
+  "files": [
+    "History.md",
+    "LICENSE",
+    "Readme.md",
+    "index.js"
+  ],
+  "engines": {
+    "node": ">= 0.8"
+  },
+  "scripts": {
+    "lint": "eslint .",
+    "test": "mocha --check-leaks --reporter spec",
+    "test-ci": "nyc --reporter=lcov --reporter=text npm test",
+    "test-cov": "nyc --reporter=html --reporter=text npm test"
+  }
+}

scraper-service/node_modules/call-bind-apply-helpers/.eslintrc

@@ -0,0 +1,17 @@
+{
+	"root": true,
+
+	"extends": "@ljharb",
+
+	"rules": {
+		"func-name-matching": 0,
+		"id-length": 0,
+		"new-cap": [2, {
+			"capIsNewExceptions": [
+				"GetIntrinsic",
+			],
+		}],
+		"no-extra-parens": 0,
+		"no-magic-numbers": 0,
+	},
+}

scraper-service/node_modules/call-bind-apply-helpers/.github/FUNDING.yml

@@ -0,0 +1,12 @@
+# These are supported funding model platforms
+
+github: [ljharb]
+patreon: # Replace with a single Patreon username
+open_collective: # Replace with a single Open Collective username
+ko_fi: # Replace with a single Ko-fi username
+tidelift: npm/call-bind-apply-helpers
+community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
+liberapay: # Replace with a single Liberapay username
+issuehunt: # Replace with a single IssueHunt username
+otechie: # Replace with a single Otechie username
+custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2']

scraper-service/node_modules/call-bind-apply-helpers/.nycrc

@@ -0,0 +1,9 @@
+{
+	"all": true,
+	"check-coverage": false,
+	"reporter": ["text-summary", "text", "html", "json"],
+	"exclude": [
+		"coverage",
+		"test"
+	]
+}

scraper-service/node_modules/call-bind-apply-helpers/CHANGELOG.md

@@ -0,0 +1,30 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [v1.0.2](https://github.com/ljharb/call-bind-apply-helpers/compare/v1.0.1...v1.0.2) - 2025-02-12
+
+### Commits
+
+- [types] improve inferred types [`e6f9586`](https://github.com/ljharb/call-bind-apply-helpers/commit/e6f95860a3c72879cb861a858cdfb8138fbedec1)
+- [Dev Deps] update `@arethetypeswrong/cli`, `@ljharb/tsconfig`, `@types/tape`, `es-value-fixtures`, `for-each`, `has-strict-mode`, `object-inspect` [`e43d540`](https://github.com/ljharb/call-bind-apply-helpers/commit/e43d5409f97543bfbb11f345d47d8ce4e066d8c1)
+
+## [v1.0.1](https://github.com/ljharb/call-bind-apply-helpers/compare/v1.0.0...v1.0.1) - 2024-12-08
+
+### Commits
+
+- [types] `reflectApply`: fix types [`4efc396`](https://github.com/ljharb/call-bind-apply-helpers/commit/4efc3965351a4f02cc55e836fa391d3d11ef2ef8)
+- [Fix] `reflectApply`: oops, Reflect is not a function [`83cc739`](https://github.com/ljharb/call-bind-apply-helpers/commit/83cc7395de6b79b7730bdf092f1436f0b1263c75)
+- [Dev Deps] update `@arethetypeswrong/cli` [`80bd5d3`](https://github.com/ljharb/call-bind-apply-helpers/commit/80bd5d3ae58b4f6b6995ce439dd5a1bcb178a940)
+
+## v1.0.0 - 2024-12-05
+
+### Commits
+
+- Initial implementation, tests, readme [`7879629`](https://github.com/ljharb/call-bind-apply-helpers/commit/78796290f9b7430c9934d6f33d94ae9bc89fce04)
+- Initial commit [`3f1dc16`](https://github.com/ljharb/call-bind-apply-helpers/commit/3f1dc164afc43285631b114a5f9dd9137b2b952f)
+- npm init [`081df04`](https://github.com/ljharb/call-bind-apply-helpers/commit/081df048c312fcee400922026f6e97281200a603)
+- Only apps should have lockfiles [`5b9ca0f`](https://github.com/ljharb/call-bind-apply-helpers/commit/5b9ca0fe8101ebfaf309c549caac4e0a017ed930)

scraper-service/node_modules/call-bind-apply-helpers/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Jordan Harband
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

scraper-service/node_modules/call-bind-apply-helpers/README.md

@@ -0,0 +1,62 @@
+# call-bind-apply-helpers <sup>[![Version Badge][npm-version-svg]][package-url]</sup>
+
+[![github actions][actions-image]][actions-url]
+[![coverage][codecov-image]][codecov-url]
+[![dependency status][deps-svg]][deps-url]
+[![dev dependency status][dev-deps-svg]][dev-deps-url]
+[![License][license-image]][license-url]
+[![Downloads][downloads-image]][downloads-url]
+
+[![npm badge][npm-badge-png]][package-url]
+
+Helper functions around Function call/apply/bind, for use in `call-bind`.
+
+The only packages that should likely ever use this package directly are `call-bind` and `get-intrinsic`.
+Please use `call-bind` unless you have a very good reason not to.
+
+## Getting started
+
+```sh
+npm install --save call-bind-apply-helpers
+```
+
+## Usage/Examples
+
+```js
+const assert = require('assert');
+const callBindBasic = require('call-bind-apply-helpers');
+
+function f(a, b) {
+	assert.equal(this, 1);
+	assert.equal(a, 2);
+	assert.equal(b, 3);
+	assert.equal(arguments.length, 2);
+}
+
+const fBound = callBindBasic([f, 1]);
+
+delete Function.prototype.call;
+delete Function.prototype.bind;
+
+fBound(2, 3);
+```
+
+## Tests
+
+Clone the repo, `npm install`, and run `npm test`
+
+[package-url]: https://npmjs.org/package/call-bind-apply-helpers
+[npm-version-svg]: https://versionbadg.es/ljharb/call-bind-apply-helpers.svg
+[deps-svg]: https://david-dm.org/ljharb/call-bind-apply-helpers.svg
+[deps-url]: https://david-dm.org/ljharb/call-bind-apply-helpers
+[dev-deps-svg]: https://david-dm.org/ljharb/call-bind-apply-helpers/dev-status.svg
+[dev-deps-url]: https://david-dm.org/ljharb/call-bind-apply-helpers#info=devDependencies
+[npm-badge-png]: https://nodei.co/npm/call-bind-apply-helpers.png?downloads=true&stars=true
+[license-image]: https://img.shields.io/npm/l/call-bind-apply-helpers.svg
+[license-url]: LICENSE
+[downloads-image]: https://img.shields.io/npm/dm/call-bind-apply-helpers.svg
+[downloads-url]: https://npm-stat.com/charts.html?package=call-bind-apply-helpers
+[codecov-image]: https://codecov.io/gh/ljharb/call-bind-apply-helpers/branch/main/graphs/badge.svg
+[codecov-url]: https://app.codecov.io/gh/ljharb/call-bind-apply-helpers/
+[actions-image]: https://img.shields.io/endpoint?url=https://github-actions-badge-u3jn4tfpocch.runkit.sh/ljharb/call-bind-apply-helpers
+[actions-url]: https://github.com/ljharb/call-bind-apply-helpers/actions

scraper-service/node_modules/call-bind-apply-helpers/actualApply.d.ts

@@ -0,0 +1 @@
+export = Reflect.apply;

scraper-service/node_modules/call-bind-apply-helpers/actualApply.js

@@ -0,0 +1,10 @@
+'use strict';
+
+var bind = require('function-bind');
+
+var $apply = require('./functionApply');
+var $call = require('./functionCall');
+var $reflectApply = require('./reflectApply');
+
+/** @type {import('./actualApply')} */
+module.exports = $reflectApply || bind.call($call, $apply);

scraper-service/node_modules/call-bind-apply-helpers/applyBind.d.ts

@@ -0,0 +1,19 @@
+import actualApply from './actualApply';
+
+type TupleSplitHead<T extends any[], N extends number> = T['length'] extends N
+  ? T
+  : T extends [...infer R, any]
+  ? TupleSplitHead<R, N>
+  : never
+
+type TupleSplitTail<T, N extends number, O extends any[] = []> = O['length'] extends N
+  ? T
+  : T extends [infer F, ...infer R]
+  ? TupleSplitTail<[...R], N, [...O, F]>
+  : never
+
+type TupleSplit<T extends any[], N extends number> = [TupleSplitHead<T, N>, TupleSplitTail<T, N>]
+
+declare function applyBind(...args: TupleSplit<Parameters<typeof actualApply>, 2>[1]): ReturnType<typeof actualApply>;
+
+export = applyBind;

scraper-service/node_modules/call-bind-apply-helpers/applyBind.js

@@ -0,0 +1,10 @@
+'use strict';
+
+var bind = require('function-bind');
+var $apply = require('./functionApply');
+var actualApply = require('./actualApply');
+
+/** @type {import('./applyBind')} */
+module.exports = function applyBind() {
+	return actualApply(bind, $apply, arguments);
+};

scraper-service/node_modules/call-bind-apply-helpers/functionApply.d.ts

@@ -0,0 +1 @@
+export = Function.prototype.apply;

scraper-service/node_modules/call-bind-apply-helpers/functionApply.js

@@ -0,0 +1,4 @@
+'use strict';
+
+/** @type {import('./functionApply')} */
+module.exports = Function.prototype.apply;

scraper-service/node_modules/call-bind-apply-helpers/functionCall.d.ts

@@ -0,0 +1 @@
+export = Function.prototype.call;

scraper-service/node_modules/call-bind-apply-helpers/functionCall.js

@@ -0,0 +1,4 @@
+'use strict';
+
+/** @type {import('./functionCall')} */
+module.exports = Function.prototype.call;

scraper-service/node_modules/call-bind-apply-helpers/index.d.ts

@@ -0,0 +1,64 @@
+type RemoveFromTuple<
+  Tuple extends readonly unknown[],
+  RemoveCount extends number,
+  Index extends 1[] = []
+> = Index["length"] extends RemoveCount
+  ? Tuple
+  : Tuple extends [infer First, ...infer Rest]
+  ? RemoveFromTuple<Rest, RemoveCount, [...Index, 1]>
+  : Tuple;
+
+type ConcatTuples<
+  Prefix extends readonly unknown[],
+  Suffix extends readonly unknown[]
+> = [...Prefix, ...Suffix];
+
+type ExtractFunctionParams<T> = T extends (this: infer TThis, ...args: infer P extends readonly unknown[]) => infer R
+  ? { thisArg: TThis; params: P; returnType: R }
+  : never;
+
+type BindFunction<
+  T extends (this: any, ...args: any[]) => any,
+  TThis,
+  TBoundArgs extends readonly unknown[],
+  ReceiverBound extends boolean
+> = ExtractFunctionParams<T> extends {
+  thisArg: infer OrigThis;
+  params: infer P extends readonly unknown[];
+  returnType: infer R;
+}
+  ? ReceiverBound extends true
+    ? (...args: RemoveFromTuple<P, Extract<TBoundArgs["length"], number>>) => R extends [OrigThis, ...infer Rest]
+      ? [TThis, ...Rest] // Replace `this` with `thisArg`
+      : R
+    : <U, RemainingArgs extends RemoveFromTuple<P, Extract<TBoundArgs["length"], number>>>(
+        thisArg: U,
+        ...args: RemainingArgs
+      ) => R extends [OrigThis, ...infer Rest]
+      ? [U, ...ConcatTuples<TBoundArgs, Rest>] // Preserve bound args in return type
+      : R
+  : never;
+
+declare function callBind<
+  const T extends (this: any, ...args: any[]) => any,
+  Extracted extends ExtractFunctionParams<T>,
+  const TBoundArgs extends Partial<Extracted["params"]> & readonly unknown[],
+  const TThis extends Extracted["thisArg"]
+>(
+  args: [fn: T, thisArg: TThis, ...boundArgs: TBoundArgs]
+): BindFunction<T, TThis, TBoundArgs, true>;
+
+declare function callBind<
+  const T extends (this: any, ...args: any[]) => any,
+  Extracted extends ExtractFunctionParams<T>,
+  const TBoundArgs extends Partial<Extracted["params"]> & readonly unknown[]
+>(
+  args: [fn: T, ...boundArgs: TBoundArgs]
+): BindFunction<T, Extracted["thisArg"], TBoundArgs, false>;
+
+declare function callBind<const TArgs extends readonly unknown[]>(
+  args: [fn: Exclude<TArgs[0], Function>, ...rest: TArgs]
+): never;
+
+// export as namespace callBind;
+export = callBind;

scraper-service/node_modules/call-bind-apply-helpers/index.js

@@ -0,0 +1,15 @@
+'use strict';
+
+var bind = require('function-bind');
+var $TypeError = require('es-errors/type');
+
+var $call = require('./functionCall');
+var $actualApply = require('./actualApply');
+
+/** @type {(args: [Function, thisArg?: unknown, ...args: unknown[]]) => Function} TODO FIXME, find a way to use import('.') */
+module.exports = function callBindBasic(args) {
+	if (args.length < 1 || typeof args[0] !== 'function') {
+		throw new $TypeError('a function is required');
+	}
+	return $actualApply(bind, $call, args);
+};

scraper-service/node_modules/call-bind-apply-helpers/package.json

@@ -0,0 +1,85 @@
+{
+	"name": "call-bind-apply-helpers",
+	"version": "1.0.2",
+	"description": "Helper functions around Function call/apply/bind, for use in `call-bind`",
+	"main": "index.js",
+	"exports": {
+		".": "./index.js",
+		"./actualApply": "./actualApply.js",
+		"./applyBind": "./applyBind.js",
+		"./functionApply": "./functionApply.js",
+		"./functionCall": "./functionCall.js",
+		"./reflectApply": "./reflectApply.js",
+		"./package.json": "./package.json"
+	},
+	"scripts": {
+		"prepack": "npmignore --auto --commentLines=auto",
+		"prepublish": "not-in-publish || npm run prepublishOnly",
+		"prepublishOnly": "safe-publish-latest",
+		"prelint": "evalmd README.md",
+		"lint": "eslint --ext=.js,.mjs .",
+		"postlint": "tsc -p . && attw -P",
+		"pretest": "npm run lint",
+		"tests-only": "nyc tape 'test/**/*.js'",
+		"test": "npm run tests-only",
+		"posttest": "npx npm@'>=10.2' audit --production",
+		"version": "auto-changelog && git add CHANGELOG.md",
+		"postversion": "auto-changelog && git add CHANGELOG.md && git commit --no-edit --amend && git tag -f \"v$(node -e \"console.log(require('./package.json').version)\")\""
+	},
+	"repository": {
+		"type": "git",
+		"url": "git+https://github.com/ljharb/call-bind-apply-helpers.git"
+	},
+	"author": "Jordan Harband <ljharb@gmail.com>",
+	"license": "MIT",
+	"bugs": {
+		"url": "https://github.com/ljharb/call-bind-apply-helpers/issues"
+	},
+	"homepage": "https://github.com/ljharb/call-bind-apply-helpers#readme",
+	"dependencies": {
+		"es-errors": "^1.3.0",
+		"function-bind": "^1.1.2"
+	},
+	"devDependencies": {
+		"@arethetypeswrong/cli": "^0.17.3",
+		"@ljharb/eslint-config": "^21.1.1",
+		"@ljharb/tsconfig": "^0.2.3",
+		"@types/for-each": "^0.3.3",
+		"@types/function-bind": "^1.1.10",
+		"@types/object-inspect": "^1.13.0",
+		"@types/tape": "^5.8.1",
+		"auto-changelog": "^2.5.0",
+		"encoding": "^0.1.13",
+		"es-value-fixtures": "^1.7.1",
+		"eslint": "=8.8.0",
+		"evalmd": "^0.0.19",
+		"for-each": "^0.3.5",
+		"has-strict-mode": "^1.1.0",
+		"in-publish": "^2.0.1",
+		"npmignore": "^0.3.1",
+		"nyc": "^10.3.2",
+		"object-inspect": "^1.13.4",
+		"safe-publish-latest": "^2.0.0",
+		"tape": "^5.9.0",
+		"typescript": "next"
+	},
+	"testling": {
+		"files": "test/index.js"
+	},
+	"auto-changelog": {
+		"output": "CHANGELOG.md",
+		"template": "keepachangelog",
+		"unreleased": false,
+		"commitLimit": false,
+		"backfillLimit": false,
+		"hideCredit": true
+	},
+	"publishConfig": {
+		"ignore": [
+			".github/workflows"
+		]
+	},
+	"engines": {
+		"node": ">= 0.4"
+	}
+}

scraper-service/node_modules/call-bind-apply-helpers/reflectApply.d.ts

@@ -0,0 +1,3 @@
+declare const reflectApply: false | typeof Reflect.apply;
+
+export = reflectApply;

scraper-service/node_modules/call-bind-apply-helpers/reflectApply.js

@@ -0,0 +1,4 @@
+'use strict';
+
+/** @type {import('./reflectApply')} */
+module.exports = typeof Reflect !== 'undefined' && Reflect && Reflect.apply;

scraper-service/node_modules/call-bind-apply-helpers/test/index.js

@@ -0,0 +1,63 @@
+'use strict';
+
+var callBind = require('../');
+var hasStrictMode = require('has-strict-mode')();
+var forEach = require('for-each');
+var inspect = require('object-inspect');
+var v = require('es-value-fixtures');
+
+var test = require('tape');
+
+test('callBindBasic', function (t) {
+	forEach(v.nonFunctions, function (nonFunction) {
+		t['throws'](
+			// @ts-expect-error
+			function () { callBind([nonFunction]); },
+			TypeError,
+			inspect(nonFunction) + ' is not a function'
+		);
+	});
+
+	var sentinel = { sentinel: true };
+	/** @type {<T, A extends number, B extends number>(this: T, a: A, b: B) => [T | undefined, A, B]} */
+	var func = function (a, b) {
+		// eslint-disable-next-line no-invalid-this
+		return [!hasStrictMode && this === global ? undefined : this, a, b];
+	};
+	t.equal(func.length, 2, 'original function length is 2');
+
+	/** type {(thisArg: unknown, a: number, b: number) => [unknown, number, number]} */
+	var bound = callBind([func]);
+	/** type {((a: number, b: number) => [typeof sentinel, typeof a, typeof b])} */
+	var boundR = callBind([func, sentinel]);
+	/** type {((b: number) => [typeof sentinel, number, typeof b])} */
+	var boundArg = callBind([func, sentinel, /** @type {const} */ (1)]);
+
+	// @ts-expect-error
+	t.deepEqual(bound(), [undefined, undefined, undefined], 'bound func with no args');
+
+	// @ts-expect-error
+	t.deepEqual(func(), [undefined, undefined, undefined], 'unbound func with too few args');
+	// @ts-expect-error
+	t.deepEqual(bound(1, 2), [hasStrictMode ? 1 : Object(1), 2, undefined], 'bound func too few args');
+	// @ts-expect-error
+	t.deepEqual(boundR(), [sentinel, undefined, undefined], 'bound func with receiver, with too few args');
+	// @ts-expect-error
+	t.deepEqual(boundArg(), [sentinel, 1, undefined], 'bound func with receiver and arg, with too few args');
+
+	t.deepEqual(func(1, 2), [undefined, 1, 2], 'unbound func with right args');
+	t.deepEqual(bound(1, 2, 3), [hasStrictMode ? 1 : Object(1), 2, 3], 'bound func with right args');
+	t.deepEqual(boundR(1, 2), [sentinel, 1, 2], 'bound func with receiver, with right args');
+	t.deepEqual(boundArg(2), [sentinel, 1, 2], 'bound func with receiver and arg, with right arg');
+
+	// @ts-expect-error
+	t.deepEqual(func(1, 2, 3), [undefined, 1, 2], 'unbound func with too many args');
+	// @ts-expect-error
+	t.deepEqual(bound(1, 2, 3, 4), [hasStrictMode ? 1 : Object(1), 2, 3], 'bound func with too many args');
+	// @ts-expect-error
+	t.deepEqual(boundR(1, 2, 3), [sentinel, 1, 2], 'bound func with receiver, with too many args');
+	// @ts-expect-error
+	t.deepEqual(boundArg(2, 3), [sentinel, 1, 2], 'bound func with receiver and arg, with too many args');
+
+	t.end();
+});

scraper-service/node_modules/call-bind-apply-helpers/tsconfig.json

@@ -0,0 +1,9 @@
+{
+	"extends": "@ljharb/tsconfig",
+	"compilerOptions": {
+		"target": "es2021",
+	},
+	"exclude": [
+		"coverage",
+	],
+}

scraper-service/node_modules/call-bound/.eslintrc

@@ -0,0 +1,13 @@
+{
+	"root": true,
+
+	"extends": "@ljharb",
+
+	"rules": {
+		"new-cap": [2, {
+			"capIsNewExceptions": [
+				"GetIntrinsic",
+			],
+		}],
+	},
+}

scraper-service/node_modules/call-bound/.github/FUNDING.yml

@@ -0,0 +1,12 @@
+# These are supported funding model platforms
+
+github: [ljharb]
+patreon: # Replace with a single Patreon username
+open_collective: # Replace with a single Open Collective username
+ko_fi: # Replace with a single Ko-fi username
+tidelift: npm/call-bound
+community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
+liberapay: # Replace with a single Liberapay username
+issuehunt: # Replace with a single IssueHunt username
+otechie: # Replace with a single Otechie username
+custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2']

scraper-service/node_modules/call-bound/.nycrc

@@ -0,0 +1,9 @@
+{
+	"all": true,
+	"check-coverage": false,
+	"reporter": ["text-summary", "text", "html", "json"],
+	"exclude": [
+		"coverage",
+		"test"
+	]
+}

scraper-service/node_modules/call-bound/CHANGELOG.md

@@ -0,0 +1,42 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [v1.0.4](https://github.com/ljharb/call-bound/compare/v1.0.3...v1.0.4) - 2025-03-03
+
+### Commits
+
+- [types] improve types [`e648922`](https://github.com/ljharb/call-bound/commit/e6489222a9e54f350fbf952ceabe51fd8b6027ff)
+- [Dev Deps] update `@arethetypeswrong/cli`, `@ljharb/tsconfig`, `@types/tape`, `es-value-fixtures`, `for-each`, `has-strict-mode`, `object-inspect` [`a42a5eb`](https://github.com/ljharb/call-bound/commit/a42a5ebe6c1b54fcdc7997c7dc64fdca9e936719)
+- [Deps] update `call-bind-apply-helpers`, `get-intrinsic` [`f529eac`](https://github.com/ljharb/call-bound/commit/f529eac132404c17156bbc23ab2297a25d0f20b8)
+
+## [v1.0.3](https://github.com/ljharb/call-bound/compare/v1.0.2...v1.0.3) - 2024-12-15
+
+### Commits
+
+- [Refactor] use `call-bind-apply-helpers` instead of `call-bind` [`5e0b134`](https://github.com/ljharb/call-bound/commit/5e0b13496df14fb7d05dae9412f088da8d3f75be)
+- [Deps] update `get-intrinsic` [`41fc967`](https://github.com/ljharb/call-bound/commit/41fc96732a22c7b7e8f381f93ccc54bb6293be2e)
+- [readme] fix example [`79a0137`](https://github.com/ljharb/call-bound/commit/79a0137723f7c6d09c9c05452bbf8d5efb5d6e49)
+- [meta] add `sideEffects` flag [`08b07be`](https://github.com/ljharb/call-bound/commit/08b07be7f1c03f67dc6f3cdaf0906259771859f7)
+
+## [v1.0.2](https://github.com/ljharb/call-bound/compare/v1.0.1...v1.0.2) - 2024-12-10
+
+### Commits
+
+- [Dev Deps] update `@arethetypeswrong/cli`, `@ljharb/tsconfig`, `gopd` [`e6a5ffe`](https://github.com/ljharb/call-bound/commit/e6a5ffe849368fe4f74dfd6cdeca1b9baa39e8d5)
+- [Deps] update `call-bind`, `get-intrinsic` [`2aeb5b5`](https://github.com/ljharb/call-bound/commit/2aeb5b521dc2b2683d1345c753ea1161de2d1c14)
+- [types] improve return type [`1a0c9fe`](https://github.com/ljharb/call-bound/commit/1a0c9fe3114471e7ca1f57d104e2efe713bb4871)
+
+## v1.0.1 - 2024-12-05
+
+### Commits
+
+- Initial implementation, tests, readme, types [`6d94121`](https://github.com/ljharb/call-bound/commit/6d94121a9243602e506334069f7a03189fe3363d)
+- Initial commit [`0eae867`](https://github.com/ljharb/call-bound/commit/0eae867334ea025c33e6e91cdecfc9df96680cf9)
+- npm init [`71b2479`](https://github.com/ljharb/call-bound/commit/71b2479c6723e0b7d91a6b663613067e98b7b275)
+- Only apps should have lockfiles [`c3754a9`](https://github.com/ljharb/call-bound/commit/c3754a949b7f9132b47e2d18c1729889736741eb)
+- [actions] skip `npm ls` in node < 10 [`74275a5`](https://github.com/ljharb/call-bound/commit/74275a5186b8caf6309b6b97472bdcb0df4683a8)
+- [Dev Deps] add missing peer dep [`1354de8`](https://github.com/ljharb/call-bound/commit/1354de8679413e4ae9c523d85f76fa7a5e032d97)

scraper-service/node_modules/call-bound/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Jordan Harband
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

scraper-service/node_modules/call-bound/README.md

@@ -0,0 +1,53 @@
+# call-bound <sup>[![Version Badge][npm-version-svg]][package-url]</sup>
+
+[![github actions][actions-image]][actions-url]
+[![coverage][codecov-image]][codecov-url]
+[![dependency status][deps-svg]][deps-url]
+[![dev dependency status][dev-deps-svg]][dev-deps-url]
+[![License][license-image]][license-url]
+[![Downloads][downloads-image]][downloads-url]
+
+[![npm badge][npm-badge-png]][package-url]
+
+Robust call-bound JavaScript intrinsics, using `call-bind` and `get-intrinsic`.
+
+## Getting started
+
+```sh
+npm install --save call-bound
+```
+
+## Usage/Examples
+
+```js
+const assert = require('assert');
+const callBound = require('call-bound');
+
+const slice = callBound('Array.prototype.slice');
+
+delete Function.prototype.call;
+delete Function.prototype.bind;
+delete Array.prototype.slice;
+
+assert.deepEqual(slice([1, 2, 3, 4], 1, -1), [2, 3]);
+```
+
+## Tests
+
+Clone the repo, `npm install`, and run `npm test`
+
+[package-url]: https://npmjs.org/package/call-bound
+[npm-version-svg]: https://versionbadg.es/ljharb/call-bound.svg
+[deps-svg]: https://david-dm.org/ljharb/call-bound.svg
+[deps-url]: https://david-dm.org/ljharb/call-bound
+[dev-deps-svg]: https://david-dm.org/ljharb/call-bound/dev-status.svg
+[dev-deps-url]: https://david-dm.org/ljharb/call-bound#info=devDependencies
+[npm-badge-png]: https://nodei.co/npm/call-bound.png?downloads=true&stars=true
+[license-image]: https://img.shields.io/npm/l/call-bound.svg
+[license-url]: LICENSE
+[downloads-image]: https://img.shields.io/npm/dm/call-bound.svg
+[downloads-url]: https://npm-stat.com/charts.html?package=call-bound
+[codecov-image]: https://codecov.io/gh/ljharb/call-bound/branch/main/graphs/badge.svg
+[codecov-url]: https://app.codecov.io/gh/ljharb/call-bound/
+[actions-image]: https://img.shields.io/endpoint?url=https://github-actions-badge-u3jn4tfpocch.runkit.sh/ljharb/call-bound
+[actions-url]: https://github.com/ljharb/call-bound/actions

scraper-service/node_modules/call-bound/index.d.ts

@@ -0,0 +1,94 @@
+type Intrinsic = typeof globalThis;
+
+type IntrinsicName = keyof Intrinsic | `%${keyof Intrinsic}%`;
+
+type IntrinsicPath = IntrinsicName | `${StripPercents<IntrinsicName>}.${string}` | `%${StripPercents<IntrinsicName>}.${string}%`;
+
+type AllowMissing = boolean;
+
+type StripPercents<T extends string> = T extends `%${infer U}%` ? U : T;
+
+type BindMethodPrecise<F> =
+  F extends (this: infer This, ...args: infer Args) => infer R
+  ? (obj: This, ...args: Args) => R
+  : F extends {
+    (this: infer This1, ...args: infer Args1): infer R1;
+    (this: infer This2, ...args: infer Args2): infer R2
+  }
+  ? {
+    (obj: This1, ...args: Args1): R1;
+    (obj: This2, ...args: Args2): R2
+  }
+  : never
+
+// Extract method type from a prototype
+type GetPrototypeMethod<T extends keyof typeof globalThis, M extends string> =
+  (typeof globalThis)[T] extends { prototype: any }
+  ? M extends keyof (typeof globalThis)[T]['prototype']
+  ? (typeof globalThis)[T]['prototype'][M]
+  : never
+  : never
+
+// Get static property/method
+type GetStaticMember<T extends keyof typeof globalThis, P extends string> =
+  P extends keyof (typeof globalThis)[T] ? (typeof globalThis)[T][P] : never
+
+// Type that maps string path to actual bound function or value with better precision
+type BoundIntrinsic<S extends string> =
+  S extends `${infer Obj}.prototype.${infer Method}`
+  ? Obj extends keyof typeof globalThis
+  ? BindMethodPrecise<GetPrototypeMethod<Obj, Method & string>>
+  : unknown
+  : S extends `${infer Obj}.${infer Prop}`
+  ? Obj extends keyof typeof globalThis
+  ? GetStaticMember<Obj, Prop & string>
+  : unknown
+  : unknown
+
+declare function arraySlice<T>(array: readonly T[], start?: number, end?: number): T[];
+declare function arraySlice<T>(array: ArrayLike<T>, start?: number, end?: number): T[];
+declare function arraySlice<T>(array: IArguments, start?: number, end?: number): T[];
+
+// Special cases for methods that need explicit typing
+interface SpecialCases {
+  '%Object.prototype.isPrototypeOf%': (thisArg: {}, obj: unknown) => boolean;
+  '%String.prototype.replace%': {
+    (str: string, searchValue: string | RegExp, replaceValue: string): string;
+    (str: string, searchValue: string | RegExp, replacer: (substring: string, ...args: any[]) => string): string
+  };
+  '%Object.prototype.toString%': (obj: {}) => string;
+  '%Object.prototype.hasOwnProperty%': (obj: {}, v: PropertyKey) => boolean;
+  '%Array.prototype.slice%': typeof arraySlice;
+  '%Array.prototype.map%': <T, U>(array: readonly T[], callbackfn: (value: T, index: number, array: readonly T[]) => U, thisArg?: any) => U[];
+  '%Array.prototype.filter%': <T>(array: readonly T[], predicate: (value: T, index: number, array: readonly T[]) => unknown, thisArg?: any) => T[];
+  '%Array.prototype.indexOf%': <T>(array: readonly T[], searchElement: T, fromIndex?: number) => number;
+  '%Function.prototype.apply%': <T, A extends any[], R>(fn: (...args: A) => R, thisArg: any, args: A) => R;
+  '%Function.prototype.call%': <T, A extends any[], R>(fn: (...args: A) => R, thisArg: any, ...args: A) => R;
+  '%Function.prototype.bind%': <T, A extends any[], R>(fn: (...args: A) => R, thisArg: any, ...args: A) => (...remainingArgs: A) => R;
+  '%Promise.prototype.then%': {
+    <T, R>(promise: Promise<T>, onfulfilled: (value: T) => R | PromiseLike<R>): Promise<R>;
+    <T, R>(promise: Promise<T>, onfulfilled: ((value: T) => R | PromiseLike<R>) | undefined | null, onrejected: (reason: any) => R | PromiseLike<R>): Promise<R>;
+  };
+  '%RegExp.prototype.test%': (regexp: RegExp, str: string) => boolean;
+  '%RegExp.prototype.exec%': (regexp: RegExp, str: string) => RegExpExecArray | null;
+  '%Error.prototype.toString%': (error: Error) => string;
+  '%TypeError.prototype.toString%': (error: TypeError) => string;
+  '%String.prototype.split%': (
+        obj: unknown,
+        splitter: string | RegExp | {
+            [Symbol.split](string: string, limit?: number): string[];
+        },
+        limit?: number | undefined
+    ) => string[];
+}
+
+/**
+ * Returns a bound function for a prototype method, or a value for a static property.
+ *
+ * @param name - The name of the intrinsic (e.g. 'Array.prototype.slice')
+ * @param {AllowMissing} [allowMissing] - Whether to allow missing intrinsics (default: false)
+ */
+declare function callBound<K extends keyof SpecialCases | StripPercents<keyof SpecialCases>, S extends IntrinsicPath>(name: K, allowMissing?: AllowMissing): SpecialCases[`%${StripPercents<K>}%`];
+declare function callBound<K extends keyof SpecialCases | StripPercents<keyof SpecialCases>, S extends IntrinsicPath>(name: S, allowMissing?: AllowMissing): BoundIntrinsic<S>;
+
+export = callBound;

scraper-service/node_modules/call-bound/index.js

@@ -0,0 +1,19 @@
+'use strict';
+
+var GetIntrinsic = require('get-intrinsic');
+
+var callBindBasic = require('call-bind-apply-helpers');
+
+/** @type {(thisArg: string, searchString: string, position?: number) => number} */
+var $indexOf = callBindBasic([GetIntrinsic('%String.prototype.indexOf%')]);
+
+/** @type {import('.')} */
+module.exports = function callBoundIntrinsic(name, allowMissing) {
+	/* eslint no-extra-parens: 0 */
+
+	var intrinsic = /** @type {(this: unknown, ...args: unknown[]) => unknown} */ (GetIntrinsic(name, !!allowMissing));
+	if (typeof intrinsic === 'function' && $indexOf(name, '.prototype.') > -1) {
+		return callBindBasic(/** @type {const} */ ([intrinsic]));
+	}
+	return intrinsic;
+};

scraper-service/node_modules/call-bound/package.json

@@ -0,0 +1,99 @@
+{
+	"name": "call-bound",
+	"version": "1.0.4",
+	"description": "Robust call-bound JavaScript intrinsics, using `call-bind` and `get-intrinsic`.",
+	"main": "index.js",
+	"exports": {
+		".": "./index.js",
+		"./package.json": "./package.json"
+	},
+	"sideEffects": false,
+	"scripts": {
+		"prepack": "npmignore --auto --commentLines=auto",
+		"prepublish": "not-in-publish || npm run prepublishOnly",
+		"prepublishOnly": "safe-publish-latest",
+		"prelint": "evalmd README.md",
+		"lint": "eslint --ext=.js,.mjs .",
+		"postlint": "tsc -p . && attw -P",
+		"pretest": "npm run lint",
+		"tests-only": "nyc tape 'test/**/*.js'",
+		"test": "npm run tests-only",
+		"posttest": "npx npm@'>=10.2' audit --production",
+		"version": "auto-changelog && git add CHANGELOG.md",
+		"postversion": "auto-changelog && git add CHANGELOG.md && git commit --no-edit --amend && git tag -f \"v$(node -e \"console.log(require('./package.json').version)\")\""
+	},
+	"repository": {
+		"type": "git",
+		"url": "git+https://github.com/ljharb/call-bound.git"
+	},
+	"keywords": [
+		"javascript",
+		"ecmascript",
+		"es",
+		"js",
+		"callbind",
+		"callbound",
+		"call",
+		"bind",
+		"bound",
+		"call-bind",
+		"call-bound",
+		"function",
+		"es-abstract"
+	],
+	"author": "Jordan Harband <ljharb@gmail.com>",
+	"funding": {
+		"url": "https://github.com/sponsors/ljharb"
+	},
+	"license": "MIT",
+	"bugs": {
+		"url": "https://github.com/ljharb/call-bound/issues"
+	},
+	"homepage": "https://github.com/ljharb/call-bound#readme",
+	"dependencies": {
+		"call-bind-apply-helpers": "^1.0.2",
+		"get-intrinsic": "^1.3.0"
+	},
+	"devDependencies": {
+		"@arethetypeswrong/cli": "^0.17.4",
+		"@ljharb/eslint-config": "^21.1.1",
+		"@ljharb/tsconfig": "^0.3.0",
+		"@types/call-bind": "^1.0.5",
+		"@types/get-intrinsic": "^1.2.3",
+		"@types/tape": "^5.8.1",
+		"auto-changelog": "^2.5.0",
+		"encoding": "^0.1.13",
+		"es-value-fixtures": "^1.7.1",
+		"eslint": "=8.8.0",
+		"evalmd": "^0.0.19",
+		"for-each": "^0.3.5",
+		"gopd": "^1.2.0",
+		"has-strict-mode": "^1.1.0",
+		"in-publish": "^2.0.1",
+		"npmignore": "^0.3.1",
+		"nyc": "^10.3.2",
+		"object-inspect": "^1.13.4",
+		"safe-publish-latest": "^2.0.0",
+		"tape": "^5.9.0",
+		"typescript": "next"
+	},
+	"testling": {
+		"files": "test/index.js"
+	},
+	"auto-changelog": {
+		"output": "CHANGELOG.md",
+		"template": "keepachangelog",
+		"unreleased": false,
+		"commitLimit": false,
+		"backfillLimit": false,
+		"hideCredit": true
+	},
+	"publishConfig": {
+		"ignore": [
+			".github/workflows"
+		]
+	},
+	"engines": {
+		"node": ">= 0.4"
+	}
+}

scraper-service/node_modules/call-bound/test/index.js

@@ -0,0 +1,61 @@
+'use strict';
+
+var test = require('tape');
+
+var callBound = require('../');
+
+/** @template {true} T @template U @typedef {T extends U ? T : never} AssertType */
+
+test('callBound', function (t) {
+	// static primitive
+	t.equal(callBound('Array.length'), Array.length, 'Array.length yields itself');
+	t.equal(callBound('%Array.length%'), Array.length, '%Array.length% yields itself');
+
+	// static non-function object
+	t.equal(callBound('Array.prototype'), Array.prototype, 'Array.prototype yields itself');
+	t.equal(callBound('%Array.prototype%'), Array.prototype, '%Array.prototype% yields itself');
+	t.equal(callBound('Array.constructor'), Array.constructor, 'Array.constructor yields itself');
+	t.equal(callBound('%Array.constructor%'), Array.constructor, '%Array.constructor% yields itself');
+
+	// static function
+	t.equal(callBound('Date.parse'), Date.parse, 'Date.parse yields itself');
+	t.equal(callBound('%Date.parse%'), Date.parse, '%Date.parse% yields itself');
+
+	// prototype primitive
+	t.equal(callBound('Error.prototype.message'), Error.prototype.message, 'Error.prototype.message yields itself');
+	t.equal(callBound('%Error.prototype.message%'), Error.prototype.message, '%Error.prototype.message% yields itself');
+
+	var x = callBound('Object.prototype.toString');
+	var y = callBound('%Object.prototype.toString%');
+
+	// prototype function
+	t.notEqual(x, Object.prototype.toString, 'Object.prototype.toString does not yield itself');
+	t.notEqual(y, Object.prototype.toString, '%Object.prototype.toString% does not yield itself');
+	t.equal(x(true), Object.prototype.toString.call(true), 'call-bound Object.prototype.toString calls into the original');
+	t.equal(y(true), Object.prototype.toString.call(true), 'call-bound %Object.prototype.toString% calls into the original');
+
+	t['throws'](
+		// @ts-expect-error
+		function () { callBound('does not exist'); },
+		SyntaxError,
+		'nonexistent intrinsic throws'
+	);
+	t['throws'](
+		// @ts-expect-error
+		function () { callBound('does not exist', true); },
+		SyntaxError,
+		'allowMissing arg still throws for unknown intrinsic'
+	);
+
+	t.test('real but absent intrinsic', { skip: typeof WeakRef !== 'undefined' }, function (st) {
+		st['throws'](
+			function () { callBound('WeakRef'); },
+			TypeError,
+			'real but absent intrinsic throws'
+		);
+		st.equal(callBound('WeakRef', true), undefined, 'allowMissing arg avoids exception');
+		st.end();
+	});
+
+	t.end();
+});

scraper-service/node_modules/call-bound/tsconfig.json

@@ -0,0 +1,10 @@
+{
+	"extends": "@ljharb/tsconfig",
+	"compilerOptions": {
+		"target": "ESNext",
+		"lib": ["es2024"],
+	},
+	"exclude": [
+		"coverage",
+	],
+}

scraper-service/node_modules/content-disposition/HISTORY.md

@@ -0,0 +1,60 @@
+0.5.4 / 2021-12-10
+==================
+
+  * deps: safe-buffer@5.2.1
+
+0.5.3 / 2018-12-17
+==================
+
+  * Use `safe-buffer` for improved Buffer API
+
+0.5.2 / 2016-12-08
+==================
+
+  * Fix `parse` to accept any linear whitespace character
+
+0.5.1 / 2016-01-17
+==================
+
+  * perf: enable strict mode
+
+0.5.0 / 2014-10-11
+==================
+
+  * Add `parse` function
+
+0.4.0 / 2014-09-21
+==================
+
+  * Expand non-Unicode `filename` to the full ISO-8859-1 charset
+
+0.3.0 / 2014-09-20
+==================
+
+  * Add `fallback` option
+  * Add `type` option
+
+0.2.0 / 2014-09-19
+==================
+
+  * Reduce ambiguity of file names with hex escape in buggy browsers
+
+0.1.2 / 2014-09-19
+==================
+
+  * Fix periodic invalid Unicode filename header
+
+0.1.1 / 2014-09-19
+==================
+
+  * Fix invalid characters appearing in `filename*` parameter
+
+0.1.0 / 2014-09-18
+==================
+
+  * Make the `filename` argument optional
+
+0.0.0 / 2014-09-18
+==================
+
+  * Initial release