Merge branch 'main' of https://git.coastit.co.za/caitlin/CRM_ENVR

ai-server/index.mjs

@@ -12,6 +12,7 @@ import http from "node:http"
 import fs from "node:fs"
 import path from "node:path"
 import { spawn } from "node:child_process"
+import crypto from "node:crypto"
 import { fileURLToPath } from "node:url"
 
 const __dirname = path.dirname(fileURLToPath(import.meta.url))
@@ -133,19 +134,21 @@ async function scrapeFacebook() {
   try {
     const body = await new Promise((resolve, reject) => {
       const parsed = new URL(SCRAPER_URL)
-      const req = http.request({ hostname: parsed.hostname, port: parsed.port || 3008, path: urlPath, method: "POST", timeout: 360000 }, (res) => {
+      let done = false
+      const req = http.request({ hostname: parsed.hostname, port: parsed.port || 3008, path: urlPath, method: "POST", timeout: 60000 }, (res) => {
         let data = ""
         res.on("data", (c) => data += c)
-        res.on("end", () => resolve(data))
-        res.on("error", reject)
+        res.on("end", () => { done = true; resolve(data) })
+        res.on("error", (e) => { if (!done) { done = true; reject(e) } })
       })
-      req.on("timeout", () => { req.destroy(); reject(new Error("timeout")) })
-      req.on("error", reject)
+      req.on("timeout", () => { if (!done) { done = true; req.destroy(); reject(new Error("scraper timeout")) } })
+      req.on("error", (e) => { if (!done) { done = true; reject(e) } })
       req.end()
     })
     const data = JSON.parse(body)
     return data
   } catch (e) {
+    console.error("scrapeFacebook error:", e.message)
     return null
   }
 }
@@ -198,6 +201,7 @@ Provide concise, actionable sales advice. When asked about a specific job catego
   const ollamaRes = await fetch(`${OLLAMA_URL}/api/chat`, {
     method: "POST",
     headers: { "Content-Type": "application/json" },
+    signal: AbortSignal.timeout(60000),
     body: JSON.stringify({
       model: MODEL,
       messages: [
@@ -320,6 +324,7 @@ const server = http.createServer(async (req, res) => {
       try {
         await new Promise((resolve, reject) => {
           const r = http.get(`${SCRAPER_URL}/health`, { timeout: 3000 }, (res) => { res.resume(); resolve() })
+          r.on("timeout", () => { r.destroy(); reject(new Error("timeout")) })
           r.on("error", reject)
         })
         results.scraper = true
@@ -328,6 +333,7 @@ const server = http.createServer(async (req, res) => {
       try {
         await new Promise((resolve, reject) => {
           const r = http.get(FRONTEND_URL, { timeout: 3000 }, (res) => { res.resume(); resolve() })
+          r.on("timeout", () => { r.destroy(); reject(new Error("timeout")) })
           r.on("error", reject)
         })
         results.frontend = true
@@ -539,38 +545,29 @@ const server = http.createServer(async (req, res) => {
     // Accepts { message, user_id?, user_role? } and returns AI response.
     // user_role must be "sales", "admin", or "super_admin" if provided.
     if (req.method === "POST" && pathname === "/ai/chat") {
-      const startTime = Date.now()
       const chunks = []
       req.on("data", c => chunks.push(c))
-      req.on("end", () => {
-        const rawBody = Buffer.concat(chunks).toString()
+      req.on("end", async () => {
         try {
+          const rawBody = Buffer.concat(chunks).toString()
           const body = JSON.parse(rawBody)
-          processRequest(req, res, body, startTime)
-        } catch {
-          sendJSON(res, 400, { error: "Invalid JSON" })
+          const { message, user_id, user_role } = body
+          if (!message) {
+            return sendJSON(res, 400, { error: "message is required" })
+          }
+          const validRoles = ["sales", "admin", "super_admin"]
+          if (user_role && !validRoles.includes(user_role)) {
+            return sendJSON(res, 403, { error: "Forbidden" })
+          }
+          const response = await handleChat(message, user_id || "", user_role || "sales")
+          sendJSON(res, 200, { response })
+        } catch (e) {
+          if (!res.headersSent) sendJSON(res, 500, { error: e.message })
         }
       })
       return
     }
 
-// Separate handler for /ai/chat (defined here due to hoisting within the IIFE)
-async function processRequest(req, res, body, startTime) {
-      const { message, user_id, user_role } = body
-
-      if (!message) {
-        return sendJSON(res, 400, { error: "message is required" })
-      }
-
-      const validRoles = ["sales", "admin", "super_admin"]
-      if (user_role && !validRoles.includes(user_role)) {
-        return sendJSON(res, 403, { error: "Forbidden" })
-      }
-
-      const response = await handleChat(message, user_id || "", user_role || "sales")
-      return sendJSON(res, 200, { response })
-    }
-
     // 404 fallback
     sendJSON(res, 404, { error: "Not found" })
   } catch (err) {

browser-use-service/main.py

@@ -666,38 +666,24 @@ TUTORING_SEARCHES = [
     "tutor near me for",
 ]
 
-def _search_list_for_query(query: str) -> list[str]:
-    """Pick the appropriate search query pool based on the search term."""
-    tl = query.lower()
-    tutoring_terms = ["tutor", "tutoring", "lessons", "homework", "teach", "learning", "child", "math", "english", "science", "exam", "homeschool", "coding", "programming", "piano", "reading"]
-    if any(t in tl for t in tutoring_terms):
-        return TUTORING_SEARCHES
-    return FB_SEARCHES
-
 # ── South African Multi-Language Queries ──────────────────────────────
-# 4 most spoken SA languages: English, Afrikaans, isiXhosa, isiZulu.
-# Each scrape searches ALL 4 languages to catch leads across all
-# language communities on Facebook.
+# 4 SA languages grouped for phase-based scanning:
+# Phase 1: English → Phase 2: Afrikaans → Phase 3: isiXhosa → Phase 4: English (final sweep)
+# Each language group has dedicated queries per category.
 
-SA_WEBSITE_QUERIES = [
-    "I need a website for my business",          # English
-    "ek benodig n webwerf",                       # Afrikaans
-    "ek soek iemand om n webwerf te bou",         # Afrikaans
-    "ndidinga iwebhusayithi yeshishini",          # isiXhosa
-    "ndifuna umntu owakha iwebhusayithi",         # isiXhosa
-    "ngidinga iwebhusayithi yebhizinisi",         # isiZulu
-    "ngifuna umuntu owakha iwebhusayithi",        # isiZulu
-]
+SA_WEBSITE_QUERIES = {
+    "english": ["I need a website for my business"],
+    "afrikaans": ["ek benodig n webwerf", "ek soek iemand om n webwerf te bou"],
+    "xhosa": ["ndidinga iwebhusayithi yeshishini", "ndifuna umntu owakha iwebhusayithi"],
+    "zulu": ["ngidinga iwebhusayithi yebhizinisi", "ngifuna umuntu owakha iwebhusayithi"],
+}
 
-SA_TUTOR_QUERIES = [
-    "I need a tutor for my child",                # English
-    "ek benodig n tutor vir my kind",             # Afrikaans
-    "ek soek n privaat onderwyser",               # Afrikaans
-    "ndifuna utitshala womntwana wam",            # isiXhosa
-    "ndidinga umfundisi-ntsapho",                 # isiXhosa
-    "ngidinga uthisha wengane yami",              # isiZulu
-    "ngifuna umfundisi wengane",                  # isiZulu
-]
+SA_TUTOR_QUERIES = {
+    "english": ["I need a tutor for my child"],
+    "afrikaans": ["ek benodig n tutor vir my kind", "ek soek n privaat onderwyser"],
+    "xhosa": ["ndifuna utitshala womntwana wam", "ndidinga umfundisi-ntsapho"],
+    "zulu": ["ngidinga uthisha wengane yami", "ngifuna umfundisi wengane"],
+}
 
 
 async def _quick_search(page, context, query: str) -> tuple:
@@ -711,11 +697,17 @@ async def _quick_search(page, context, query: str) -> tuple:
         if '/login' in current_url.lower():
             logger.warning("Quick search redirected to login for '%s'", query[:40])
             return page, []
-        await page.wait_for_timeout(random.randint(4000, 6000))
-        await page.evaluate("window.scrollBy(0, 600)")
-        await page.wait_for_timeout(random.randint(2000, 4000))
-        await page.evaluate("window.scrollBy(0, 600)")
-        await page.wait_for_timeout(random.randint(2000, 3000))
+        await page.wait_for_timeout(random.randint(3000, 7000))
+        await page.evaluate(f"window.scrollBy(0, {random.randint(400, 900)})")
+        await page.wait_for_timeout(random.randint(2000, 5000))
+        if random.random() < 0.35:
+            await page.evaluate(f"window.scrollBy(0, -{random.randint(100, 400)})")
+            await page.wait_for_timeout(random.randint(1500, 3500))
+        await page.evaluate(f"window.scrollBy(0, {random.randint(400, 900)})")
+        await page.wait_for_timeout(random.randint(2000, 5000))
+        if random.random() < 0.25:
+            await page.evaluate("window.scrollTo(0, 0)")
+            await page.wait_for_timeout(random.randint(1000, 3000))
         raw_articles = await _get_article_elements(page)
         posts = _extract_posts_from_elements(raw_articles, url) if raw_articles else []
         raw = await page.evaluate('document.body.innerText')
@@ -865,7 +857,7 @@ def _parse_fb_date(block: list[str]) -> str:
     return datetime.now().strftime('%Y-%m-%d')
 
 
-def _is_within_days(date_str: str, max_days: int = 3) -> bool:
+def _is_within_days(date_str: str, max_days: int = 2) -> bool:
     """Check if date is within max_days from now. Empty/unparseable = keep."""
     if not date_str:
         return True
@@ -1355,6 +1347,94 @@ def cleanup_chrome():
         pass
 
 
+# ── 4-Phase Language Pipeline ─────────────────────────────────────────
+# Runs searches in ordered language phases:
+#   Phase 1: English (main query + supplementary EN searches)
+#   Phase 2: Afrikaans
+#   Phase 3: isiXhosa
+#   Phase 4: English (final sweep with different EN queries)
+# Each phase extracts posts and deduplicates on the fly.
+# Pipeline check (classify_leads) runs at end to quality-filter.
+
+PHASE_ORDER = ["english", "afrikaans", "xhosa", "zulu"]
+
+
+async def _run_phases(page, context, query: str | None = None) -> list[dict]:
+    """4-phase language pipeline: English → Afrikaans → Xhosa → Zulu.
+    Each phase finds leads in that language. Pipeline check at end filters + sorts by freshness.
+    Returns top 10 newest, most relevant leads."""
+    all_posts: list[dict] = []
+    seen_keys: set[str] = set()
+    tutoring = False
+
+    if query:
+        tl = query.lower()
+        tutoring = any(t in tl for t in ["tutor", "tutoring", "lessons", "homework", "teach", "learning", "child"])
+
+    lang_pool = SA_TUTOR_QUERIES if tutoring else SA_WEBSITE_QUERIES
+    en_pool = TUTORING_SEARCHES if tutoring else FB_SEARCHES
+
+    # Build phase query lists
+    supplement = random.sample(en_pool, k=min(3, len(en_pool))) if query else []
+    phase_queries: list[list[str]] = []
+
+    # Phase 1: English (user query + supplement from EN pool)
+    phase_queries.append(([query] + supplement) if query else random.sample(en_pool, k=min(4, len(en_pool))))
+
+    # Phase 2: Afrikaans
+    phase_queries.append(lang_pool.get("afrikaans", []))
+
+    # Phase 3: isiXhosa
+    phase_queries.append(lang_pool.get("xhosa", []))
+
+    # Phase 4: isiZulu
+    phase_queries.append(lang_pool.get("zulu", []))
+
+    # Execute phases
+    for phase_idx, queries in enumerate(phase_queries):
+        if not queries:
+            continue
+        phase_posts: list[dict] = []
+        for i, q in enumerate(queries):
+            is_first = (phase_idx == 0 and i == 0 and query is not None)
+            if is_first:
+                page, posts = await search_facebook(page, context, q)
+            else:
+                page, posts = await _quick_search(page, context, q)
+
+            for p in posts:
+                key = p.get('content', '')[:100]
+                if key and key not in seen_keys:
+                    seen_keys.add(key)
+                    phase_posts.append(p)
+
+        all_posts.extend(phase_posts)
+
+        # Stealth delay between phases (not after last)
+        if phase_idx < len(phase_queries) - 1 and phase_posts:
+            await page.wait_for_timeout(random.uniform(5000, 12000))
+            if random.random() < 0.2:
+                await random_idle(page)
+
+    # Pipeline check: date filter (2 days max) + AI/keyword classification
+    all_posts = [p for p in all_posts if _is_within_days(p.get('date', ''), 2)]
+
+    leads = all_posts[:20]
+    if leads:
+        leads = await classify_leads(leads, tutoring=tutoring)
+
+    # Sort by freshness — newest leads first
+    def _sort_key(l):
+        try:
+            return datetime.strptime((l.get('date') or '').strip()[:10], '%Y-%m-%d')
+        except (ValueError, IndexError):
+            return datetime.min
+
+    leads.sort(key=_sort_key, reverse=True)
+
+    return leads[:10]
+
+
 # ── Main Scrape Dispatcher ────────────────────────────────────────────
 # scrape_facebook() is the main entry point. It:
 #   1. Resolves the browser profile path (from SELECTED_BROWSER env var or auto-detect)
@@ -1409,17 +1489,17 @@ async def scrape_facebook(profile_path: str | None = None, force: bool = False,
     # Firefox path
     if browser_type == "firefox":
         result = await _scrape_with_firefox(effective_path, force, query)
-        if result.get("success") or not result.get("flagged"):
+        if result.get("success"):
             return result
-        logger.warning("Firefox flagged (%s), trying Agent", result.get("flag_reason", "unknown"))
-        return await _scrape_with_agent(force)
+        logger.warning("Firefox failed (reason: %s), trying Agent", result.get("flag_reason") or result.get("error", "unknown"))
+        return await _scrape_with_agent(force, query)
 
     # Chromium-based (chrome / opera / edge)
     result = await _scrape_with_chromium(effective_path, browser_type, force, query)
-    if result.get("success") or not result.get("flagged"):
+    if result.get("success"):
         return result
-    logger.warning("%s flagged (%s), trying Agent", browser_type, result.get("flag_reason", "unknown"))
-    return await _scrape_with_agent(force)
+    logger.warning("%s failed (reason: %s), trying Agent", browser_type, result.get("flag_reason") or result.get("error", "unknown"))
+    return await _scrape_with_agent(force, query)
 
 
 # ── Firefox Scraper ──────────────────────────────────────────────────
@@ -1442,6 +1522,7 @@ async def _scrape_with_firefox(profile_path: str, force: bool, query: str | None
             context = await pw.firefox.launch_persistent_context(
                 user_data_dir=profile_dir,
                 headless=True,
+                viewport=random.choice(VIEWPORTS),
                 firefox_user_prefs={
                     "dom.webdriver.enabled": False,
                     "dom.webdriver.timeout": 0,
@@ -1461,85 +1542,35 @@ async def _scrape_with_firefox(profile_path: str, force: bool, query: str | None
             except Exception:
                 logger.warning("Google navigation failed, trying Facebook directly")
 
-            await page.goto('https://www.facebook.com/', wait_until='domcontentloaded', timeout=30000)
-            await page.wait_for_timeout(random.randint(3000, 8000))
+            try:
+                await page.goto('https://www.facebook.com/', wait_until='domcontentloaded', timeout=30000)
+                await page.wait_for_timeout(random.randint(3000, 8000))
 
-            url = page.url
-            page_text = await page.evaluate('document.body.innerText') if '/login' in url.lower() else ''
-            det = check_detection_signals(url, page_text)
-            if det or '/login' in url.lower():
-                logger.warning("Facebook login page detected — flag: %s", det or "login_page")
-                await context.close()
-                return {"success": False, "leads": [], "flagged": True, "flag_reason": det or "login_page", "error": "Facebook login page detected"}
+                url = page.url
+                page_text = await page.evaluate('document.body.innerText') if '/login' in url.lower() else ''
+                det = check_detection_signals(url, page_text)
+                if det or '/login' in url.lower():
+                    logger.warning("Facebook login page detected — flag: %s", det or "login_page")
+                    return {"success": False, "leads": [], "flagged": True, "flag_reason": det or "login_page", "error": "Facebook login page detected"}
 
-            await human_scroll(page, steps=random.randint(2, 4), total_delay=random.uniform(8, 20))
-            if random.random() < 0.25:
-                await page.evaluate("window.scrollTo(0, 0)")
-                await page.wait_for_timeout(random.randint(2000, 5000))
-                await human_scroll(page, steps=random.randint(1, 2))
+                await human_scroll(page, steps=random.randint(2, 4), total_delay=random.uniform(8, 20))
+                if random.random() < 0.25:
+                    await page.evaluate("window.scrollTo(0, 0)")
+                    await page.wait_for_timeout(random.randint(2000, 5000))
+                    await human_scroll(page, steps=random.randint(1, 2))
 
-            if not force and random.random() < 0.3:
-                await page.wait_for_timeout(random.randint(8000, 20000))
-                await context.close()
-                return {"success": True, "leads": [], "flagged": False, "flag_reason": None, "error": None}
+                if not force and random.random() < 0.3:
+                    await page.wait_for_timeout(random.randint(8000, 20000))
+                    return {"success": True, "leads": [], "flagged": False, "flag_reason": None, "error": None}
 
-            all_posts = []
-            tutoring = False
-            if query:
-                tl = query.lower()
-                tutoring = any(t in tl for t in ["tutor", "tutoring", "lessons", "homework", "teach", "learning", "child"])
-                lang_pool = SA_TUTOR_QUERIES if tutoring else SA_WEBSITE_QUERIES
-                non_english = [q for q in lang_pool if q.strip().lower() != tl]
-                query_pool = _search_list_for_query(query)
-                supp_k = random.randint(3, 4) if tutoring else random.randint(2, 3)
-                supplement = random.sample(query_pool, k=supp_k)
-                searches = [query] + supplement + non_english
-                english_count = 1 + len(supplement)
-            else:
-                searches = random.sample(FB_SEARCHES + SA_WEBSITE_QUERIES + SA_TUTOR_QUERIES, k=random.randint(4, 7))
-            for i, sq in enumerate(searches):
-                page, posts = await search_facebook(page, context, sq) if i == 0 else await _quick_search(page, context, sq)
-                all_posts.extend(posts)
-                if not posts:
-                    continue
-                if i > 0:
-                    await page.wait_for_timeout(random.uniform(3000, 7000))
-                    continue
-                if random.random() < 0.4:
-                    await page.evaluate(f"window.scrollBy(0, {random.randint(-300, 300)})")
-                delay = random.uniform(8, 25)
-                await page.wait_for_timeout(int(delay * 1000))
-                if i == random.randint(0, 1) and random.random() < 0.15:
-                    new_page = await context.new_page()
-                    try:
-                        await new_page.goto('https://www.facebook.com/groups/', wait_until='domcontentloaded', timeout=15000)
-                        await new_page.wait_for_timeout(random.randint(3000, 8000))
-                    except Exception:
-                        pass
-                    await new_page.close()
-                    page = await _ensure_page(page, context)
+                leads = await _run_phases(page, context, query)
 
-            if random.random() < 0.5:
-                await page.wait_for_timeout(random.randint(3000, 10000))
-
-            await context.close()
-
-            seen = set()
-            deduped = []
-            for p in all_posts:
-                key = p.get('content', '')[:100]
-                if key not in seen:
-                    seen.add(key)
-                    deduped.append(p)
-
-            # Filter to last 3 days only
-            deduped = [p for p in deduped if _is_within_days(p.get('date', ''), 7)]
-
-            leads = deduped[:20]
-            if leads:
-                leads = await classify_leads(leads, tutoring=tutoring)
-
-            return {"success": True, "leads": leads[:15], "flagged": False, "flag_reason": None, "error": None}
+                return {"success": True, "leads": leads[:15], "flagged": False, "flag_reason": None, "error": None}
+            finally:
+                try:
+                    await context.close()
+                except Exception:
+                    pass
 
     except Exception as e:
         logger.error("Firefox scrape failed: %s", e)
@@ -1589,6 +1620,7 @@ async def _scrape_with_chromium(profile_path: str, browser: str, force: bool = F
             launch_kwargs = dict(
                 user_data_dir=profile_dir,
                 headless=True,
+                viewport=random.choice(VIEWPORTS),
                 args=CHROME_LAUNCH_ARGS,
             )
             if channel:
@@ -1607,83 +1639,35 @@ async def _scrape_with_chromium(profile_path: str, browser: str, force: bool = F
             except Exception:
                 logger.warning("Google navigation failed, trying Facebook directly")
 
-            await page.goto('https://www.facebook.com/', wait_until='domcontentloaded', timeout=30000)
-            await page.wait_for_timeout(random.randint(3000, 8000))
+            try:
+                await page.goto('https://www.facebook.com/', wait_until='domcontentloaded', timeout=30000)
+                await page.wait_for_timeout(random.randint(3000, 8000))
 
-            url = page.url
-            page_text = await page.evaluate('document.body.innerText') if '/login' in url.lower() else ''
-            det = check_detection_signals(url, page_text)
-            if det or '/login' in url.lower():
-                logger.warning("Facebook login page detected — flag: %s", det or "login_page")
-                await context.close()
-                return {"success": False, "leads": [], "flagged": True, "flag_reason": det or "login_page", "error": "Facebook login page detected"}
+                url = page.url
+                page_text = await page.evaluate('document.body.innerText') if '/login' in url.lower() else ''
+                det = check_detection_signals(url, page_text)
+                if det or '/login' in url.lower():
+                    logger.warning("Facebook login page detected — flag: %s", det or "login_page")
+                    return {"success": False, "leads": [], "flagged": True, "flag_reason": det or "login_page", "error": "Facebook login page detected"}
 
-            await human_scroll(page, steps=random.randint(2, 4), total_delay=random.uniform(8, 20))
-            if random.random() < 0.25:
-                await page.evaluate("window.scrollTo(0, 0)")
-                await page.wait_for_timeout(random.randint(2000, 5000))
-                await human_scroll(page, steps=random.randint(1, 2))
+                await human_scroll(page, steps=random.randint(2, 4), total_delay=random.uniform(8, 20))
+                if random.random() < 0.25:
+                    await page.evaluate("window.scrollTo(0, 0)")
+                    await page.wait_for_timeout(random.randint(2000, 5000))
+                    await human_scroll(page, steps=random.randint(1, 2))
 
-            if not force and random.random() < 0.3:
-                await page.wait_for_timeout(random.randint(8000, 20000))
-                await context.close()
-                return {"success": True, "leads": [], "flagged": False, "flag_reason": None, "error": None}
+                if not force and random.random() < 0.3:
+                    await page.wait_for_timeout(random.randint(8000, 20000))
+                    return {"success": True, "leads": [], "flagged": False, "flag_reason": None, "error": None}
 
-            all_posts = []
-            tutoring = False
-            if query:
-                tl = query.lower()
-                tutoring = any(t in tl for t in ["tutor", "tutoring", "lessons", "homework", "teach", "learning", "child"])
-                lang_pool = SA_TUTOR_QUERIES if tutoring else SA_WEBSITE_QUERIES
-                non_english = [q for q in lang_pool if q.strip().lower() != tl]
-                query_pool = _search_list_for_query(query)
-                supp_k = random.randint(3, 4) if tutoring else random.randint(2, 3)
-                supplement = random.sample(query_pool, k=supp_k)
-                searches = [query] + supplement + non_english
-                english_count = 1 + len(supplement)
-            else:
-                searches = random.sample(FB_SEARCHES + SA_WEBSITE_QUERIES + SA_TUTOR_QUERIES, k=random.randint(4, 7))
-            for i, sq in enumerate(searches):
-                page, posts = await search_facebook(page, context, sq) if i == 0 else await _quick_search(page, context, sq)
-                all_posts.extend(posts)
-                if not posts:
-                    continue
-                if i > 0:
-                    await page.wait_for_timeout(random.uniform(3000, 7000))
-                    continue
-                if random.random() < 0.4:
-                    await page.evaluate(f"window.scrollBy(0, {random.randint(-300, 300)})")
-                delay = random.uniform(8, 25)
-                await page.wait_for_timeout(int(delay * 1000))
-                if i == random.randint(0, 1) and random.random() < 0.15:
-                    new_page = await context.new_page()
-                    try:
-                        await new_page.goto('https://www.facebook.com/groups/', wait_until='domcontentloaded', timeout=15000)
-                        await new_page.wait_for_timeout(random.randint(3000, 8000))
-                    except Exception:
-                        pass
-                    await new_page.close()
-                    page = await _ensure_page(page, context)
+                leads = await _run_phases(page, context, query)
 
-            if random.random() < 0.5:
-                await page.wait_for_timeout(random.randint(3000, 10000))
-
-            await context.close()
-
-            seen = set()
-            deduped = []
-            for p in all_posts:
-                key = p.get('content', '')[:100]
-                if key not in seen:
-                    seen.add(key)
-                    deduped.append(p)
-
-            deduped = [p for p in deduped if _is_within_days(p.get('date', ''), 7)]
-            leads = deduped[:20]
-            if leads:
-                leads = await classify_leads(leads, tutoring=tutoring)
-
-            return {"success": True, "leads": leads[:15], "flagged": False, "flag_reason": None, "error": None}
+                return {"success": True, "leads": leads[:15], "flagged": False, "flag_reason": None, "error": None}
+            finally:
+                try:
+                    await context.close()
+                except Exception:
+                    pass
 
     except Exception as e:
         logger.error("%s scrape failed: %s", browser, e)
@@ -1703,7 +1687,7 @@ async def _scrape_with_chromium(profile_path: str, browser: str, force: bool = F
 # Uses Chromium headless with the same launch args as _scrape_with_chromium.
 # The Agent is prompted to extract structured post data and return JSON.
 
-async def _scrape_with_agent(force: bool = False) -> dict:
+async def _scrape_with_agent(force: bool = False, query: str | None = None) -> dict:
     """Fallback scraper — browser-use Agent + ChatOllama (free/local, Chromium)."""
     cleanup_chrome()
     profile_dir = None
@@ -1722,7 +1706,13 @@ async def _scrape_with_agent(force: bool = False) -> dict:
         await browser.start()
 
         all_posts = []
-        pool = FB_SEARCHES + random.sample(SA_WEBSITE_QUERIES, k=min(4, len(SA_WEBSITE_QUERIES)))
+        tutoring_agent = False
+        if query:
+            tl = query.lower()
+            tutoring_agent = any(t in tl for t in ["tutor", "tutoring", "lessons", "homework", "teach", "learning", "child"])
+        sa_dict = SA_TUTOR_QUERIES if tutoring_agent else SA_WEBSITE_QUERIES
+        sa_all = sa_dict.get("afrikaans", []) + sa_dict.get("xhosa", []) + sa_dict.get("zulu", [])
+        pool = FB_SEARCHES + sa_all
         for query in random.sample(pool, k=random.randint(2, 4)):
             agent = _make_agent(
                 task=f"""You are logged into Facebook. Do the following:
@@ -1734,7 +1724,7 @@ async def _scrape_with_agent(force: bool = False) -> dict:
    - The post text content
    - The post URL (if visible)
    - The post date
-5. ONLY include posts from the last 3 days
+5. ONLY include posts from the last 2 days
 6. Collect as many posts as you can (aim for 5-10 per search)
 
 When done, return the data as a JSON list with keys: content, author, url, date.""",
@@ -1763,12 +1753,12 @@ When done, return the data as a JSON list with keys: content, author, url, date.
                 seen.add(key)
                 deduped.append(p)
 
-        # Filter to last 3 days only
-        deduped = [p for p in deduped if _is_within_days(p.get('date', ''), 7)]
+        # Filter to last 2 days only
+        deduped = [p for p in deduped if _is_within_days(p.get('date', ''), 2)]
 
         leads = deduped[:20]
         if leads:
-            leads = await classify_leads(leads, tutoring=tutoring)
+            leads = await classify_leads(leads, tutoring=tutoring_agent)
 
         return {"success": True, "leads": leads[:15], "flagged": False, "flag_reason": None, "error": None}
     except Exception as e:
@@ -1808,7 +1798,7 @@ async def classify_leads(results: list[dict], tutoring: bool = False) -> list[di
         return []
 
     # ── 1. AI classification ─────────────────────────────────────────
-    briefs = [r["title"][:200] for r in results]
+    briefs = [(r.get("title") or r.get("content") or "")[:200] for r in results]
     if tutoring:
         lead_desc = "someone REQUESTING/LOOKING FOR/WANTING a tutor, teacher, or lessons for their child or themselves"
         lead_examples = '"Looking for a tutor for my child", "Need a math tutor for my son", "Need help with homework", "Looking for piano lessons for my daughter", "Need a reading tutor"'
@@ -2009,7 +1999,7 @@ Return a JSON array like ["yes","no","yes"] matching the order above."""
         'website design ideas', 'website inspiration',
     ]
     for r in results:
-        t = r['title'].lower()
+        t = (r.get('title') or r.get('content') or '').lower()
         has_target = any(kw in t for kw in target_terms)
         has_request = any(kw in t for kw in request_terms)
         if not has_target or not has_request:
@@ -2021,11 +2011,11 @@ Return a JSON array like ["yes","no","yes"] matching the order above."""
         keyword_leads.append(r)
 
     # ── 3. Merge: prefer AI leads, supplement with keywords ──
-    seen_titles: set[int] = set()
+    seen_titles: set[str] = set()
     merged: list[dict] = []
     for r in ai_leads + keyword_leads:
-        key = hash(r.get('title', ''))
-        if key not in seen_titles:
+        key = (r.get('title') or '').strip()[:200]
+        if key and key not in seen_titles:
             seen_titles.add(key)
             merged.append(r)
     # Final sweep: strip any remaining offers or group posts from merged

data/ai/ai.md

@@ -34,18 +34,27 @@ The scraper lives at `browser-use-service/main.py` port 3008.
 ### How It Works
 1. **Browser detection** — tries Firefox profile first, then Chromium-based (Chrome/Opera/Edge), falls back to browser-use Agent
 2. **Profile paths** — configured via env vars (`FX_PROFILE`, `CHROME_PROFILE`, `OPERA_PROFILE`, `EDGE_PROFILE`) or auto-detected on first run
-3. **Search dispatch** — per scrape run:
-   - 1 English primary search (full scroll with human-like delays)
-   - 2-3 English supplementary searches (quick searches)
-   - 6-7 non-English quick searches (Afrikaans, isiXhosa, isiZulu — 2 queries each per category)
-   - Total: ~14 searches per scrape, completed in 2-4 minutes
-4. **Quick searches** — load page, double-scroll, extract visible posts (~12-18s each)
-5. **Date filter** — only posts within 7 days are considered
-6. **2-pass classification (dead-accurate)**:
+3. **4-phase language pipeline** (English → Afrikaans → Xhosa → Zulu):
+   - **Phase 1 (English)**: User's selected query + 2-3 supplementary English searches from the English search pool. First query gets full human-like scroll, rest use quick search. This phase does the heavy lifting.
+   - **Phase 2 (Afrikaans)**: 2 Afrikaans queries targeting Afrikaans-speaking communities.
+   - **Phase 3 (isiXhosa)**: 2 Xhosa queries targeting Xhosa-speaking communities.
+   - **Phase 4 (isiZulu)**: 2 Zulu queries targeting Zulu-speaking communities.
+   - After all phases: pipeline check (date filter 2 days → AI + keyword classification → sort by freshness). Newest leads ranked first.
+   - Each phase extracts posts, deduplicates against all prior phases, then passes through a stealth delay (5-12s + mouse idle) before the next phase.
+4. **Quick searches** — load page, double-scroll, extract visible posts (~12-18s each). Scroll-back behavior (35% chance to scroll up) and random return-to-top (25% chance) for stealth.
+5. **Date filter** — only posts within **2 days** are considered. Anything older is discarded. Fresh leads only.
+6. **Stealth mechanics**:
+   - Random viewport dimensions (1280×800 to 1920×1080) — never the same size twice
+   - Variable delays between searches (5-12 seconds) with mouse idle actions mixed in
+   - Human-like scroll patterns: scroll down, pause, sometimes scroll back up, sometimes return to top
+   - Canvas/WebGL/audio fingerprint spoofing via injected init scripts
+   - Random decoy page visits (e.g., Facebook Groups) between searches
+   - Profile directory is temp-copied and cleaned up after each scrape
+   - Detection signal monitoring (checkpoint, login pages, security challenges)
+7. **2-pass classification (dead-accurate)**:
    - **Pass 1 (AI)**: Ollama classifies each post as LEAD or NOT using a strict prompt per category. This is the primary filter and most accurate.
    - **Pass 2 (Keyword)**: Only posts matching BOTH a target term AND a request term are kept. Requires multi-word phrases — standalone words like "need", "want", "help" are NOT used as they cause false positives. Aggressive reject list catches service offers, self-promotions, portfolio posts, learning-requests, and existing-site issues.
    - **No loose fill**: Unlike the old approach, there is NO third pass that accepts posts matching EITHER term. Every returned lead has passed both AI and/or strict keyword validation. If fewer than 5 posts pass, that means only genuine leads are returned — no noise to pad the count.
-7. **Anti-detection** — random delays, human-like scrolling, user-agent rotation, proxy support
 8. **Scrape timing** — 3-6 minutes for a complete run. Returns 5-10 leads with high confidence.
 
 ### Lead Categories
@@ -63,12 +72,12 @@ Two categories, selectable when starting a scrape:
 - Request terms: same as website category — must co-occur with a target keyword
 - Strict reject: people offering tutoring, educational products, homeschool programs, free trials, general study tips
 
-### Multi-Language Support
-Searches in 4 South African languages:
-- English — 1 primary + 2-3 supplementary queries
-- Afrikaans — 2 queries (e.g., "ek benodig n webwerf", "ek soek n privaat onderwyser")
-- isiXhosa — 2 queries (e.g., "ndidinga iwebhusayithi yeshishini", "ndifuna utitshala womntwana wam")
-- isiZulu — 2 queries (e.g., "ngidinga iwebhusayithi yebhizinisi", "ngifuna umfundisi wengane")
+### Multi-Language Pipeline (Phase Order)
+4 South African languages in structured phases:
+- **Phase 1 (English)**: primary query + supplementary English searches
+- **Phase 2 (Afrikaans)**: 2 queries targeting Afrikaans speakers
+- **Phase 3 (isiXhosa)**: 2 queries targeting Xhosa speakers
+- **Phase 4 (isiZulu)**: 2 queries targeting Zulu speakers
 
 ### Output Format
 Each lead returned includes:

rust-ai/src/AI.md

@@ -34,18 +34,27 @@ The scraper lives at `browser-use-service/main.py` port 3008.
 ### How It Works
 1. **Browser detection** — tries Firefox profile first, then Chromium-based (Chrome/Opera/Edge), falls back to browser-use Agent
 2. **Profile paths** — configured via env vars (`FX_PROFILE`, `CHROME_PROFILE`, `OPERA_PROFILE`, `EDGE_PROFILE`) or auto-detected on first run
-3. **Search dispatch** — per scrape run:
-   - 1 English primary search (full scroll with human-like delays)
-   - 2-3 English supplementary searches (quick searches)
-   - 6-7 non-English quick searches (Afrikaans, isiXhosa, isiZulu — 2 queries each per category)
-   - Total: ~14 searches per scrape, completed in 2-4 minutes
-4. **Quick searches** — load page, double-scroll, extract visible posts (~12-18s each)
-5. **Date filter** — only posts within 7 days are considered
-6. **2-pass classification (dead-accurate)**:
+3. **4-phase language pipeline** (English → Afrikaans → Xhosa → Zulu):
+   - **Phase 1 (English)**: User's selected query + 2-3 supplementary English searches from the English search pool. First query gets full human-like scroll, rest use quick search. This phase does the heavy lifting.
+   - **Phase 2 (Afrikaans)**: 2 Afrikaans queries targeting Afrikaans-speaking communities.
+   - **Phase 3 (isiXhosa)**: 2 Xhosa queries targeting Xhosa-speaking communities.
+   - **Phase 4 (isiZulu)**: 2 Zulu queries targeting Zulu-speaking communities.
+   - After all phases: pipeline check (date filter 2 days → AI + keyword classification → sort by freshness). Newest leads ranked first.
+   - Each phase extracts posts, deduplicates against all prior phases, then passes through a stealth delay (5-12s + mouse idle) before the next phase.
+4. **Quick searches** — load page, double-scroll, extract visible posts (~12-18s each). Scroll-back behavior (35% chance to scroll up) and random return-to-top (25% chance) for stealth.
+5. **Date filter** — only posts within **2 days** are considered. Anything older is discarded. Fresh leads only.
+6. **Stealth mechanics**:
+   - Random viewport dimensions (1280×800 to 1920×1080) — never the same size twice
+   - Variable delays between searches (5-12 seconds) with mouse idle actions mixed in
+   - Human-like scroll patterns: scroll down, pause, sometimes scroll back up, sometimes return to top
+   - Canvas/WebGL/audio fingerprint spoofing via injected init scripts
+   - Random decoy page visits (e.g., Facebook Groups) between searches
+   - Profile directory is temp-copied and cleaned up after each scrape
+   - Detection signal monitoring (checkpoint, login pages, security challenges)
+7. **2-pass classification (dead-accurate)**:
    - **Pass 1 (AI)**: Ollama classifies each post as LEAD or NOT using a strict prompt per category. This is the primary filter and most accurate.
    - **Pass 2 (Keyword)**: Only posts matching BOTH a target term AND a request term are kept. Requires multi-word phrases — standalone words like "need", "want", "help" are NOT used as they cause false positives. Aggressive reject list catches service offers, self-promotions, portfolio posts, learning-requests, and existing-site issues.
    - **No loose fill**: Unlike the old approach, there is NO third pass that accepts posts matching EITHER term. Every returned lead has passed both AI and/or strict keyword validation. If fewer than 5 posts pass, that means only genuine leads are returned — no noise to pad the count.
-7. **Anti-detection** — random delays, human-like scrolling, user-agent rotation, proxy support
 8. **Scrape timing** — 3-6 minutes for a complete run. Returns 5-10 leads with high confidence.
 
 ### Lead Categories
@@ -63,12 +72,12 @@ Two categories, selectable when starting a scrape:
 - Request terms: same as website category — must co-occur with a target keyword
 - Strict reject: people offering tutoring, educational products, homeschool programs, free trials, general study tips
 
-### Multi-Language Support
-Searches in 4 South African languages:
-- English — 1 primary + 2-3 supplementary queries
-- Afrikaans — 2 queries (e.g., "ek benodig n webwerf", "ek soek n privaat onderwyser")
-- isiXhosa — 2 queries (e.g., "ndidinga iwebhusayithi yeshishini", "ndifuna utitshala womntwana wam")
-- isiZulu — 2 queries (e.g., "ngidinga iwebhusayithi yebhizinisi", "ngifuna umfundisi wengane")
+### Multi-Language Pipeline (Phase Order)
+4 South African languages in structured phases:
+- **Phase 1 (English)**: primary query + supplementary English searches
+- **Phase 2 (Afrikaans)**: 2 queries targeting Afrikaans speakers
+- **Phase 3 (isiXhosa)**: 2 queries targeting Xhosa speakers
+- **Phase 4 (isiZulu)**: 2 queries targeting Zulu speakers
 
 ### Output Format
 Each lead returned includes:
@@ -76,7 +85,7 @@ Each lead returned includes:
 - `author` — poster's name (may include location in name)
 - `content` — extracted post text
 - `url` — direct link to the post
-- `date` — when posted (filtered within 7 days)
+- `date` — when posted (filtered within 2 days)
 - `category` — "website" or "tutor"
 
 Target is 5-10 dead-accurate leads per scrape. Quality over quantity — no loose padding.

src/app/(dashboard)/ai-assistant/page.tsx

@@ -17,7 +17,7 @@ export default function AIAssistantPage() {
 
   const handleSearch = useCallback(async (job: NonNullable<typeof selectedJob>) => {
     setSearching(true)
-    const keyword = job.keywords[0]
+    const keyword = job.keywords?.[0] || job.job_title
     aiChatRef.current?.addAssistantMessage(`🔍 Searching Facebook for **${job.job_title}** leads...`)
 
     const controller = new AbortController()
@@ -34,9 +34,12 @@ export default function AIAssistantPage() {
       clearTimeout(statusId)
       const data = await res.json()
       if (data.success && data.leads?.length > 0) {
-        const leadsText = data.leads.map((lead: any, i: number) =>
-          `**${i + 1}.** ${lead.author || "Unknown"}\n> ${(lead.content || "").slice(0, 300)}\n> 🔗 ${lead.url || "(no link available)"}`
-        ).join("\n\n")
+        const leadLines = data.leads
+          .filter(Boolean)
+          .map((lead: Record<string, string>, i: number) =>
+            `**${i + 1}.** ${lead?.author || "Unknown"}\n> ${(lead?.content || "").slice(0, 300)}\n> 🔗 ${lead?.url || "(no link available)"}`
+          )
+        const leadsText = leadLines.join("\n\n")
         aiChatRef.current?.addAssistantMessage(`✅ Found **${data.leads.length}** leads:\n\n${leadsText}`)
       } else {
         const reason = data.error || data.flag_reason || "No leads found this time"

src/components/ai/job-selector.tsx

@@ -62,7 +62,7 @@ export function JobSelector({ onSelect, onSearch, searching }: JobSelectorProps)
                 className="w-full text-left px-4 py-3 text-sm text-muted-foreground hover:bg-muted hover:text-foreground transition-all duration-150 border-b border-border/20 last:border-b-0 border-l-2 border-l-transparent hover:border-l-primary/40"
               >
                 <div className="font-medium">{job.job_title}</div>
-                <div className="text-xs text-muted-foreground/60 mt-0.5">{job.industry} &mdash; {job.description}</div>
+                <div className="text-xs text-muted-foreground/60 mt-0.5">{job.industry} — {job.description}</div>
               </button>
             ))}
             {jobs.length === 0 && !loading && (

src/components/settings/theme-settings.tsx

@@ -90,7 +90,7 @@ export function ThemeSettings() {
                 <Label
                   htmlFor={`color-${value}`}
                   className={cn(
-                    "flex flex-col items-center gap-3 rounded-lg border-2 p-4 hover:bg-accent cursor-pointer transition-all",
+                    "flex flex-col items-center gap-3 rounded-lg border-2 p-4 hover:bg-muted cursor-pointer transition-all",
                     "peer-data-[state=checked]:border-primary peer-data-[state=checked]:bg-primary/5"
                   )}
                 >
@@ -118,7 +118,7 @@ export function ThemeSettings() {
                 <Label
                   htmlFor={`bg-${value}`}
                   className={cn(
-                    "flex flex-col items-center gap-3 rounded-lg border-2 p-4 hover:bg-accent cursor-pointer transition-all",
+                    "flex flex-col items-center gap-3 rounded-lg border-2 p-4 hover:bg-muted cursor-pointer transition-all",
                     "peer-data-[state=checked]:border-primary peer-data-[state=checked]:bg-primary/5"
                   )}
                 >