Added finishing touch on other languages

ai-server/index.mjs

@@ -12,6 +12,7 @@ import http from "node:http"
 import fs from "node:fs"
 import path from "node:path"
 import { spawn } from "node:child_process"
+import crypto from "node:crypto"
 import { fileURLToPath } from "node:url"
 
 const __dirname = path.dirname(fileURLToPath(import.meta.url))
@@ -133,19 +134,21 @@ async function scrapeFacebook() {
   try {
     const body = await new Promise((resolve, reject) => {
       const parsed = new URL(SCRAPER_URL)
-      const req = http.request({ hostname: parsed.hostname, port: parsed.port || 3008, path: urlPath, method: "POST", timeout: 360000 }, (res) => {
+      let done = false
+      const req = http.request({ hostname: parsed.hostname, port: parsed.port || 3008, path: urlPath, method: "POST", timeout: 60000 }, (res) => {
         let data = ""
         res.on("data", (c) => data += c)
-        res.on("end", () => resolve(data))
+        res.on("end", () => { done = true; resolve(data) })
-        res.on("error", reject)
+        res.on("error", (e) => { if (!done) { done = true; reject(e) } })
       })
-      req.on("timeout", () => { req.destroy(); reject(new Error("timeout")) })
+      req.on("timeout", () => { if (!done) { done = true; req.destroy(); reject(new Error("scraper timeout")) } })
-      req.on("error", reject)
+      req.on("error", (e) => { if (!done) { done = true; reject(e) } })
       req.end()
     })
     const data = JSON.parse(body)
     return data
   } catch (e) {
+    console.error("scrapeFacebook error:", e.message)
     return null
   }
 }
@@ -198,6 +201,7 @@ Provide concise, actionable sales advice. When asked about a specific job catego
   const ollamaRes = await fetch(`${OLLAMA_URL}/api/chat`, {
     method: "POST",
     headers: { "Content-Type": "application/json" },
+    signal: AbortSignal.timeout(60000),
     body: JSON.stringify({
       model: MODEL,
       messages: [
@@ -320,6 +324,7 @@ const server = http.createServer(async (req, res) => {
       try {
         await new Promise((resolve, reject) => {
           const r = http.get(`${SCRAPER_URL}/health`, { timeout: 3000 }, (res) => { res.resume(); resolve() })
+          r.on("timeout", () => { r.destroy(); reject(new Error("timeout")) })
           r.on("error", reject)
         })
         results.scraper = true
@@ -328,6 +333,7 @@ const server = http.createServer(async (req, res) => {
       try {
         await new Promise((resolve, reject) => {
           const r = http.get(FRONTEND_URL, { timeout: 3000 }, (res) => { res.resume(); resolve() })
+          r.on("timeout", () => { r.destroy(); reject(new Error("timeout")) })
           r.on("error", reject)
         })
         results.frontend = true
@@ -539,36 +545,27 @@ const server = http.createServer(async (req, res) => {
     // Accepts { message, user_id?, user_role? } and returns AI response.
     // user_role must be "sales", "admin", or "super_admin" if provided.
     if (req.method === "POST" && pathname === "/ai/chat") {
-      const startTime = Date.now()
       const chunks = []
       req.on("data", c => chunks.push(c))
-      req.on("end", () => {
+      req.on("end", async () => {
-        const rawBody = Buffer.concat(chunks).toString()
         try {
+          const rawBody = Buffer.concat(chunks).toString()
           const body = JSON.parse(rawBody)
-          processRequest(req, res, body, startTime)
-        } catch {
-          sendJSON(res, 400, { error: "Invalid JSON" })
-        }
-      })
-      return
-    }
-
-// Separate handler for /ai/chat (defined here due to hoisting within the IIFE)
-async function processRequest(req, res, body, startTime) {
           const { message, user_id, user_role } = body
-
           if (!message) {
             return sendJSON(res, 400, { error: "message is required" })
           }
-
           const validRoles = ["sales", "admin", "super_admin"]
           if (user_role && !validRoles.includes(user_role)) {
             return sendJSON(res, 403, { error: "Forbidden" })
           }
-
           const response = await handleChat(message, user_id || "", user_role || "sales")
-      return sendJSON(res, 200, { response })
+          sendJSON(res, 200, { response })
+        } catch (e) {
+          if (!res.headersSent) sendJSON(res, 500, { error: e.message })
+        }
+      })
+      return
     }
 
     // 404 fallback

browser-use-service/main.py

@@ -666,38 +666,24 @@ TUTORING_SEARCHES = [
     "tutor near me for",
 ]
 
-def _search_list_for_query(query: str) -> list[str]:
-    """Pick the appropriate search query pool based on the search term."""
-    tl = query.lower()
-    tutoring_terms = ["tutor", "tutoring", "lessons", "homework", "teach", "learning", "child", "math", "english", "science", "exam", "homeschool", "coding", "programming", "piano", "reading"]
-    if any(t in tl for t in tutoring_terms):
-        return TUTORING_SEARCHES
-    return FB_SEARCHES
-
 # ── South African Multi-Language Queries ──────────────────────────────
-# 4 most spoken SA languages: English, Afrikaans, isiXhosa, isiZulu.
+# 4 SA languages grouped for phase-based scanning:
-# Each scrape searches ALL 4 languages to catch leads across all
+# Phase 1: English → Phase 2: Afrikaans → Phase 3: isiXhosa → Phase 4: English (final sweep)
-# language communities on Facebook.
+# Each language group has dedicated queries per category.
 
-SA_WEBSITE_QUERIES = [
+SA_WEBSITE_QUERIES = {
-    "I need a website for my business",          # English
+    "english": ["I need a website for my business"],
-    "ek benodig n webwerf",                       # Afrikaans
+    "afrikaans": ["ek benodig n webwerf", "ek soek iemand om n webwerf te bou"],
-    "ek soek iemand om n webwerf te bou",         # Afrikaans
+    "xhosa": ["ndidinga iwebhusayithi yeshishini", "ndifuna umntu owakha iwebhusayithi"],
-    "ndidinga iwebhusayithi yeshishini",          # isiXhosa
+    "zulu": ["ngidinga iwebhusayithi yebhizinisi", "ngifuna umuntu owakha iwebhusayithi"],
-    "ndifuna umntu owakha iwebhusayithi",         # isiXhosa
+}
-    "ngidinga iwebhusayithi yebhizinisi",         # isiZulu
-    "ngifuna umuntu owakha iwebhusayithi",        # isiZulu
-]
 
-SA_TUTOR_QUERIES = [
+SA_TUTOR_QUERIES = {
-    "I need a tutor for my child",                # English
+    "english": ["I need a tutor for my child"],
-    "ek benodig n tutor vir my kind",             # Afrikaans
+    "afrikaans": ["ek benodig n tutor vir my kind", "ek soek n privaat onderwyser"],
-    "ek soek n privaat onderwyser",               # Afrikaans
+    "xhosa": ["ndifuna utitshala womntwana wam", "ndidinga umfundisi-ntsapho"],
-    "ndifuna utitshala womntwana wam",            # isiXhosa
+    "zulu": ["ngidinga uthisha wengane yami", "ngifuna umfundisi wengane"],
-    "ndidinga umfundisi-ntsapho",                 # isiXhosa
+}
-    "ngidinga uthisha wengane yami",              # isiZulu
-    "ngifuna umfundisi wengane",                  # isiZulu
-]
 
 
 async def _quick_search(page, context, query: str) -> tuple:
@@ -1361,6 +1347,94 @@ def cleanup_chrome():
         pass
 
 
+# ── 4-Phase Language Pipeline ─────────────────────────────────────────
+# Runs searches in ordered language phases:
+#   Phase 1: English (main query + supplementary EN searches)
+#   Phase 2: Afrikaans
+#   Phase 3: isiXhosa
+#   Phase 4: English (final sweep with different EN queries)
+# Each phase extracts posts and deduplicates on the fly.
+# Pipeline check (classify_leads) runs at end to quality-filter.
+
+PHASE_ORDER = ["english", "afrikaans", "xhosa", "zulu"]
+
+
+async def _run_phases(page, context, query: str | None = None) -> list[dict]:
+    """4-phase language pipeline: English → Afrikaans → Xhosa → Zulu.
+    Each phase finds leads in that language. Pipeline check at end filters + sorts by freshness.
+    Returns top 10 newest, most relevant leads."""
+    all_posts: list[dict] = []
+    seen_keys: set[str] = set()
+    tutoring = False
+
+    if query:
+        tl = query.lower()
+        tutoring = any(t in tl for t in ["tutor", "tutoring", "lessons", "homework", "teach", "learning", "child"])
+
+    lang_pool = SA_TUTOR_QUERIES if tutoring else SA_WEBSITE_QUERIES
+    en_pool = TUTORING_SEARCHES if tutoring else FB_SEARCHES
+
+    # Build phase query lists
+    supplement = random.sample(en_pool, k=min(3, len(en_pool))) if query else []
+    phase_queries: list[list[str]] = []
+
+    # Phase 1: English (user query + supplement from EN pool)
+    phase_queries.append(([query] + supplement) if query else random.sample(en_pool, k=min(4, len(en_pool))))
+
+    # Phase 2: Afrikaans
+    phase_queries.append(lang_pool.get("afrikaans", []))
+
+    # Phase 3: isiXhosa
+    phase_queries.append(lang_pool.get("xhosa", []))
+
+    # Phase 4: isiZulu
+    phase_queries.append(lang_pool.get("zulu", []))
+
+    # Execute phases
+    for phase_idx, queries in enumerate(phase_queries):
+        if not queries:
+            continue
+        phase_posts: list[dict] = []
+        for i, q in enumerate(queries):
+            is_first = (phase_idx == 0 and i == 0 and query is not None)
+            if is_first:
+                page, posts = await search_facebook(page, context, q)
+            else:
+                page, posts = await _quick_search(page, context, q)
+
+            for p in posts:
+                key = p.get('content', '')[:100]
+                if key and key not in seen_keys:
+                    seen_keys.add(key)
+                    phase_posts.append(p)
+
+        all_posts.extend(phase_posts)
+
+        # Stealth delay between phases (not after last)
+        if phase_idx < len(phase_queries) - 1 and phase_posts:
+            await page.wait_for_timeout(random.uniform(5000, 12000))
+            if random.random() < 0.2:
+                await random_idle(page)
+
+    # Pipeline check: date filter (2 days max) + AI/keyword classification
+    all_posts = [p for p in all_posts if _is_within_days(p.get('date', ''), 2)]
+
+    leads = all_posts[:20]
+    if leads:
+        leads = await classify_leads(leads, tutoring=tutoring)
+
+    # Sort by freshness — newest leads first
+    def _sort_key(l):
+        try:
+            return datetime.strptime((l.get('date') or '').strip()[:10], '%Y-%m-%d')
+        except (ValueError, IndexError):
+            return datetime.min
+
+    leads.sort(key=_sort_key, reverse=True)
+
+    return leads[:10]
+
+
 # ── Main Scrape Dispatcher ────────────────────────────────────────────
 # scrape_facebook() is the main entry point. It:
 #   1. Resolves the browser profile path (from SELECTED_BROWSER env var or auto-detect)
@@ -1415,17 +1489,17 @@ async def scrape_facebook(profile_path: str | None = None, force: bool = False,
     # Firefox path
     if browser_type == "firefox":
         result = await _scrape_with_firefox(effective_path, force, query)
-        if result.get("success") or not result.get("flagged"):
+        if result.get("success"):
             return result
-        logger.warning("Firefox flagged (%s), trying Agent", result.get("flag_reason", "unknown"))
+        logger.warning("Firefox failed (reason: %s), trying Agent", result.get("flag_reason") or result.get("error", "unknown"))
-        return await _scrape_with_agent(force)
+        return await _scrape_with_agent(force, query)
 
     # Chromium-based (chrome / opera / edge)
     result = await _scrape_with_chromium(effective_path, browser_type, force, query)
-    if result.get("success") or not result.get("flagged"):
+    if result.get("success"):
         return result
-    logger.warning("%s flagged (%s), trying Agent", browser_type, result.get("flag_reason", "unknown"))
+    logger.warning("%s failed (reason: %s), trying Agent", browser_type, result.get("flag_reason") or result.get("error", "unknown"))
-    return await _scrape_with_agent(force)
+    return await _scrape_with_agent(force, query)
 
 
 # ── Firefox Scraper ──────────────────────────────────────────────────
@@ -1468,6 +1542,7 @@ async def _scrape_with_firefox(profile_path: str, force: bool, query: str | None
             except Exception:
                 logger.warning("Google navigation failed, trying Facebook directly")
 
+            try:
                 await page.goto('https://www.facebook.com/', wait_until='domcontentloaded', timeout=30000)
                 await page.wait_for_timeout(random.randint(3000, 8000))
 
@@ -1476,7 +1551,6 @@ async def _scrape_with_firefox(profile_path: str, force: bool, query: str | None
                 det = check_detection_signals(url, page_text)
                 if det or '/login' in url.lower():
                     logger.warning("Facebook login page detected — flag: %s", det or "login_page")
-                await context.close()
                     return {"success": False, "leads": [], "flagged": True, "flag_reason": det or "login_page", "error": "Facebook login page detected"}
 
                 await human_scroll(page, steps=random.randint(2, 4), total_delay=random.uniform(8, 20))
@@ -1487,68 +1561,16 @@ async def _scrape_with_firefox(profile_path: str, force: bool, query: str | None
 
                 if not force and random.random() < 0.3:
                     await page.wait_for_timeout(random.randint(8000, 20000))
-                await context.close()
                     return {"success": True, "leads": [], "flagged": False, "flag_reason": None, "error": None}
 
-            all_posts = []
+                leads = await _run_phases(page, context, query)
-            tutoring = False
-            if query:
-                tl = query.lower()
-                tutoring = any(t in tl for t in ["tutor", "tutoring", "lessons", "homework", "teach", "learning", "child"])
-                lang_pool = SA_TUTOR_QUERIES if tutoring else SA_WEBSITE_QUERIES
-                non_english = [q for q in lang_pool if q.strip().lower() != tl]
-                query_pool = _search_list_for_query(query)
-                supp_k = random.randint(3, 4) if tutoring else random.randint(2, 3)
-                supplement = random.sample(query_pool, k=supp_k)
-                searches = [query] + supplement + non_english
-                english_count = 1 + len(supplement)
-            else:
-                searches = random.sample(FB_SEARCHES + SA_WEBSITE_QUERIES + SA_TUTOR_QUERIES, k=random.randint(4, 7))
-            for i, sq in enumerate(searches):
-                page, posts = await search_facebook(page, context, sq) if i == 0 else await _quick_search(page, context, sq)
-                all_posts.extend(posts)
-                if not posts:
-                    continue
-                if i > 0:
-                    await page.wait_for_timeout(random.uniform(5000, 12000))
-                    if random.random() < 0.2:
-                        await random_idle(page)
-                    continue
-                if random.random() < 0.4:
-                    await page.evaluate(f"window.scrollBy(0, {random.randint(-300, 300)})")
-                delay = random.uniform(8, 25)
-                await page.wait_for_timeout(int(delay * 1000))
-                if i == random.randint(0, 1) and random.random() < 0.15:
-                    new_page = await context.new_page()
-                    try:
-                        await new_page.goto('https://www.facebook.com/groups/', wait_until='domcontentloaded', timeout=15000)
-                        await new_page.wait_for_timeout(random.randint(3000, 8000))
-                    except Exception:
-                        pass
-                    await new_page.close()
-                    page = await _ensure_page(page, context)
-
-            if random.random() < 0.5:
-                await page.wait_for_timeout(random.randint(3000, 10000))
-
-            await context.close()
-
-            seen = set()
-            deduped = []
-            for p in all_posts:
-                key = p.get('content', '')[:100]
-                if key not in seen:
-                    seen.add(key)
-                    deduped.append(p)
-
-            # Filter to last 2 days only
-            deduped = [p for p in deduped if _is_within_days(p.get('date', ''), 2)]
-
-            leads = deduped[:20]
-            if leads:
-                leads = await classify_leads(leads, tutoring=tutoring)
 
                 return {"success": True, "leads": leads[:15], "flagged": False, "flag_reason": None, "error": None}
+            finally:
+                try:
+                    await context.close()
+                except Exception:
+                    pass
 
     except Exception as e:
         logger.error("Firefox scrape failed: %s", e)
@@ -1617,6 +1639,7 @@ async def _scrape_with_chromium(profile_path: str, browser: str, force: bool = F
             except Exception:
                 logger.warning("Google navigation failed, trying Facebook directly")
 
+            try:
                 await page.goto('https://www.facebook.com/', wait_until='domcontentloaded', timeout=30000)
                 await page.wait_for_timeout(random.randint(3000, 8000))
 
@@ -1625,7 +1648,6 @@ async def _scrape_with_chromium(profile_path: str, browser: str, force: bool = F
                 det = check_detection_signals(url, page_text)
                 if det or '/login' in url.lower():
                     logger.warning("Facebook login page detected — flag: %s", det or "login_page")
-                await context.close()
                     return {"success": False, "leads": [], "flagged": True, "flag_reason": det or "login_page", "error": "Facebook login page detected"}
 
                 await human_scroll(page, steps=random.randint(2, 4), total_delay=random.uniform(8, 20))
@@ -1636,66 +1658,16 @@ async def _scrape_with_chromium(profile_path: str, browser: str, force: bool = F
 
                 if not force and random.random() < 0.3:
                     await page.wait_for_timeout(random.randint(8000, 20000))
-                await context.close()
                     return {"success": True, "leads": [], "flagged": False, "flag_reason": None, "error": None}
 
-            all_posts = []
+                leads = await _run_phases(page, context, query)
-            tutoring = False
-            if query:
-                tl = query.lower()
-                tutoring = any(t in tl for t in ["tutor", "tutoring", "lessons", "homework", "teach", "learning", "child"])
-                lang_pool = SA_TUTOR_QUERIES if tutoring else SA_WEBSITE_QUERIES
-                non_english = [q for q in lang_pool if q.strip().lower() != tl]
-                query_pool = _search_list_for_query(query)
-                supp_k = random.randint(3, 4) if tutoring else random.randint(2, 3)
-                supplement = random.sample(query_pool, k=supp_k)
-                searches = [query] + supplement + non_english
-                english_count = 1 + len(supplement)
-            else:
-                searches = random.sample(FB_SEARCHES + SA_WEBSITE_QUERIES + SA_TUTOR_QUERIES, k=random.randint(4, 7))
-            for i, sq in enumerate(searches):
-                page, posts = await search_facebook(page, context, sq) if i == 0 else await _quick_search(page, context, sq)
-                all_posts.extend(posts)
-                if not posts:
-                    continue
-                if i > 0:
-                    await page.wait_for_timeout(random.uniform(5000, 12000))
-                    if random.random() < 0.2:
-                        await random_idle(page)
-                    continue
-                if random.random() < 0.4:
-                    await page.evaluate(f"window.scrollBy(0, {random.randint(-300, 300)})")
-                delay = random.uniform(8, 25)
-                await page.wait_for_timeout(int(delay * 1000))
-                if i == random.randint(0, 1) and random.random() < 0.15:
-                    new_page = await context.new_page()
-                    try:
-                        await new_page.goto('https://www.facebook.com/groups/', wait_until='domcontentloaded', timeout=15000)
-                        await new_page.wait_for_timeout(random.randint(3000, 8000))
-                    except Exception:
-                        pass
-                    await new_page.close()
-                    page = await _ensure_page(page, context)
-
-            if random.random() < 0.5:
-                await page.wait_for_timeout(random.randint(3000, 10000))
-
-            await context.close()
-
-            seen = set()
-            deduped = []
-            for p in all_posts:
-                key = p.get('content', '')[:100]
-                if key not in seen:
-                    seen.add(key)
-                    deduped.append(p)
-
-            deduped = [p for p in deduped if _is_within_days(p.get('date', ''), 2)]
-            leads = deduped[:20]
-            if leads:
-                leads = await classify_leads(leads, tutoring=tutoring)
 
                 return {"success": True, "leads": leads[:15], "flagged": False, "flag_reason": None, "error": None}
+            finally:
+                try:
+                    await context.close()
+                except Exception:
+                    pass
 
     except Exception as e:
         logger.error("%s scrape failed: %s", browser, e)
@@ -1715,7 +1687,7 @@ async def _scrape_with_chromium(profile_path: str, browser: str, force: bool = F
 # Uses Chromium headless with the same launch args as _scrape_with_chromium.
 # The Agent is prompted to extract structured post data and return JSON.
 
-async def _scrape_with_agent(force: bool = False) -> dict:
+async def _scrape_with_agent(force: bool = False, query: str | None = None) -> dict:
     """Fallback scraper — browser-use Agent + ChatOllama (free/local, Chromium)."""
     cleanup_chrome()
     profile_dir = None
@@ -1734,7 +1706,13 @@ async def _scrape_with_agent(force: bool = False) -> dict:
         await browser.start()
 
         all_posts = []
-        pool = FB_SEARCHES + random.sample(SA_WEBSITE_QUERIES, k=min(4, len(SA_WEBSITE_QUERIES)))
+        tutoring_agent = False
+        if query:
+            tl = query.lower()
+            tutoring_agent = any(t in tl for t in ["tutor", "tutoring", "lessons", "homework", "teach", "learning", "child"])
+        sa_dict = SA_TUTOR_QUERIES if tutoring_agent else SA_WEBSITE_QUERIES
+        sa_all = sa_dict.get("afrikaans", []) + sa_dict.get("xhosa", []) + sa_dict.get("zulu", [])
+        pool = FB_SEARCHES + sa_all
         for query in random.sample(pool, k=random.randint(2, 4)):
             agent = _make_agent(
                 task=f"""You are logged into Facebook. Do the following:
@@ -1780,7 +1758,7 @@ When done, return the data as a JSON list with keys: content, author, url, date.
 
         leads = deduped[:20]
         if leads:
-            leads = await classify_leads(leads, tutoring=tutoring)
+            leads = await classify_leads(leads, tutoring=tutoring_agent)
 
         return {"success": True, "leads": leads[:15], "flagged": False, "flag_reason": None, "error": None}
     except Exception as e:
@@ -1820,7 +1798,7 @@ async def classify_leads(results: list[dict], tutoring: bool = False) -> list[di
         return []
 
     # ── 1. AI classification ─────────────────────────────────────────
-    briefs = [r["title"][:200] for r in results]
+    briefs = [(r.get("title") or r.get("content") or "")[:200] for r in results]
     if tutoring:
         lead_desc = "someone REQUESTING/LOOKING FOR/WANTING a tutor, teacher, or lessons for their child or themselves"
         lead_examples = '"Looking for a tutor for my child", "Need a math tutor for my son", "Need help with homework", "Looking for piano lessons for my daughter", "Need a reading tutor"'
@@ -2021,7 +1999,7 @@ Return a JSON array like ["yes","no","yes"] matching the order above."""
         'website design ideas', 'website inspiration',
     ]
     for r in results:
-        t = r['title'].lower()
+        t = (r.get('title') or r.get('content') or '').lower()
         has_target = any(kw in t for kw in target_terms)
         has_request = any(kw in t for kw in request_terms)
         if not has_target or not has_request:
@@ -2033,11 +2011,11 @@ Return a JSON array like ["yes","no","yes"] matching the order above."""
         keyword_leads.append(r)
 
     # ── 3. Merge: prefer AI leads, supplement with keywords ──
-    seen_titles: set[int] = set()
+    seen_titles: set[str] = set()
     merged: list[dict] = []
     for r in ai_leads + keyword_leads:
-        key = hash(r.get('title', ''))
+        key = (r.get('title') or '').strip()[:200]
-        if key not in seen_titles:
+        if key and key not in seen_titles:
             seen_titles.add(key)
             merged.append(r)
     # Final sweep: strip any remaining offers or group posts from merged

data/ai/ai.md

@@ -34,12 +34,14 @@ The scraper lives at `browser-use-service/main.py` port 3008.
 ### How It Works
 1. **Browser detection** — tries Firefox profile first, then Chromium-based (Chrome/Opera/Edge), falls back to browser-use Agent
 2. **Profile paths** — configured via env vars (`FX_PROFILE`, `CHROME_PROFILE`, `OPERA_PROFILE`, `EDGE_PROFILE`) or auto-detected on first run
-3. **Search dispatch** — per scrape run:
+3. **4-phase language pipeline** (English → Afrikaans → Xhosa → Zulu):
-   - 1 English primary search (full scroll with human-like delays)
+   - **Phase 1 (English)**: User's selected query + 2-3 supplementary English searches from the English search pool. First query gets full human-like scroll, rest use quick search. This phase does the heavy lifting.
-   - 2-3 English supplementary searches (quick searches)
+   - **Phase 2 (Afrikaans)**: 2 Afrikaans queries targeting Afrikaans-speaking communities.
-   - 6-7 non-English quick searches (Afrikaans, isiXhosa, isiZulu — 2 queries each per category)
+   - **Phase 3 (isiXhosa)**: 2 Xhosa queries targeting Xhosa-speaking communities.
-   - Total: ~14 searches per scrape, completed in 2-4 minutes
+   - **Phase 4 (isiZulu)**: 2 Zulu queries targeting Zulu-speaking communities.
-4. **Quick searches** — load page, double-scroll, extract visible posts (~12-18s each)
+   - After all phases: pipeline check (date filter 2 days → AI + keyword classification → sort by freshness). Newest leads ranked first.
+   - Each phase extracts posts, deduplicates against all prior phases, then passes through a stealth delay (5-12s + mouse idle) before the next phase.
+4. **Quick searches** — load page, double-scroll, extract visible posts (~12-18s each). Scroll-back behavior (35% chance to scroll up) and random return-to-top (25% chance) for stealth.
 5. **Date filter** — only posts within **2 days** are considered. Anything older is discarded. Fresh leads only.
 6. **Stealth mechanics**:
    - Random viewport dimensions (1280×800 to 1920×1080) — never the same size twice
@@ -70,12 +72,12 @@ Two categories, selectable when starting a scrape:
 - Request terms: same as website category — must co-occur with a target keyword
 - Strict reject: people offering tutoring, educational products, homeschool programs, free trials, general study tips
 
-### Multi-Language Support
+### Multi-Language Pipeline (Phase Order)
-Searches in 4 South African languages:
+4 South African languages in structured phases:
-- English — 1 primary + 2-3 supplementary queries
+- **Phase 1 (English)**: primary query + supplementary English searches
-- Afrikaans — 2 queries (e.g., "ek benodig n webwerf", "ek soek n privaat onderwyser")
+- **Phase 2 (Afrikaans)**: 2 queries targeting Afrikaans speakers
-- isiXhosa — 2 queries (e.g., "ndidinga iwebhusayithi yeshishini", "ndifuna utitshala womntwana wam")
+- **Phase 3 (isiXhosa)**: 2 queries targeting Xhosa speakers
-- isiZulu — 2 queries (e.g., "ngidinga iwebhusayithi yebhizinisi", "ngifuna umfundisi wengane")
+- **Phase 4 (isiZulu)**: 2 queries targeting Zulu speakers
 
 ### Output Format
 Each lead returned includes:

rust-ai/src/AI.md

@@ -34,12 +34,14 @@ The scraper lives at `browser-use-service/main.py` port 3008.
 ### How It Works
 1. **Browser detection** — tries Firefox profile first, then Chromium-based (Chrome/Opera/Edge), falls back to browser-use Agent
 2. **Profile paths** — configured via env vars (`FX_PROFILE`, `CHROME_PROFILE`, `OPERA_PROFILE`, `EDGE_PROFILE`) or auto-detected on first run
-3. **Search dispatch** — per scrape run:
+3. **4-phase language pipeline** (English → Afrikaans → Xhosa → Zulu):
-   - 1 English primary search (full scroll with human-like delays)
+   - **Phase 1 (English)**: User's selected query + 2-3 supplementary English searches from the English search pool. First query gets full human-like scroll, rest use quick search. This phase does the heavy lifting.
-   - 2-3 English supplementary searches (quick searches)
+   - **Phase 2 (Afrikaans)**: 2 Afrikaans queries targeting Afrikaans-speaking communities.
-   - 6-7 non-English quick searches (Afrikaans, isiXhosa, isiZulu — 2 queries each per category)
+   - **Phase 3 (isiXhosa)**: 2 Xhosa queries targeting Xhosa-speaking communities.
-   - Total: ~14 searches per scrape, completed in 2-4 minutes
+   - **Phase 4 (isiZulu)**: 2 Zulu queries targeting Zulu-speaking communities.
-4. **Quick searches** — load page, double-scroll, extract visible posts (~12-18s each)
+   - After all phases: pipeline check (date filter 2 days → AI + keyword classification → sort by freshness). Newest leads ranked first.
+   - Each phase extracts posts, deduplicates against all prior phases, then passes through a stealth delay (5-12s + mouse idle) before the next phase.
+4. **Quick searches** — load page, double-scroll, extract visible posts (~12-18s each). Scroll-back behavior (35% chance to scroll up) and random return-to-top (25% chance) for stealth.
 5. **Date filter** — only posts within **2 days** are considered. Anything older is discarded. Fresh leads only.
 6. **Stealth mechanics**:
    - Random viewport dimensions (1280×800 to 1920×1080) — never the same size twice
@@ -70,12 +72,12 @@ Two categories, selectable when starting a scrape:
 - Request terms: same as website category — must co-occur with a target keyword
 - Strict reject: people offering tutoring, educational products, homeschool programs, free trials, general study tips
 
-### Multi-Language Support
+### Multi-Language Pipeline (Phase Order)
-Searches in 4 South African languages:
+4 South African languages in structured phases:
-- English — 1 primary + 2-3 supplementary queries
+- **Phase 1 (English)**: primary query + supplementary English searches
-- Afrikaans — 2 queries (e.g., "ek benodig n webwerf", "ek soek n privaat onderwyser")
+- **Phase 2 (Afrikaans)**: 2 queries targeting Afrikaans speakers
-- isiXhosa — 2 queries (e.g., "ndidinga iwebhusayithi yeshishini", "ndifuna utitshala womntwana wam")
+- **Phase 3 (isiXhosa)**: 2 queries targeting Xhosa speakers
-- isiZulu — 2 queries (e.g., "ngidinga iwebhusayithi yebhizinisi", "ngifuna umfundisi wengane")
+- **Phase 4 (isiZulu)**: 2 queries targeting Zulu speakers
 
 ### Output Format
 Each lead returned includes:

src/app/(dashboard)/ai-assistant/page.tsx

@@ -17,7 +17,7 @@ export default function AIAssistantPage() {
 
   const handleSearch = useCallback(async (job: NonNullable<typeof selectedJob>) => {
     setSearching(true)
-    const keyword = job.keywords[0]
+    const keyword = job.keywords?.[0] || job.job_title
     aiChatRef.current?.addAssistantMessage(`🔍 Searching Facebook for **${job.job_title}** leads...`)
 
     const controller = new AbortController()
@@ -34,9 +34,12 @@ export default function AIAssistantPage() {
       clearTimeout(statusId)
       const data = await res.json()
       if (data.success && data.leads?.length > 0) {
-        const leadsText = data.leads.map((lead: any, i: number) =>
+        const leadLines = data.leads
-          `**${i + 1}.** ${lead.author || "Unknown"}\n> ${(lead.content || "").slice(0, 300)}\n> 🔗 ${lead.url || "(no link available)"}`
+          .filter(Boolean)
-        ).join("\n\n")
+          .map((lead: Record<string, string>, i: number) =>
+            `**${i + 1}.** ${lead?.author || "Unknown"}\n> ${(lead?.content || "").slice(0, 300)}\n> 🔗 ${lead?.url || "(no link available)"}`
+          )
+        const leadsText = leadLines.join("\n\n")
         aiChatRef.current?.addAssistantMessage(`✅ Found **${data.leads.length}** leads:\n\n${leadsText}`)
       } else {
         const reason = data.error || data.flag_reason || "No leads found this time"

src/components/ai/job-selector.tsx

@@ -62,7 +62,7 @@ export function JobSelector({ onSelect, onSearch, searching }: JobSelectorProps)
                 className="w-full text-left px-4 py-3 text-sm text-muted-foreground hover:bg-muted hover:text-foreground transition-all duration-150 border-b border-border/20 last:border-b-0 border-l-2 border-l-transparent hover:border-l-primary/40"
               >
                 <div className="font-medium">{job.job_title}</div>
-                <div className="text-xs text-muted-foreground/60 mt-0.5">{job.industry} &mdash; {job.description}</div>
+                <div className="text-xs text-muted-foreground/60 mt-0.5">{job.industry} — {job.description}</div>
               </button>
             ))}
             {jobs.length === 0 && !loading && (