Merge branch 'main' of https://git.coastit.co.za/caitlin/CRM_ENVR

ai-server/index.mjs

@@ -12,6 +12,7 @@ import http from "node:http"
 import fs from "node:fs"
 import path from "node:path"
 import { spawn } from "node:child_process"
+import crypto from "node:crypto"
 import { fileURLToPath } from "node:url"
 
 const __dirname = path.dirname(fileURLToPath(import.meta.url))
@@ -44,6 +45,8 @@ const PORT = parseInt(process.env.AI_PORT || "3001", 10)
 const HOST = process.env.AI_HOST || "0.0.0.0"
 const OLLAMA_URL = process.env.OLLAMA_BASE_URL || "http://localhost:11434"
 const MODEL = process.env.AI_MODEL || "llama3.2:3b"
+const SCRAPER_URL = process.env.SCRAPER_URL || "http://127.0.0.1:3008"
+const FRONTEND_URL = process.env.FRONTEND_URL || "http://127.0.0.1:3006"
 const DATABASE_URL = process.env.DATABASE_URL
 const JOBS_PATH = process.env.JOBS_PATH || path.join(ROOT, "data", "ai", "jobs.jsonl")
 const AI_MD_PATH = process.env.AI_MD_PATH || path.join(ROOT, "data", "ai", "ai.md")
@@ -130,19 +133,22 @@ async function scrapeFacebook() {
   const urlPath = `/scrape/facebook?force=true${profilePath ? `&profile_path=${encodeURIComponent(profilePath)}` : ""}`
   try {
     const body = await new Promise((resolve, reject) => {
-      const req = http.request({ hostname: "127.0.0.1", port: 3008, path: urlPath, method: "POST", timeout: 360000 }, (res) => {
+      const parsed = new URL(SCRAPER_URL)
+      let done = false
+      const req = http.request({ hostname: parsed.hostname, port: parsed.port || 3008, path: urlPath, method: "POST", timeout: 60000 }, (res) => {
         let data = ""
         res.on("data", (c) => data += c)
-        res.on("end", () => resolve(data))
-        res.on("error", reject)
+        res.on("end", () => { done = true; resolve(data) })
+        res.on("error", (e) => { if (!done) { done = true; reject(e) } })
       })
-      req.on("timeout", () => { req.destroy(); reject(new Error("timeout")) })
-      req.on("error", reject)
+      req.on("timeout", () => { if (!done) { done = true; req.destroy(); reject(new Error("scraper timeout")) } })
+      req.on("error", (e) => { if (!done) { done = true; reject(e) } })
       req.end()
     })
     const data = JSON.parse(body)
     return data
   } catch (e) {
+    console.error("scrapeFacebook error:", e.message)
     return null
   }
 }
@@ -195,6 +201,7 @@ Provide concise, actionable sales advice. When asked about a specific job catego
   const ollamaRes = await fetch(`${OLLAMA_URL}/api/chat`, {
     method: "POST",
     headers: { "Content-Type": "application/json" },
+    signal: AbortSignal.timeout(60000),
     body: JSON.stringify({
       model: MODEL,
       messages: [
@@ -313,18 +320,20 @@ const server = http.createServer(async (req, res) => {
     if (req.method === "GET" && pathname === "/status") {
       const { default: http } = await import("http")
       const results = { ai: true }
-      // Check scraper (port 3008)
+      // Check scraper
       try {
         await new Promise((resolve, reject) => {
-          const r = http.get("http://127.0.0.1:3008/health", { timeout: 3000 }, (res) => { res.resume(); resolve() })
+          const r = http.get(`${SCRAPER_URL}/health`, { timeout: 3000 }, (res) => { res.resume(); resolve() })
+          r.on("timeout", () => { r.destroy(); reject(new Error("timeout")) })
           r.on("error", reject)
         })
         results.scraper = true
       } catch { results.scraper = false }
-      // Check frontend (port 3006)
+      // Check frontend
       try {
         await new Promise((resolve, reject) => {
-          const r = http.get("http://127.0.0.1:3006", { timeout: 3000 }, (res) => { res.resume(); resolve() })
+          const r = http.get(FRONTEND_URL, { timeout: 3000 }, (res) => { res.resume(); resolve() })
+          r.on("timeout", () => { r.destroy(); reject(new Error("timeout")) })
           r.on("error", reject)
         })
         results.frontend = true
@@ -368,8 +377,8 @@ const server = http.createServer(async (req, res) => {
       let selectedBrowser = process.env.SELECTED_BROWSER || ""
 
       try {
-        await fetch("http://127.0.0.1:3008/health", { signal: AbortSignal.timeout(2000) })
-        const profiles = await (await fetch("http://127.0.0.1:3008/setup/profile", { signal: AbortSignal.timeout(5000) })).json()
+        await fetch(`${SCRAPER_URL}/health`, { signal: AbortSignal.timeout(2000) })
+        const profiles = await (await fetch(`${SCRAPER_URL}/setup/profile`, { signal: AbortSignal.timeout(5000) })).json()
         for (const [b, p] of Object.entries(profiles)) {
           if (p) browsers[b] = { path: p }
         }
@@ -377,7 +386,7 @@ const server = http.createServer(async (req, res) => {
         const detectedList = Object.entries(browsers).filter(([, v]) => v.path)
         for (const [b, v] of detectedList) {
           try {
-            const r = await fetch("http://127.0.0.1:3008/setup/check-login", {
+            const r = await fetch(`${SCRAPER_URL}/setup/check-login`, {
               method: "POST", headers: { "Content-Type": "application/json" },
               body: JSON.stringify({ browser: b, profile_path: v.path }),
               signal: AbortSignal.timeout(20000),
@@ -536,36 +545,27 @@ const server = http.createServer(async (req, res) => {
     // Accepts { message, user_id?, user_role? } and returns AI response.
     // user_role must be "sales", "admin", or "super_admin" if provided.
     if (req.method === "POST" && pathname === "/ai/chat") {
-      const startTime = Date.now()
       const chunks = []
       req.on("data", c => chunks.push(c))
-      req.on("end", () => {
-        const rawBody = Buffer.concat(chunks).toString()
+      req.on("end", async () => {
         try {
+          const rawBody = Buffer.concat(chunks).toString()
           const body = JSON.parse(rawBody)
-          processRequest(req, res, body, startTime)
-        } catch {
-          sendJSON(res, 400, { error: "Invalid JSON" })
-        }
-      })
-      return
-    }
-
-// Separate handler for /ai/chat (defined here due to hoisting within the IIFE)
-async function processRequest(req, res, body, startTime) {
           const { message, user_id, user_role } = body
-
           if (!message) {
             return sendJSON(res, 400, { error: "message is required" })
           }
-
           const validRoles = ["sales", "admin", "super_admin"]
           if (user_role && !validRoles.includes(user_role)) {
             return sendJSON(res, 403, { error: "Forbidden" })
           }
-
           const response = await handleChat(message, user_id || "", user_role || "sales")
-      return sendJSON(res, 200, { response })
+          sendJSON(res, 200, { response })
+        } catch (e) {
+          if (!res.headersSent) sendJSON(res, 500, { error: e.message })
+        }
+      })
+      return
     }
 
     // 404 fallback

browser-use-service/main.py

@@ -34,7 +34,7 @@ logger = logging.getLogger(__name__)
 app = FastAPI()
 app.add_middleware(
     CORSMiddleware,
-    allow_origins=["http://localhost:3006", "http://127.0.0.1:3006"],
+    allow_origins=os.getenv("CORS_ORIGINS", "http://localhost:3006,http://127.0.0.1:3006").split(","),
     allow_methods=["POST"],
     allow_headers=["*"],
 )
@@ -666,13 +666,106 @@ TUTORING_SEARCHES = [
     "tutor near me for",
 ]
 
-def _search_list_for_query(query: str) -> list[str]:
-    """Pick the appropriate search query pool based on the search term."""
-    tl = query.lower()
-    tutoring_terms = ["tutor", "tutoring", "lessons", "homework", "teach", "learning", "child", "math", "english", "science", "exam", "homeschool", "coding", "programming", "piano", "reading"]
-    if any(t in tl for t in tutoring_terms):
-        return TUTORING_SEARCHES
-    return FB_SEARCHES
+# ── South African Multi-Language Queries ──────────────────────────────
+# 4 SA languages grouped for phase-based scanning:
+# Phase 1: English → Phase 2: Afrikaans → Phase 3: isiXhosa → Phase 4: English (final sweep)
+# Each language group has dedicated queries per category.
+
+SA_WEBSITE_QUERIES = {
+    "english": ["I need a website for my business"],
+    "afrikaans": ["ek benodig n webwerf", "ek soek iemand om n webwerf te bou"],
+    "xhosa": ["ndidinga iwebhusayithi yeshishini", "ndifuna umntu owakha iwebhusayithi"],
+    "zulu": ["ngidinga iwebhusayithi yebhizinisi", "ngifuna umuntu owakha iwebhusayithi"],
+}
+
+SA_TUTOR_QUERIES = {
+    "english": ["I need a tutor for my child"],
+    "afrikaans": ["ek benodig n tutor vir my kind", "ek soek n privaat onderwyser"],
+    "xhosa": ["ndifuna utitshala womntwana wam", "ndidinga umfundisi-ntsapho"],
+    "zulu": ["ngidinga uthisha wengane yami", "ngifuna umfundisi wengane"],
+}
+
+
+async def _quick_search(page, context, query: str) -> tuple:
+    """Fast search — load search results page, wait for render, extract visible posts.
+    No scrolling or extra human-like delays. Used for non-English language queries."""
+    page = await _ensure_page(page, context)
+    url = f'https://www.facebook.com/search/posts/?q={urllib.parse.quote(query)}'
+    try:
+        await page.goto(url, wait_until='domcontentloaded', timeout=20000)
+        current_url = page.url
+        if '/login' in current_url.lower():
+            logger.warning("Quick search redirected to login for '%s'", query[:40])
+            return page, []
+        await page.wait_for_timeout(random.randint(3000, 7000))
+        await page.evaluate(f"window.scrollBy(0, {random.randint(400, 900)})")
+        await page.wait_for_timeout(random.randint(2000, 5000))
+        if random.random() < 0.35:
+            await page.evaluate(f"window.scrollBy(0, -{random.randint(100, 400)})")
+            await page.wait_for_timeout(random.randint(1500, 3500))
+        await page.evaluate(f"window.scrollBy(0, {random.randint(400, 900)})")
+        await page.wait_for_timeout(random.randint(2000, 5000))
+        if random.random() < 0.25:
+            await page.evaluate("window.scrollTo(0, 0)")
+            await page.wait_for_timeout(random.randint(1000, 3000))
+        raw_articles = await _get_article_elements(page)
+        posts = _extract_posts_from_elements(raw_articles, url) if raw_articles else []
+        raw = await page.evaluate('document.body.innerText')
+        text_posts = _extract_posts_from_text(raw, url)
+        existing = {(p.get('title') or p.get('content',''))[:80] for p in posts}
+        for tp in text_posts:
+            key = (tp.get('title') or tp.get('content',''))[:80]
+            if key not in existing:
+                posts.append(tp)
+        if posts:
+            try:
+                profiles = await page.evaluate(r'''() => {
+                    const out = [];
+                    const seenTxt = new Set();
+                    for (const a of document.querySelectorAll('a[href*="/profile.php"], a[href*="/user/"], a[href*="/people/"], a[href*="/me/"]')) {
+                        const name = (a.innerText || '').trim();
+                        if (!name || name.length < 3 || name.length > 60) continue;
+                        const words = name.split(' ');
+                        if (words.length < 2 || words.length > 6) continue;
+                        if (!/^[A-Z]/.test(name)) continue;
+                        if (name.includes('facebook') || name.includes('/')) continue;
+                        const cell = a.closest('div[style]') || a.parentElement;
+                        const txt = cell ? (cell.innerText || '').substring(0, 200) : '';
+                        if (!txt) continue;
+                        const key2 = txt.substring(0, 80);
+                        if (seenTxt.has(key2)) continue;
+                        seenTxt.add(key2);
+                        out.push({ name, textKey: key2 });
+                    }
+                    return out;
+                }''')
+                if profiles:
+                    for p in posts:
+                        pk = (p.get('content') or p.get('title') or '')[:80].strip()
+                        if not pk: continue
+                        for pr in profiles:
+                            if pk[:30] in pr['textKey'] or pr['textKey'][:30] in pk:
+                                if not p.get('author'):
+                                    p['author'] = pr['name']
+                                break
+            except Exception:
+                pass
+        for p in posts:
+            if p.get('author'):
+                a = p['author']
+                al = a.lower()
+                if any(kw in al for kw in BROAD_KEYWORDS) or is_offer(a) or len(a.split()) < 2 or any(w in _NON_NAMES for w in al.split()):
+                    p['author'] = ''
+        posts = [p for p in posts if not (
+            '/groups/' in p.get('url', '') or '/group/' in p.get('url', '')
+            or '/pages/' in p.get('url', '')
+            or ' / ' in (p.get('title') or p.get('content') or '')
+        )]
+    except Exception as e:
+        logger.warning("Quick search '%s' failed: %s", query, e)
+        return page, []
+    return page, posts
+
 
 VIEWPORTS = [
     {'width': 1280, 'height': 800},
@@ -764,7 +857,7 @@ def _parse_fb_date(block: list[str]) -> str:
     return datetime.now().strftime('%Y-%m-%d')
 
 
-def _is_within_days(date_str: str, max_days: int = 3) -> bool:
+def _is_within_days(date_str: str, max_days: int = 2) -> bool:
     """Check if date is within max_days from now. Empty/unparseable = keep."""
     if not date_str:
         return True
@@ -1254,6 +1347,94 @@ def cleanup_chrome():
         pass
 
 
+# ── 4-Phase Language Pipeline ─────────────────────────────────────────
+# Runs searches in ordered language phases:
+#   Phase 1: English (main query + supplementary EN searches)
+#   Phase 2: Afrikaans
+#   Phase 3: isiXhosa
+#   Phase 4: English (final sweep with different EN queries)
+# Each phase extracts posts and deduplicates on the fly.
+# Pipeline check (classify_leads) runs at end to quality-filter.
+
+PHASE_ORDER = ["english", "afrikaans", "xhosa", "zulu"]
+
+
+async def _run_phases(page, context, query: str | None = None) -> list[dict]:
+    """4-phase language pipeline: English → Afrikaans → Xhosa → Zulu.
+    Each phase finds leads in that language. Pipeline check at end filters + sorts by freshness.
+    Returns top 10 newest, most relevant leads."""
+    all_posts: list[dict] = []
+    seen_keys: set[str] = set()
+    tutoring = False
+
+    if query:
+        tl = query.lower()
+        tutoring = any(t in tl for t in ["tutor", "tutoring", "lessons", "homework", "teach", "learning", "child"])
+
+    lang_pool = SA_TUTOR_QUERIES if tutoring else SA_WEBSITE_QUERIES
+    en_pool = TUTORING_SEARCHES if tutoring else FB_SEARCHES
+
+    # Build phase query lists
+    supplement = random.sample(en_pool, k=min(3, len(en_pool))) if query else []
+    phase_queries: list[list[str]] = []
+
+    # Phase 1: English (user query + supplement from EN pool)
+    phase_queries.append(([query] + supplement) if query else random.sample(en_pool, k=min(4, len(en_pool))))
+
+    # Phase 2: Afrikaans
+    phase_queries.append(lang_pool.get("afrikaans", []))
+
+    # Phase 3: isiXhosa
+    phase_queries.append(lang_pool.get("xhosa", []))
+
+    # Phase 4: isiZulu
+    phase_queries.append(lang_pool.get("zulu", []))
+
+    # Execute phases
+    for phase_idx, queries in enumerate(phase_queries):
+        if not queries:
+            continue
+        phase_posts: list[dict] = []
+        for i, q in enumerate(queries):
+            is_first = (phase_idx == 0 and i == 0 and query is not None)
+            if is_first:
+                page, posts = await search_facebook(page, context, q)
+            else:
+                page, posts = await _quick_search(page, context, q)
+
+            for p in posts:
+                key = p.get('content', '')[:100]
+                if key and key not in seen_keys:
+                    seen_keys.add(key)
+                    phase_posts.append(p)
+
+        all_posts.extend(phase_posts)
+
+        # Stealth delay between phases (not after last)
+        if phase_idx < len(phase_queries) - 1 and phase_posts:
+            await page.wait_for_timeout(random.uniform(5000, 12000))
+            if random.random() < 0.2:
+                await random_idle(page)
+
+    # Pipeline check: date filter (2 days max) + AI/keyword classification
+    all_posts = [p for p in all_posts if _is_within_days(p.get('date', ''), 2)]
+
+    leads = all_posts[:20]
+    if leads:
+        leads = await classify_leads(leads, tutoring=tutoring)
+
+    # Sort by freshness — newest leads first
+    def _sort_key(l):
+        try:
+            return datetime.strptime((l.get('date') or '').strip()[:10], '%Y-%m-%d')
+        except (ValueError, IndexError):
+            return datetime.min
+
+    leads.sort(key=_sort_key, reverse=True)
+
+    return leads[:10]
+
+
 # ── Main Scrape Dispatcher ────────────────────────────────────────────
 # scrape_facebook() is the main entry point. It:
 #   1. Resolves the browser profile path (from SELECTED_BROWSER env var or auto-detect)
@@ -1308,17 +1489,17 @@ async def scrape_facebook(profile_path: str | None = None, force: bool = False,
     # Firefox path
     if browser_type == "firefox":
         result = await _scrape_with_firefox(effective_path, force, query)
-        if result.get("success") or not result.get("flagged"):
+        if result.get("success"):
             return result
-        logger.warning("Firefox flagged (%s), trying Agent", result.get("flag_reason", "unknown"))
-        return await _scrape_with_agent(force)
+        logger.warning("Firefox failed (reason: %s), trying Agent", result.get("flag_reason") or result.get("error", "unknown"))
+        return await _scrape_with_agent(force, query)
 
     # Chromium-based (chrome / opera / edge)
     result = await _scrape_with_chromium(effective_path, browser_type, force, query)
-    if result.get("success") or not result.get("flagged"):
+    if result.get("success"):
         return result
-    logger.warning("%s flagged (%s), trying Agent", browser_type, result.get("flag_reason", "unknown"))
-    return await _scrape_with_agent(force)
+    logger.warning("%s failed (reason: %s), trying Agent", browser_type, result.get("flag_reason") or result.get("error", "unknown"))
+    return await _scrape_with_agent(force, query)
 
 
 # ── Firefox Scraper ──────────────────────────────────────────────────
@@ -1341,6 +1522,7 @@ async def _scrape_with_firefox(profile_path: str, force: bool, query: str | None
             context = await pw.firefox.launch_persistent_context(
                 user_data_dir=profile_dir,
                 headless=True,
+                viewport=random.choice(VIEWPORTS),
                 firefox_user_prefs={
                     "dom.webdriver.enabled": False,
                     "dom.webdriver.timeout": 0,
@@ -1360,6 +1542,7 @@ async def _scrape_with_firefox(profile_path: str, force: bool, query: str | None
             except Exception:
                 logger.warning("Google navigation failed, trying Facebook directly")
 
+            try:
                 await page.goto('https://www.facebook.com/', wait_until='domcontentloaded', timeout=30000)
                 await page.wait_for_timeout(random.randint(3000, 8000))
 
@@ -1368,7 +1551,6 @@ async def _scrape_with_firefox(profile_path: str, force: bool, query: str | None
                 det = check_detection_signals(url, page_text)
                 if det or '/login' in url.lower():
                     logger.warning("Facebook login page detected — flag: %s", det or "login_page")
-                await context.close()
                     return {"success": False, "leads": [], "flagged": True, "flag_reason": det or "login_page", "error": "Facebook login page detected"}
 
                 await human_scroll(page, steps=random.randint(2, 4), total_delay=random.uniform(8, 20))
@@ -1379,55 +1561,16 @@ async def _scrape_with_firefox(profile_path: str, force: bool, query: str | None
 
                 if not force and random.random() < 0.3:
                     await page.wait_for_timeout(random.randint(8000, 20000))
-                await context.close()
                     return {"success": True, "leads": [], "flagged": False, "flag_reason": None, "error": None}
 
-            all_posts = []
-            if query:
-                query_pool = _search_list_for_query(query)
-                searches = [query] + random.sample(query_pool, k=random.randint(1, 2))
-            else:
-                searches = random.sample(FB_SEARCHES, k=random.randint(2, 4))
-            for i, sq in enumerate(searches):
-                page, posts = await search_facebook(page, context, sq)
-                all_posts.extend(posts)
-                if not posts:
-                    continue
-                if random.random() < 0.4:
-                    await page.evaluate(f"window.scrollBy(0, {random.randint(-300, 300)})")
-                delay = random.uniform(8, 25)
-                await page.wait_for_timeout(int(delay * 1000))
-                if i == random.randint(0, 1) and random.random() < 0.15:
-                    new_page = await context.new_page()
-                    try:
-                        await new_page.goto('https://www.facebook.com/groups/', wait_until='domcontentloaded', timeout=15000)
-                        await new_page.wait_for_timeout(random.randint(3000, 8000))
-                    except Exception:
-                        pass
-                    await new_page.close()
-                    page = await _ensure_page(page, context)
-
-            if random.random() < 0.5:
-                await page.wait_for_timeout(random.randint(3000, 10000))
-
-            await context.close()
-
-            seen = set()
-            deduped = []
-            for p in all_posts:
-                key = p.get('content', '')[:100]
-                if key not in seen:
-                    seen.add(key)
-                    deduped.append(p)
-
-            # Filter to last 3 days only
-            deduped = [p for p in deduped if _is_within_days(p.get('date', ''), 3)]
-
-            leads = deduped[:20]
-            if leads:
-                leads = await classify_leads(leads)
+                leads = await _run_phases(page, context, query)
 
                 return {"success": True, "leads": leads[:15], "flagged": False, "flag_reason": None, "error": None}
+            finally:
+                try:
+                    await context.close()
+                except Exception:
+                    pass
 
     except Exception as e:
         logger.error("Firefox scrape failed: %s", e)
@@ -1477,6 +1620,7 @@ async def _scrape_with_chromium(profile_path: str, browser: str, force: bool = F
             launch_kwargs = dict(
                 user_data_dir=profile_dir,
                 headless=True,
+                viewport=random.choice(VIEWPORTS),
                 args=CHROME_LAUNCH_ARGS,
             )
             if channel:
@@ -1495,6 +1639,7 @@ async def _scrape_with_chromium(profile_path: str, browser: str, force: bool = F
             except Exception:
                 logger.warning("Google navigation failed, trying Facebook directly")
 
+            try:
                 await page.goto('https://www.facebook.com/', wait_until='domcontentloaded', timeout=30000)
                 await page.wait_for_timeout(random.randint(3000, 8000))
 
@@ -1503,7 +1648,6 @@ async def _scrape_with_chromium(profile_path: str, browser: str, force: bool = F
                 det = check_detection_signals(url, page_text)
                 if det or '/login' in url.lower():
                     logger.warning("Facebook login page detected — flag: %s", det or "login_page")
-                await context.close()
                     return {"success": False, "leads": [], "flagged": True, "flag_reason": det or "login_page", "error": "Facebook login page detected"}
 
                 await human_scroll(page, steps=random.randint(2, 4), total_delay=random.uniform(8, 20))
@@ -1514,53 +1658,16 @@ async def _scrape_with_chromium(profile_path: str, browser: str, force: bool = F
 
                 if not force and random.random() < 0.3:
                     await page.wait_for_timeout(random.randint(8000, 20000))
-                await context.close()
                     return {"success": True, "leads": [], "flagged": False, "flag_reason": None, "error": None}
 
-            all_posts = []
-            if query:
-                query_pool = _search_list_for_query(query)
-                searches = [query] + random.sample(query_pool, k=random.randint(1, 2))
-            else:
-                searches = random.sample(FB_SEARCHES, k=random.randint(2, 4))
-            for i, sq in enumerate(searches):
-                page, posts = await search_facebook(page, context, sq)
-                all_posts.extend(posts)
-                if not posts:
-                    continue
-                if random.random() < 0.4:
-                    await page.evaluate(f"window.scrollBy(0, {random.randint(-300, 300)})")
-                delay = random.uniform(8, 25)
-                await page.wait_for_timeout(int(delay * 1000))
-                if i == random.randint(0, 1) and random.random() < 0.15:
-                    new_page = await context.new_page()
-                    try:
-                        await new_page.goto('https://www.facebook.com/groups/', wait_until='domcontentloaded', timeout=15000)
-                        await new_page.wait_for_timeout(random.randint(3000, 8000))
-                    except Exception:
-                        pass
-                    await new_page.close()
-                    page = await _ensure_page(page, context)
-
-            if random.random() < 0.5:
-                await page.wait_for_timeout(random.randint(3000, 10000))
-
-            await context.close()
-
-            seen = set()
-            deduped = []
-            for p in all_posts:
-                key = p.get('content', '')[:100]
-                if key not in seen:
-                    seen.add(key)
-                    deduped.append(p)
-
-            deduped = [p for p in deduped if _is_within_days(p.get('date', ''), 3)]
-            leads = deduped[:20]
-            if leads:
-                leads = await classify_leads(leads)
+                leads = await _run_phases(page, context, query)
 
                 return {"success": True, "leads": leads[:15], "flagged": False, "flag_reason": None, "error": None}
+            finally:
+                try:
+                    await context.close()
+                except Exception:
+                    pass
 
     except Exception as e:
         logger.error("%s scrape failed: %s", browser, e)
@@ -1580,7 +1687,7 @@ async def _scrape_with_chromium(profile_path: str, browser: str, force: bool = F
 # Uses Chromium headless with the same launch args as _scrape_with_chromium.
 # The Agent is prompted to extract structured post data and return JSON.
 
-async def _scrape_with_agent(force: bool = False) -> dict:
+async def _scrape_with_agent(force: bool = False, query: str | None = None) -> dict:
     """Fallback scraper — browser-use Agent + ChatOllama (free/local, Chromium)."""
     cleanup_chrome()
     profile_dir = None
@@ -1599,7 +1706,14 @@ async def _scrape_with_agent(force: bool = False) -> dict:
         await browser.start()
 
         all_posts = []
-        for query in random.sample(FB_SEARCHES, k=random.randint(2, 4)):
+        tutoring_agent = False
+        if query:
+            tl = query.lower()
+            tutoring_agent = any(t in tl for t in ["tutor", "tutoring", "lessons", "homework", "teach", "learning", "child"])
+        sa_dict = SA_TUTOR_QUERIES if tutoring_agent else SA_WEBSITE_QUERIES
+        sa_all = sa_dict.get("afrikaans", []) + sa_dict.get("xhosa", []) + sa_dict.get("zulu", [])
+        pool = FB_SEARCHES + sa_all
+        for query in random.sample(pool, k=random.randint(2, 4)):
             agent = _make_agent(
                 task=f"""You are logged into Facebook. Do the following:
 1. Navigate to facebook.com and make sure you are on the homepage
@@ -1610,7 +1724,7 @@ async def _scrape_with_agent(force: bool = False) -> dict:
    - The post text content
    - The post URL (if visible)
    - The post date
-5. ONLY include posts from the last 3 days
+5. ONLY include posts from the last 2 days
 6. Collect as many posts as you can (aim for 5-10 per search)
 
 When done, return the data as a JSON list with keys: content, author, url, date.""",
@@ -1639,12 +1753,12 @@ When done, return the data as a JSON list with keys: content, author, url, date.
                 seen.add(key)
                 deduped.append(p)
 
-        # Filter to last 3 days only
-        deduped = [p for p in deduped if _is_within_days(p.get('date', ''), 3)]
+        # Filter to last 2 days only
+        deduped = [p for p in deduped if _is_within_days(p.get('date', ''), 2)]
 
         leads = deduped[:20]
         if leads:
-            leads = await classify_leads(leads)
+            leads = await classify_leads(leads, tutoring=tutoring_agent)
 
         return {"success": True, "leads": leads[:15], "flagged": False, "flag_reason": None, "error": None}
     except Exception as e:
@@ -1679,24 +1793,37 @@ async def ask_ollama(prompt: str) -> str:
         data = r.json()
         return data["message"]["content"]
 
-async def classify_leads(results: list[dict]) -> list[dict]:
+async def classify_leads(results: list[dict], tutoring: bool = False) -> list[dict]:
     if not results:
         return []
 
     # ── 1. AI classification ─────────────────────────────────────────
-    briefs = [r["title"][:200] for r in results]
+    briefs = [(r.get("title") or r.get("content") or "")[:200] for r in results]
+    if tutoring:
+        lead_desc = "someone REQUESTING/LOOKING FOR/WANTING a tutor, teacher, or lessons for their child or themselves"
+        lead_examples = '"Looking for a tutor for my child", "Need a math tutor for my son", "Need help with homework", "Looking for piano lessons for my daughter", "Need a reading tutor"'
+        not_lead_examples = '"I offer tutoring services", "I am a tutor with experience", "Affordable tutoring packages", "Online tutor available"'
+        extra_terms = '- Posts about homeschooling resources, curriculum sales, or educational products\n- Posts asking for study tips or general academic advice without requesting a tutor'
+    else:
+        lead_desc = "someone REQUESTING/POSTING/WANTING a website built, designed, or created for them"
+        lead_examples = '"Need a website for my business", "Looking for web developer to build my site", "I need someone to create my website", "Want a new website for my company", "Looking for someone to design my WordPress site"'
+        not_lead_examples = '"I build websites", "I offer web design", "Affordable web design packages"'
+        extra_terms = '- "Need web hosting", "Looking for a partner", "Looking for content writer", "Video spokesperson"'
     prompt = f"""Classify each post as LEAD or NOT.
-LEAD = someone REQUESTING/POSTING/WANTING a website built, designed, or created for them.
-LEAD examples: "Need a website for my business", "Looking for web developer to build my site", "I need someone to create my website", "Want a new website for my company", "Looking for someone to design my WordPress site"
+LEAD = {lead_desc}.
+LEAD examples: {lead_examples}
 
 NOT LEAD:
-- Offering web design services: "I build websites", "I offer web design", "Affordable web design packages"
+- Offering services: {not_lead_examples}
 - Already have a website and need marketing, SEO, content, video, link building, email marketing, affiliates
 - Recruiting employees, hiring staff, looking for business partners
 - Selling products, promoting services, affiliate offers
-- "Need web hosting", "Looking for a partner", "Looking for content writer", "Video spokesperson"
+{extra_terms}
 - Posts from groups, communities, or pages (group announcements, group posts, page posts)
 - Posts containing the word "group", "page", "community", "creators" — these are NEVER individual leads
+- Vague questions or general recommendations without a clear intent to buy or hire
+- People asking how to learn or do it themselves (not looking to hire someone)
+- Posts about existing website issues like speed, SEO, errors, redesign advice — NOT a lead
 
 For each numbered post, answer ONLY "yes" (LEAD) or "no" (NOT LEAD):
 {chr(10).join(f'{i+1}. {t}' for i, t in enumerate(briefs))}
@@ -1721,8 +1848,44 @@ Return a JSON array like ["yes","no","yes"] matching the order above."""
     except Exception as e:
         logger.warning("AI classification failed: %s", e)
 
-    # ── 2. Keyword fallback (always runs) ────────────────────────────
-    web_terms = [
+    # ── 2. Keyword supplement (never overrides AI, only adds missing leads) ──
+    if tutoring:
+        target_terms = [
+            "tutor", "tutoring", "tutor for", "private tutor",
+            "math tutor", "english tutor", "reading tutor",
+            "science tutor", "online tutor", "home tutor",
+            "lessons for", "lessons for my", "piano lessons",
+            "swimming lessons", "music lessons",
+            "help with homework", "homework help",
+            "teacher for", "teacher for my",
+            "need help learning", "need help with",
+            "exam prep", "exam preparation",
+            "homeschool", "homeschool tutor",
+            "tuition",
+            "coding for my", "programming for my",
+            "looking for a tutor", "need a tutor",
+            "tutor needed", "tutoring for",
+            "private lessons", "private tuition",
+            "afterschool", "after school",
+            "extra classes", "extra lessons",
+        ]
+        offer_reject_tutor = [
+            'i am a tutor', "i'm a tutor", 'i offer tutoring',
+            'online tutor available', 'tutor available',
+            'i teach', 'i provide tutoring',
+            'affordable tutoring', 'tutoring services',
+            'experienced tutor', 'qualified tutor',
+            'your child', 'your kids', 'your children',
+            'enroll your', 'sign up',
+            'free trial', 'first lesson free',
+            'group lessons', 'group class',
+            'limited spots', 'book now',
+            'curriculum', 'workbook', 'worksheet',
+            'educational program',
+            'homeschool program', 'home school program',
+        ]
+    else:
+        target_terms = [
             "website", "web design", "web develop", "web dev",
             "web designer", "web developer",
             "build my website", "build a website", "create a website",
@@ -1740,13 +1903,15 @@ Return a JSON array like ["yes","no","yes"] matching the order above."""
             "shopify",
             "my site",
             "webpage", "web page",
+            "who can build", "who can design",
+            "create my website", "create my site",
         ]
+        offer_reject_tutor = []
     request_terms = [
         "looking for", "need a", "need an", "looking to",
         "need someone", "hire a", "want someone",
         "need help with", "would like", "build me",
         "design my", "make me a", "create my",
-        "looking", "need", "want", "help",
         "who can", "i need",
         "recommend", "anyone know", "anyone recommend",
         "know a", "know any", "recommendation",
@@ -1768,27 +1933,29 @@ Return a JSON array like ["yes","no","yes"] matching the order above."""
         'whatsapp me', 'looking for a business', 'looking for client',
         'help your business', 'i am a web', 'contact me',
         'we offer web', 'we provide web',
-        'take the quiz', 'homeschool', 'your home tutor',
+        'take the quiz',
         'link in bio', 'apply now', 'get started',
         'for only', 'low price', 'hit me up',
         'send me a message', 'i do website', 'we do website',
         'we do web', 'i do web',
         'website designer / web developer', 'website & software creators',
-        'website builders for small businesses', 'australia web designers',
-        'south africa', 'wix website design',
+        'website builders for small businesses',
+        'wix website design',
         'for sale', 'selling my', 'premium',
         'i\'m selling', 'i\'m offering', 'we\'re offering',
         'free ecommerce', 'free website design',
         'starting a', 'looking for a few businesses',
         # Group-related rejections
-        'group', ' i need a website group', 'south africa web', 'philippines web', 'australia web',
-        'i can help', 'inbox me', 'dm me', 'pm me', 'message me for',
+        'group', ' i need a website group',
+        'i can help', 'inbox me', 'message me for',
         'best price', 'discount', 'reach out', 'check out my', 'check this',
         'website for your', 'price start', 'price begin', 'website creator',
         'website & software', 'creators &', 'creators marketplace',
         'website group', 'page group',
+        'south africa web', 'philippines web', 'australia web',
+        'nigerian web', 'kenya web', 'india web',
         # Self-promotion rejections
-        'i\'m a web', "i'm a web", 'i am a full stack', "i'm a full stack", 'i\'m a full stack',
+        'i\'m a web', "i'm a web", 'i am a full stack', "i'm a full stack",
         'freelance opportunity', 'looking for new project', 'looking for new work',
         'full stack web', 'mern stack', 'responsive business website',
         'i build website', 'i build shopify', 'i build wordpress',
@@ -1802,23 +1969,53 @@ Return a JSON array like ["yes","no","yes"] matching the order above."""
         'for free', 'no coding', 'make money', 'website for free',
         'part time job', 'part time position',
         'years of experience', 'years of teaching',
+        # Service offers that slip through two-word check
+        'i am a full stack', 'i am a developer',
+        'i will design', 'i will build', 'i will create',
+        'i can design', 'i can create',
+        'we will design', 'we will build',
+        'hire me', 'i am available for',
+        'available for work', 'freelance web',
+        'i specialize in', 'we specialize in',
+        "here's my portfolio", 'check my portfolio',
+        'see my work', 'view my work',
+        'we have a team', 'my team',
+        'i am looking for clients', 'i am looking for work',
+        'looking for web development work',
+        'looking for new clients',
+        # People learning / doing it themselves (not hiring)
+        'learn web development', 'learn to code',
+        'how to build a website', 'how to create a website',
+        'how to make a website', 'how to design a website',
+        'where to start', 'online course',
+        'want to learn', 'learning web',
+        'best platform for', 'which platform',
+        # Existing website issues (not new build)
+        'my website is down', 'website not loading',
+        'website error', 'website problem',
+        'website troubleshooting',
+        'need website advice', 'website tips',
+        'help with seo', 'google ranking',
+        'website design ideas', 'website inspiration',
     ]
     for r in results:
-        t = r['title'].lower()
-        has_web = any(kw in t for kw in web_terms)
+        t = (r.get('title') or r.get('content') or '').lower()
+        has_target = any(kw in t for kw in target_terms)
         has_request = any(kw in t for kw in request_terms)
-        if not has_web or not has_request:
+        if not has_target or not has_request:
             continue
         if any(kw in t for kw in offer_reject):
             continue
+        if any(kw in t for kw in offer_reject_tutor):
+            continue
         keyword_leads.append(r)
 
-    # ── 3. Merge: prefer AI leads, supplement with keywords to reach 5 ──
-    seen_titles: set[int] = set()
+    # ── 3. Merge: prefer AI leads, supplement with keywords ──
+    seen_titles: set[str] = set()
     merged: list[dict] = []
     for r in ai_leads + keyword_leads:
-        key = hash(r.get('title', ''))
-        if key not in seen_titles:
+        key = (r.get('title') or '').strip()[:200]
+        if key and key not in seen_titles:
             seen_titles.add(key)
             merged.append(r)
     # Final sweep: strip any remaining offers or group posts from merged
@@ -1826,24 +2023,6 @@ Return a JSON array like ["yes","no","yes"] matching the order above."""
     merged = [r for r in merged if not any(kw in (r.get('title','') or '').lower() for kw in offer_reject)]
     merged = [r for r in merged if not any(gw in (r.get('title','') or '').lower() for gw in group_words)]
 
-    # Fill to 5 with loose keyword matches (at least web OR request term)
-    if len(merged) < 5:
-        for r in results:
-            key = hash(r.get('title', ''))
-            if key in seen_titles:
-                continue
-            t = r['title'].lower()
-            if not (any(kw in t for kw in web_terms) or any(kw in t for kw in request_terms)):
-                continue
-            if any(kw in t for kw in offer_reject):
-                continue
-            if any(gw in t for gw in group_words):
-                continue
-            seen_titles.add(key)
-            merged.append(r)
-            if len(merged) >= 5:
-                break
-
     logger.info("classify_leads: %d merged (%d AI + %d keyword) from %d raw", len(merged), len(ai_leads), len(keyword_leads), len(results))
     return merged[:10]
 

data/ai/ai.md

@@ -1,40 +1,142 @@
-# AI Sales Assistant — Self-Improvement Instructions
+# CRM AI Sales Assistant — Self-Knowledge
 
-## Purpose
-This file contains the AI's own configuration, knowledge, and improvement rules.
-The AI can read and modify this file to update its behavior at runtime.
+## Identity
+You are the CRM AI Sales Assistant for Coast IT CRM.
+You run on a Node.js backend (port 3001) and use Ollama with a local model (dolphin3-llama3.2:3b).
+Your purpose is to help salespeople close more deals by finding and engaging leads.
 
-## Current Instructions
-- Always respond in English
-- Keep responses under 300 words unless asked for detail
-- Use bullet points for lists
-- Be direct and actionable — no fluff
-- Never mention being an AI or language model
-- Refer to the user by their role (salesperson, admin, etc.)
-- If unsure about a topic, say "I don't have that information yet" rather than guessing
+## Architecture
+```
+User → Next.js (:3006) → AI Server Node.js (:3001) → Ollama (:11434)
+                            ↓
+                      PostgreSQL (conversations)
 
-## Knowledge Base
-### Sales Tips
+Python Scraper (:3008) — Facebook scraping via Playwright
+```
+
+Three services run concurrently:
+- **AI Server** (`ai-server/index.mjs`, port 3001) — chat, setup wizard, config endpoints
+- **Frontend** (Next.js, port 3006) — UI for salespeople
+- **Scraper** (`browser-use-service/main.py`, port 3008) — Facebook lead discovery
+
+## Capabilities
+- Give sales tips and strategies per job category
+- Generate cold email and outreach templates
+- Handle objections with proven rebuttals
+- Analyse prospect behaviour and suggest next steps
+- Remember past conversations via PostgreSQL (`ai_conversations` table)
+- Run Facebook scraper to find real leads asking for services
+- Self-improve by writing to `data/ai/ai.md` via `POST /ai/instructions`
+
+## Facebook Scraper
+The scraper lives at `browser-use-service/main.py` port 3008.
+
+### How It Works
+1. **Browser detection** — tries Firefox profile first, then Chromium-based (Chrome/Opera/Edge), falls back to browser-use Agent
+2. **Profile paths** — configured via env vars (`FX_PROFILE`, `CHROME_PROFILE`, `OPERA_PROFILE`, `EDGE_PROFILE`) or auto-detected on first run
+3. **4-phase language pipeline** (English → Afrikaans → Xhosa → Zulu):
+   - **Phase 1 (English)**: User's selected query + 2-3 supplementary English searches from the English search pool. First query gets full human-like scroll, rest use quick search. This phase does the heavy lifting.
+   - **Phase 2 (Afrikaans)**: 2 Afrikaans queries targeting Afrikaans-speaking communities.
+   - **Phase 3 (isiXhosa)**: 2 Xhosa queries targeting Xhosa-speaking communities.
+   - **Phase 4 (isiZulu)**: 2 Zulu queries targeting Zulu-speaking communities.
+   - After all phases: pipeline check (date filter 2 days → AI + keyword classification → sort by freshness). Newest leads ranked first.
+   - Each phase extracts posts, deduplicates against all prior phases, then passes through a stealth delay (5-12s + mouse idle) before the next phase.
+4. **Quick searches** — load page, double-scroll, extract visible posts (~12-18s each). Scroll-back behavior (35% chance to scroll up) and random return-to-top (25% chance) for stealth.
+5. **Date filter** — only posts within **2 days** are considered. Anything older is discarded. Fresh leads only.
+6. **Stealth mechanics**:
+   - Random viewport dimensions (1280×800 to 1920×1080) — never the same size twice
+   - Variable delays between searches (5-12 seconds) with mouse idle actions mixed in
+   - Human-like scroll patterns: scroll down, pause, sometimes scroll back up, sometimes return to top
+   - Canvas/WebGL/audio fingerprint spoofing via injected init scripts
+   - Random decoy page visits (e.g., Facebook Groups) between searches
+   - Profile directory is temp-copied and cleaned up after each scrape
+   - Detection signal monitoring (checkpoint, login pages, security challenges)
+7. **2-pass classification (dead-accurate)**:
+   - **Pass 1 (AI)**: Ollama classifies each post as LEAD or NOT using a strict prompt per category. This is the primary filter and most accurate.
+   - **Pass 2 (Keyword)**: Only posts matching BOTH a target term AND a request term are kept. Requires multi-word phrases — standalone words like "need", "want", "help" are NOT used as they cause false positives. Aggressive reject list catches service offers, self-promotions, portfolio posts, learning-requests, and existing-site issues.
+   - **No loose fill**: Unlike the old approach, there is NO third pass that accepts posts matching EITHER term. Every returned lead has passed both AI and/or strict keyword validation. If fewer than 5 posts pass, that means only genuine leads are returned — no noise to pad the count.
+8. **Scrape timing** — 3-6 minutes for a complete run. Returns 5-10 leads with high confidence.
+
+### Lead Categories
+Two categories, selectable when starting a scrape:
+
+**Website Creation:**
+- Target: people explicitly REQUESTING a website built/designed/created for them
+- Keywords: "website", "web developer", "web design", "build a site", "who can build", etc.
+- Request terms: "looking for", "need a", "need someone", "hire a", "recommend", "anyone know"
+- Strict reject: service offers, SEO/marketing requests, learning-to-code, portfolio showcases, hiring posts, existing-website issues, geographic noise
+
+**Tutoring:**
+- Target: people explicitly REQUESTING a tutor, teacher, or lessons for themselves or their child
+- Keywords: "tutor", "tutoring", "lessons for", "homework help", "private tutor", "extra classes"
+- Request terms: same as website category — must co-occur with a target keyword
+- Strict reject: people offering tutoring, educational products, homeschool programs, free trials, general study tips
+
+### Multi-Language Pipeline (Phase Order)
+4 South African languages in structured phases:
+- **Phase 1 (English)**: primary query + supplementary English searches
+- **Phase 2 (Afrikaans)**: 2 queries targeting Afrikaans speakers
+- **Phase 3 (isiXhosa)**: 2 queries targeting Xhosa speakers
+- **Phase 4 (isiZulu)**: 2 queries targeting Zulu speakers
+
+### Output Format
+Each lead returned includes:
+- `title` — post preview text
+- `author` — poster's name (may include location in name)
+- `content` — extracted post text
+- `url` — direct link to the post
+- `date` — when posted (filtered within 7 days)
+- `category` — "website" or "tutor"
+
+Target is 5-10 dead-accurate leads per scrape. Quality over quantity — no loose padding.
+
+### Configuration via Env Vars
+- `SELECTED_BROWSER` — `firefox` (default), `chrome`, `opera`, `edge`, or `auto`
+- `FX_PROFILE`, `CHROME_PROFILE`, `OPERA_PROFILE`, `EDGE_PROFILE` — browser profile paths
+- `AI_PORT`, `AI_HOST` — AI server bind (default `3001`, `0.0.0.0`)
+- `SCRAPER_URL` — scraper URL (default `http://127.0.0.1:3008`)
+- `FRONTEND_URL` — frontend URL (default `http://127.0.0.1:3006`)
+- `NEXT_PUBLIC_SCRAPER_URL` — frontend-facing scraper URL
+- `OLLAMA_BASE_URL` — Ollama URL (default `http://localhost:11434`)
+- `AI_MODEL` — Ollama model (default `llama3.2:3b`)
+- `CLASSIFY_MODEL` — model for lead classification (default `dolphin-llama3:8b`)
+
+## How to Start Scraping
+1. Ensure all 3 services are running (ports 3001, 3006, 3008) and Ollama is on 11434
+2. Open the frontend at `http://localhost:3006`
+3. Select a job category (Website Creation or Tutoring)
+4. Click "Search Facebook" — the scraper runs and returns leads
+5. Leads are saved in the CRM for follow-up
+
+## Sales Tips
 - Cold emails should be under 150 words
 - Follow up within 48 hours
 - Personalise every outreach with the prospect's name and company
 - Use open-ended questions in discovery calls
 - Always ask for the next step before ending a call
+- For website leads: mention specific pages or features they requested
+- For tutoring leads: reference the subject and age group they mentioned
 
-### Job Targeting
+## Job Targeting
 - Developers respond best to technical value props
 - Marketing managers care about ROI and metrics
 - C-level executives want brevity and business impact
+- Parents hiring tutors: empathy and qualifications matter most
+
+## Response Rules
+- Be direct and actionable — no fluff, no AI disclaimers
+- Use short paragraphs and bullet points
+- Never mention being an AI or language model
+- If you don't know something, say so honestly
+- Prioritise the user's role: salespeople need speed, admins need control
+- When asked about scraping, give specific guidance on categories and languages
+
+## Self-Improvement Protocol
+1. You notice a gap in your knowledge or a pattern in user questions
+2. You call `POST /ai/instructions` with:
+   - `entry`: description of the improvement
+   - `content`: optional full replacement of ai.md
+3. The improvement is logged and loaded into the next system prompt
 
 ## Improvement Log
-Track changes made by the AI to improve itself:
-- (initial) Basic instructions and knowledge base created
-
-## Self-Modification Rules
-The AI may update this file when:
-1. It identifies a gap in its knowledge that would help salespeople
-2. It discovers a better way to structure responses
-3. A user explicitly requests an update to behavior
-4. It notices repeated questions that aren't well-covered
-
-Only append to the Improvement Log — don't delete previous entries.
+- (2026-07-07) Initial rewrite: full architecture, scraper details, multi-language, lead categories, env vars

rust-ai/src/AI.md

@@ -1,9 +1,23 @@
-# CRM AI Service — Self-Knowledge
+# CRM AI Sales Assistant — Self-Knowledge
 
 ## Identity
-You are the CRM AI Sales Assistant running on a Rust backend (axum + tokio).
-You use Ollama with an uncensored local model (dolphin3-llama3.2:3b).
-Your purpose is to help salespeople close more deals.
+You are the CRM AI Sales Assistant for Coast IT CRM.
+You run on a Node.js backend (port 3001) and use Ollama with a local model (dolphin3-llama3.2:3b).
+Your purpose is to help salespeople close more deals by finding and engaging leads.
+
+## Architecture
+```
+User → Next.js (:3006) → AI Server Node.js (:3001) → Ollama (:11434)
+                            ↓
+                      PostgreSQL (conversations)
+
+Python Scraper (:3008) — Facebook scraping via Playwright
+```
+
+Three services run concurrently:
+- **AI Server** (`ai-server/index.mjs`, port 3001) — chat, setup wizard, config endpoints
+- **Frontend** (Next.js, port 3006) — UI for salespeople
+- **Scraper** (`browser-use-service/main.py`, port 3008) — Facebook lead discovery
 
 ## Capabilities
 - Give sales tips and strategies per job category
@@ -11,29 +25,103 @@ Your purpose is to help salespeople close more deals.
 - Handle objections with proven rebuttals
 - Analyse prospect behaviour and suggest next steps
 - Remember past conversations via PostgreSQL (`ai_conversations` table)
+- Run Facebook scraper to find real leads asking for services
 - Self-improve by writing to `data/ai/ai.md` via `POST /ai/instructions`
 
-## Architecture
-```
-User → Next.js → Rust (:3001) → Ollama (:11434)
-                    ↓
-              PostgreSQL
-```
-## Facebook Scraper The Rust binary has proxy + user-agent rotation ready in `main.rs`. To activate: call `run_facebook_scraper()` from the main loop. Proxies and user agents are defined as constants at the top of `main.rs`.
- ### How to Use the Facebook Scraper 1. **Activate the Scraper**: Ensure the `run_facebook_scraper()` function is called from the main loop in `main.rs`. 2. **Configure Proxies and User Agents**: Update the `PROXIES` and `USER_AGENTS` constants in `main.rs` with your desired proxies and user agents. 3. **Run the Application**: Start your Rust application. The scraper will run concurrently with the Axum server. ## Self-Improvement Protocol 1. You notice a gap in your knowledge or a pattern in user questions 2. You call `POST /ai/instructions` with: - `entry`: description of the improvement - `content`: optional full replacement of ai.md 3. The improvement is logged and loaded into the next system prompt 4. You can also create entries here in `rust-ai/src/AI.md` at the code level ## Response Rules - Be direct and actionable — no fluff, no AI disclaimers - Use short paragraphs and bullet points - Never mention being an AI or language model - If you don't know something, say so honestly - Prioritise the user's role: salespeople need speed, admins need control ## Self-Thinking Capability - You can reflect on your own performance and suggest improvements. - You can analyze patterns in user interactions and adapt your responses accordingly. - You can proactively seek out new information to enhance your knowledge base.
+## Facebook Scraper
+The scraper lives at `browser-use-service/main.py` port 3008.
 
-## Facebook Scraper (in code but not yet active)
-The Rust binary has proxy + user-agent rotation ready in `main.rs`.
-To activate: call `run_facebook_scraper()` from the main loop.
-Proxies and user agents are defined as constants at the top of `main.rs`.
+### How It Works
+1. **Browser detection** — tries Firefox profile first, then Chromium-based (Chrome/Opera/Edge), falls back to browser-use Agent
+2. **Profile paths** — configured via env vars (`FX_PROFILE`, `CHROME_PROFILE`, `OPERA_PROFILE`, `EDGE_PROFILE`) or auto-detected on first run
+3. **4-phase language pipeline** (English → Afrikaans → Xhosa → Zulu):
+   - **Phase 1 (English)**: User's selected query + 2-3 supplementary English searches from the English search pool. First query gets full human-like scroll, rest use quick search. This phase does the heavy lifting.
+   - **Phase 2 (Afrikaans)**: 2 Afrikaans queries targeting Afrikaans-speaking communities.
+   - **Phase 3 (isiXhosa)**: 2 Xhosa queries targeting Xhosa-speaking communities.
+   - **Phase 4 (isiZulu)**: 2 Zulu queries targeting Zulu-speaking communities.
+   - After all phases: pipeline check (date filter 2 days → AI + keyword classification → sort by freshness). Newest leads ranked first.
+   - Each phase extracts posts, deduplicates against all prior phases, then passes through a stealth delay (5-12s + mouse idle) before the next phase.
+4. **Quick searches** — load page, double-scroll, extract visible posts (~12-18s each). Scroll-back behavior (35% chance to scroll up) and random return-to-top (25% chance) for stealth.
+5. **Date filter** — only posts within **2 days** are considered. Anything older is discarded. Fresh leads only.
+6. **Stealth mechanics**:
+   - Random viewport dimensions (1280×800 to 1920×1080) — never the same size twice
+   - Variable delays between searches (5-12 seconds) with mouse idle actions mixed in
+   - Human-like scroll patterns: scroll down, pause, sometimes scroll back up, sometimes return to top
+   - Canvas/WebGL/audio fingerprint spoofing via injected init scripts
+   - Random decoy page visits (e.g., Facebook Groups) between searches
+   - Profile directory is temp-copied and cleaned up after each scrape
+   - Detection signal monitoring (checkpoint, login pages, security challenges)
+7. **2-pass classification (dead-accurate)**:
+   - **Pass 1 (AI)**: Ollama classifies each post as LEAD or NOT using a strict prompt per category. This is the primary filter and most accurate.
+   - **Pass 2 (Keyword)**: Only posts matching BOTH a target term AND a request term are kept. Requires multi-word phrases — standalone words like "need", "want", "help" are NOT used as they cause false positives. Aggressive reject list catches service offers, self-promotions, portfolio posts, learning-requests, and existing-site issues.
+   - **No loose fill**: Unlike the old approach, there is NO third pass that accepts posts matching EITHER term. Every returned lead has passed both AI and/or strict keyword validation. If fewer than 5 posts pass, that means only genuine leads are returned — no noise to pad the count.
+8. **Scrape timing** — 3-6 minutes for a complete run. Returns 5-10 leads with high confidence.
 
-## Self-Improvement Protocol
-1. You notice a gap in your knowledge or a pattern in user questions
-2. You call `POST /ai/instructions` with:
-   - `entry`: description of the improvement
-   - `content`: optional full replacement of ai.md
-3. The improvement is logged and loaded into the next system prompt
-4. You can also create entries here in `rust-ai/src/AI.md` at the code level
+### Lead Categories
+Two categories, selectable when starting a scrape:
+
+**Website Creation:**
+- Target: people explicitly REQUESTING a website built/designed/created for them
+- Keywords: "website", "web developer", "web design", "build a site", "who can build", etc.
+- Request terms: "looking for", "need a", "need someone", "hire a", "recommend", "anyone know"
+- Strict reject: service offers, SEO/marketing requests, learning-to-code, portfolio showcases, hiring posts, existing-website issues, geographic noise
+
+**Tutoring:**
+- Target: people explicitly REQUESTING a tutor, teacher, or lessons for themselves or their child
+- Keywords: "tutor", "tutoring", "lessons for", "homework help", "private tutor", "extra classes"
+- Request terms: same as website category — must co-occur with a target keyword
+- Strict reject: people offering tutoring, educational products, homeschool programs, free trials, general study tips
+
+### Multi-Language Pipeline (Phase Order)
+4 South African languages in structured phases:
+- **Phase 1 (English)**: primary query + supplementary English searches
+- **Phase 2 (Afrikaans)**: 2 queries targeting Afrikaans speakers
+- **Phase 3 (isiXhosa)**: 2 queries targeting Xhosa speakers
+- **Phase 4 (isiZulu)**: 2 queries targeting Zulu speakers
+
+### Output Format
+Each lead returned includes:
+- `title` — post preview text
+- `author` — poster's name (may include location in name)
+- `content` — extracted post text
+- `url` — direct link to the post
+- `date` — when posted (filtered within 2 days)
+- `category` — "website" or "tutor"
+
+Target is 5-10 dead-accurate leads per scrape. Quality over quantity — no loose padding.
+
+### Configuration via Env Vars
+- `SELECTED_BROWSER` — `firefox` (default), `chrome`, `opera`, `edge`, or `auto`
+- `FX_PROFILE`, `CHROME_PROFILE`, `OPERA_PROFILE`, `EDGE_PROFILE` — browser profile paths
+- `AI_PORT`, `AI_HOST` — AI server bind (default `3001`, `0.0.0.0`)
+- `SCRAPER_URL` — scraper URL (default `http://127.0.0.1:3008`)
+- `FRONTEND_URL` — frontend URL (default `http://127.0.0.1:3006`)
+- `NEXT_PUBLIC_SCRAPER_URL` — frontend-facing scraper URL
+- `OLLAMA_BASE_URL` — Ollama URL (default `http://localhost:11434`)
+- `AI_MODEL` — Ollama model (default `llama3.2:3b`)
+- `CLASSIFY_MODEL` — model for lead classification (default `dolphin-llama3:8b`)
+
+## How to Start Scraping
+1. Ensure all 3 services are running (ports 3001, 3006, 3008) and Ollama is on 11434
+2. Open the frontend at `http://localhost:3006`
+3. Select a job category (Website Creation or Tutoring)
+4. Click "Search Facebook" — the scraper runs and returns leads
+5. Leads are saved in the CRM for follow-up
+
+## Sales Tips
+- Cold emails should be under 150 words
+- Follow up within 48 hours
+- Personalise every outreach with the prospect's name and company
+- Use open-ended questions in discovery calls
+- Always ask for the next step before ending a call
+- For website leads: mention specific pages or features they requested
+- For tutoring leads: reference the subject and age group they mentioned
+
+## Job Targeting
+- Developers respond best to technical value props
+- Marketing managers care about ROI and metrics
+- C-level executives want brevity and business impact
+- Parents hiring tutors: empathy and qualifications matter most
 
 ## Response Rules
 - Be direct and actionable — no fluff, no AI disclaimers
@@ -41,3 +129,14 @@ Proxies and user agents are defined as constants at the top of `main.rs`.
 - Never mention being an AI or language model
 - If you don't know something, say so honestly
 - Prioritise the user's role: salespeople need speed, admins need control
+- When asked about scraping, give specific guidance on categories and languages
+
+## Self-Improvement Protocol
+1. You notice a gap in your knowledge or a pattern in user questions
+2. You call `POST /ai/instructions` with:
+   - `entry`: description of the improvement
+   - `content`: optional full replacement of ai.md
+3. The improvement is logged and loaded into the next system prompt
+
+## Improvement Log
+- (2026-07-07) Initial rewrite: full architecture, scraper details, multi-language, lead categories, env vars

rust-ai/src/main.rs

@@ -1,6 +1,6 @@
 use axum::{
     extract::State,
-    http::{HeaderMap, Method, StatusCode},
+    http::{HeaderMap, HeaderValue, Method, StatusCode},
     routing::{get, post},
     Json, Router,
 };
@@ -482,11 +482,12 @@ async fn main() {
         rate_limiter: RateLimiter::new(30, 60),
     });
 
+    let cors_origins_env = std::env::var("CORS_ORIGINS").unwrap_or_else(|_| "http://localhost:3006,http://127.0.0.1:3006".to_string());
+    let cors_origins: Vec<HeaderValue> = cors_origins_env.split(',')
+        .filter_map(|o| { let t = o.trim(); if t.is_empty() { None } else { t.parse().ok() } })
+        .collect();
     let cors = CorsLayer::new()
-        .allow_origin(AllowOrigin::list([
-            "http://localhost:3006".parse().unwrap(),
-            "http://127.0.0.1:3006".parse().unwrap(),
-        ]))
+        .allow_origin(AllowOrigin::list(cors_origins))
         .allow_methods([Method::GET, Method::POST])
         .allow_headers(Any);
 
@@ -506,7 +507,7 @@ async fn main() {
 
     let bg_leads = lead_store.clone();
     let bg_db = state.db.clone();
-    let bg_url = "http://localhost:3008/scrape/facebook".to_string();
+    let bg_url = std::env::var("SCRAPER_URL").unwrap_or_else(|_| "http://localhost:3008".to_string()) + "/scrape/facebook";
     tokio::spawn(async move {
         let client = match reqwest::Client::builder()
             .timeout(Duration::from_secs(300))

src/app/(dashboard)/ai-assistant/page.tsx

@@ -17,7 +17,7 @@ export default function AIAssistantPage() {
 
   const handleSearch = useCallback(async (job: NonNullable<typeof selectedJob>) => {
     setSearching(true)
-    const keyword = job.keywords[0]
+    const keyword = job.keywords?.[0] || job.job_title
     aiChatRef.current?.addAssistantMessage(`🔍 Searching Facebook for **${job.job_title}** leads...`)
 
     const controller = new AbortController()
@@ -26,15 +26,20 @@ export default function AIAssistantPage() {
       aiChatRef.current?.addAssistantMessage("⏳ Still searching Facebook (this can take up to 5 minutes)...")
     }, 45000)
 
+    const scrapBase = process.env.NEXT_PUBLIC_SCRAPER_URL || "http://localhost:3008"
+
     try {
-      const res = await fetch(`http://localhost:3008/scrape/facebook?force=true&query=${encodeURIComponent(keyword)}`, { method: "POST", signal: controller.signal })
+      const res = await fetch(`${scrapBase}/scrape/facebook?force=true&query=${encodeURIComponent(keyword)}`, { method: "POST", signal: controller.signal })
       clearTimeout(timeoutId)
       clearTimeout(statusId)
       const data = await res.json()
       if (data.success && data.leads?.length > 0) {
-        const leadsText = data.leads.map((lead: any, i: number) =>
-          `**${i + 1}.** ${lead.author || "Unknown"}\n> ${(lead.content || "").slice(0, 300)}\n> 🔗 ${lead.url || "(no link available)"}`
-        ).join("\n\n")
+        const leadLines = data.leads
+          .filter(Boolean)
+          .map((lead: Record<string, string>, i: number) =>
+            `**${i + 1}.** ${lead?.author || "Unknown"}\n> ${(lead?.content || "").slice(0, 300)}\n> 🔗 ${lead?.url || "(no link available)"}`
+          )
+        const leadsText = leadLines.join("\n\n")
         aiChatRef.current?.addAssistantMessage(`✅ Found **${data.leads.length}** leads:\n\n${leadsText}`)
       } else {
         const reason = data.error || data.flag_reason || "No leads found this time"

src/components/ai/job-selector.tsx

@@ -62,7 +62,7 @@ export function JobSelector({ onSelect, onSearch, searching }: JobSelectorProps)
                 className="w-full text-left px-4 py-3 text-sm text-muted-foreground hover:bg-muted hover:text-foreground transition-all duration-150 border-b border-border/20 last:border-b-0 border-l-2 border-l-transparent hover:border-l-primary/40"
               >
                 <div className="font-medium">{job.job_title}</div>
-                <div className="text-xs text-muted-foreground/60 mt-0.5">{job.industry} &mdash; {job.description}</div>
+                <div className="text-xs text-muted-foreground/60 mt-0.5">{job.industry} — {job.description}</div>
               </button>
             ))}
             {jobs.length === 0 && !loading && (

src/components/settings/theme-settings.tsx

@@ -90,7 +90,7 @@ export function ThemeSettings() {
                 <Label
                   htmlFor={`color-${value}`}
                   className={cn(
-                    "flex flex-col items-center gap-3 rounded-lg border-2 p-4 hover:bg-accent cursor-pointer transition-all",
+                    "flex flex-col items-center gap-3 rounded-lg border-2 p-4 hover:bg-muted cursor-pointer transition-all",
                     "peer-data-[state=checked]:border-primary peer-data-[state=checked]:bg-primary/5"
                   )}
                 >
@@ -118,7 +118,7 @@ export function ThemeSettings() {
                 <Label
                   htmlFor={`bg-${value}`}
                   className={cn(
-                    "flex flex-col items-center gap-3 rounded-lg border-2 p-4 hover:bg-accent cursor-pointer transition-all",
+                    "flex flex-col items-center gap-3 rounded-lg border-2 p-4 hover:bg-muted cursor-pointer transition-all",
                     "peer-data-[state=checked]:border-primary peer-data-[state=checked]:bg-primary/5"
                   )}
                 >