diff --git a/ai-server/index.mjs b/ai-server/index.mjs index 327dc44..f5c81df 100644 --- a/ai-server/index.mjs +++ b/ai-server/index.mjs @@ -12,6 +12,7 @@ import http from "node:http" import fs from "node:fs" import path from "node:path" import { spawn } from "node:child_process" +import crypto from "node:crypto" import { fileURLToPath } from "node:url" const __dirname = path.dirname(fileURLToPath(import.meta.url)) @@ -133,19 +134,21 @@ async function scrapeFacebook() { try { const body = await new Promise((resolve, reject) => { const parsed = new URL(SCRAPER_URL) - const req = http.request({ hostname: parsed.hostname, port: parsed.port || 3008, path: urlPath, method: "POST", timeout: 360000 }, (res) => { + let done = false + const req = http.request({ hostname: parsed.hostname, port: parsed.port || 3008, path: urlPath, method: "POST", timeout: 60000 }, (res) => { let data = "" res.on("data", (c) => data += c) - res.on("end", () => resolve(data)) - res.on("error", reject) + res.on("end", () => { done = true; resolve(data) }) + res.on("error", (e) => { if (!done) { done = true; reject(e) } }) }) - req.on("timeout", () => { req.destroy(); reject(new Error("timeout")) }) - req.on("error", reject) + req.on("timeout", () => { if (!done) { done = true; req.destroy(); reject(new Error("scraper timeout")) } }) + req.on("error", (e) => { if (!done) { done = true; reject(e) } }) req.end() }) const data = JSON.parse(body) return data } catch (e) { + console.error("scrapeFacebook error:", e.message) return null } } @@ -198,6 +201,7 @@ Provide concise, actionable sales advice. When asked about a specific job catego const ollamaRes = await fetch(`${OLLAMA_URL}/api/chat`, { method: "POST", headers: { "Content-Type": "application/json" }, + signal: AbortSignal.timeout(60000), body: JSON.stringify({ model: MODEL, messages: [ @@ -320,6 +324,7 @@ const server = http.createServer(async (req, res) => { try { await new Promise((resolve, reject) => { const r = http.get(`${SCRAPER_URL}/health`, { timeout: 3000 }, (res) => { res.resume(); resolve() }) + r.on("timeout", () => { r.destroy(); reject(new Error("timeout")) }) r.on("error", reject) }) results.scraper = true @@ -328,6 +333,7 @@ const server = http.createServer(async (req, res) => { try { await new Promise((resolve, reject) => { const r = http.get(FRONTEND_URL, { timeout: 3000 }, (res) => { res.resume(); resolve() }) + r.on("timeout", () => { r.destroy(); reject(new Error("timeout")) }) r.on("error", reject) }) results.frontend = true @@ -539,38 +545,29 @@ const server = http.createServer(async (req, res) => { // Accepts { message, user_id?, user_role? } and returns AI response. // user_role must be "sales", "admin", or "super_admin" if provided. if (req.method === "POST" && pathname === "/ai/chat") { - const startTime = Date.now() const chunks = [] req.on("data", c => chunks.push(c)) - req.on("end", () => { - const rawBody = Buffer.concat(chunks).toString() + req.on("end", async () => { try { + const rawBody = Buffer.concat(chunks).toString() const body = JSON.parse(rawBody) - processRequest(req, res, body, startTime) - } catch { - sendJSON(res, 400, { error: "Invalid JSON" }) + const { message, user_id, user_role } = body + if (!message) { + return sendJSON(res, 400, { error: "message is required" }) + } + const validRoles = ["sales", "admin", "super_admin"] + if (user_role && !validRoles.includes(user_role)) { + return sendJSON(res, 403, { error: "Forbidden" }) + } + const response = await handleChat(message, user_id || "", user_role || "sales") + sendJSON(res, 200, { response }) + } catch (e) { + if (!res.headersSent) sendJSON(res, 500, { error: e.message }) } }) return } -// Separate handler for /ai/chat (defined here due to hoisting within the IIFE) -async function processRequest(req, res, body, startTime) { - const { message, user_id, user_role } = body - - if (!message) { - return sendJSON(res, 400, { error: "message is required" }) - } - - const validRoles = ["sales", "admin", "super_admin"] - if (user_role && !validRoles.includes(user_role)) { - return sendJSON(res, 403, { error: "Forbidden" }) - } - - const response = await handleChat(message, user_id || "", user_role || "sales") - return sendJSON(res, 200, { response }) - } - // 404 fallback sendJSON(res, 404, { error: "Not found" }) } catch (err) { diff --git a/browser-use-service/main.py b/browser-use-service/main.py index 5ca213d..c41a9f2 100644 --- a/browser-use-service/main.py +++ b/browser-use-service/main.py @@ -666,38 +666,24 @@ TUTORING_SEARCHES = [ "tutor near me for", ] -def _search_list_for_query(query: str) -> list[str]: - """Pick the appropriate search query pool based on the search term.""" - tl = query.lower() - tutoring_terms = ["tutor", "tutoring", "lessons", "homework", "teach", "learning", "child", "math", "english", "science", "exam", "homeschool", "coding", "programming", "piano", "reading"] - if any(t in tl for t in tutoring_terms): - return TUTORING_SEARCHES - return FB_SEARCHES - # ── South African Multi-Language Queries ────────────────────────────── -# 4 most spoken SA languages: English, Afrikaans, isiXhosa, isiZulu. -# Each scrape searches ALL 4 languages to catch leads across all -# language communities on Facebook. +# 4 SA languages grouped for phase-based scanning: +# Phase 1: English → Phase 2: Afrikaans → Phase 3: isiXhosa → Phase 4: English (final sweep) +# Each language group has dedicated queries per category. -SA_WEBSITE_QUERIES = [ - "I need a website for my business", # English - "ek benodig n webwerf", # Afrikaans - "ek soek iemand om n webwerf te bou", # Afrikaans - "ndidinga iwebhusayithi yeshishini", # isiXhosa - "ndifuna umntu owakha iwebhusayithi", # isiXhosa - "ngidinga iwebhusayithi yebhizinisi", # isiZulu - "ngifuna umuntu owakha iwebhusayithi", # isiZulu -] +SA_WEBSITE_QUERIES = { + "english": ["I need a website for my business"], + "afrikaans": ["ek benodig n webwerf", "ek soek iemand om n webwerf te bou"], + "xhosa": ["ndidinga iwebhusayithi yeshishini", "ndifuna umntu owakha iwebhusayithi"], + "zulu": ["ngidinga iwebhusayithi yebhizinisi", "ngifuna umuntu owakha iwebhusayithi"], +} -SA_TUTOR_QUERIES = [ - "I need a tutor for my child", # English - "ek benodig n tutor vir my kind", # Afrikaans - "ek soek n privaat onderwyser", # Afrikaans - "ndifuna utitshala womntwana wam", # isiXhosa - "ndidinga umfundisi-ntsapho", # isiXhosa - "ngidinga uthisha wengane yami", # isiZulu - "ngifuna umfundisi wengane", # isiZulu -] +SA_TUTOR_QUERIES = { + "english": ["I need a tutor for my child"], + "afrikaans": ["ek benodig n tutor vir my kind", "ek soek n privaat onderwyser"], + "xhosa": ["ndifuna utitshala womntwana wam", "ndidinga umfundisi-ntsapho"], + "zulu": ["ngidinga uthisha wengane yami", "ngifuna umfundisi wengane"], +} async def _quick_search(page, context, query: str) -> tuple: @@ -1361,6 +1347,94 @@ def cleanup_chrome(): pass +# ── 4-Phase Language Pipeline ───────────────────────────────────────── +# Runs searches in ordered language phases: +# Phase 1: English (main query + supplementary EN searches) +# Phase 2: Afrikaans +# Phase 3: isiXhosa +# Phase 4: English (final sweep with different EN queries) +# Each phase extracts posts and deduplicates on the fly. +# Pipeline check (classify_leads) runs at end to quality-filter. + +PHASE_ORDER = ["english", "afrikaans", "xhosa", "zulu"] + + +async def _run_phases(page, context, query: str | None = None) -> list[dict]: + """4-phase language pipeline: English → Afrikaans → Xhosa → Zulu. + Each phase finds leads in that language. Pipeline check at end filters + sorts by freshness. + Returns top 10 newest, most relevant leads.""" + all_posts: list[dict] = [] + seen_keys: set[str] = set() + tutoring = False + + if query: + tl = query.lower() + tutoring = any(t in tl for t in ["tutor", "tutoring", "lessons", "homework", "teach", "learning", "child"]) + + lang_pool = SA_TUTOR_QUERIES if tutoring else SA_WEBSITE_QUERIES + en_pool = TUTORING_SEARCHES if tutoring else FB_SEARCHES + + # Build phase query lists + supplement = random.sample(en_pool, k=min(3, len(en_pool))) if query else [] + phase_queries: list[list[str]] = [] + + # Phase 1: English (user query + supplement from EN pool) + phase_queries.append(([query] + supplement) if query else random.sample(en_pool, k=min(4, len(en_pool)))) + + # Phase 2: Afrikaans + phase_queries.append(lang_pool.get("afrikaans", [])) + + # Phase 3: isiXhosa + phase_queries.append(lang_pool.get("xhosa", [])) + + # Phase 4: isiZulu + phase_queries.append(lang_pool.get("zulu", [])) + + # Execute phases + for phase_idx, queries in enumerate(phase_queries): + if not queries: + continue + phase_posts: list[dict] = [] + for i, q in enumerate(queries): + is_first = (phase_idx == 0 and i == 0 and query is not None) + if is_first: + page, posts = await search_facebook(page, context, q) + else: + page, posts = await _quick_search(page, context, q) + + for p in posts: + key = p.get('content', '')[:100] + if key and key not in seen_keys: + seen_keys.add(key) + phase_posts.append(p) + + all_posts.extend(phase_posts) + + # Stealth delay between phases (not after last) + if phase_idx < len(phase_queries) - 1 and phase_posts: + await page.wait_for_timeout(random.uniform(5000, 12000)) + if random.random() < 0.2: + await random_idle(page) + + # Pipeline check: date filter (2 days max) + AI/keyword classification + all_posts = [p for p in all_posts if _is_within_days(p.get('date', ''), 2)] + + leads = all_posts[:20] + if leads: + leads = await classify_leads(leads, tutoring=tutoring) + + # Sort by freshness — newest leads first + def _sort_key(l): + try: + return datetime.strptime((l.get('date') or '').strip()[:10], '%Y-%m-%d') + except (ValueError, IndexError): + return datetime.min + + leads.sort(key=_sort_key, reverse=True) + + return leads[:10] + + # ── Main Scrape Dispatcher ──────────────────────────────────────────── # scrape_facebook() is the main entry point. It: # 1. Resolves the browser profile path (from SELECTED_BROWSER env var or auto-detect) @@ -1415,17 +1489,17 @@ async def scrape_facebook(profile_path: str | None = None, force: bool = False, # Firefox path if browser_type == "firefox": result = await _scrape_with_firefox(effective_path, force, query) - if result.get("success") or not result.get("flagged"): + if result.get("success"): return result - logger.warning("Firefox flagged (%s), trying Agent", result.get("flag_reason", "unknown")) - return await _scrape_with_agent(force) + logger.warning("Firefox failed (reason: %s), trying Agent", result.get("flag_reason") or result.get("error", "unknown")) + return await _scrape_with_agent(force, query) # Chromium-based (chrome / opera / edge) result = await _scrape_with_chromium(effective_path, browser_type, force, query) - if result.get("success") or not result.get("flagged"): + if result.get("success"): return result - logger.warning("%s flagged (%s), trying Agent", browser_type, result.get("flag_reason", "unknown")) - return await _scrape_with_agent(force) + logger.warning("%s failed (reason: %s), trying Agent", browser_type, result.get("flag_reason") or result.get("error", "unknown")) + return await _scrape_with_agent(force, query) # ── Firefox Scraper ────────────────────────────────────────────────── @@ -1468,87 +1542,35 @@ async def _scrape_with_firefox(profile_path: str, force: bool, query: str | None except Exception: logger.warning("Google navigation failed, trying Facebook directly") - await page.goto('https://www.facebook.com/', wait_until='domcontentloaded', timeout=30000) - await page.wait_for_timeout(random.randint(3000, 8000)) + try: + await page.goto('https://www.facebook.com/', wait_until='domcontentloaded', timeout=30000) + await page.wait_for_timeout(random.randint(3000, 8000)) - url = page.url - page_text = await page.evaluate('document.body.innerText') if '/login' in url.lower() else '' - det = check_detection_signals(url, page_text) - if det or '/login' in url.lower(): - logger.warning("Facebook login page detected — flag: %s", det or "login_page") - await context.close() - return {"success": False, "leads": [], "flagged": True, "flag_reason": det or "login_page", "error": "Facebook login page detected"} + url = page.url + page_text = await page.evaluate('document.body.innerText') if '/login' in url.lower() else '' + det = check_detection_signals(url, page_text) + if det or '/login' in url.lower(): + logger.warning("Facebook login page detected — flag: %s", det or "login_page") + return {"success": False, "leads": [], "flagged": True, "flag_reason": det or "login_page", "error": "Facebook login page detected"} - await human_scroll(page, steps=random.randint(2, 4), total_delay=random.uniform(8, 20)) - if random.random() < 0.25: - await page.evaluate("window.scrollTo(0, 0)") - await page.wait_for_timeout(random.randint(2000, 5000)) - await human_scroll(page, steps=random.randint(1, 2)) + await human_scroll(page, steps=random.randint(2, 4), total_delay=random.uniform(8, 20)) + if random.random() < 0.25: + await page.evaluate("window.scrollTo(0, 0)") + await page.wait_for_timeout(random.randint(2000, 5000)) + await human_scroll(page, steps=random.randint(1, 2)) - if not force and random.random() < 0.3: - await page.wait_for_timeout(random.randint(8000, 20000)) - await context.close() - return {"success": True, "leads": [], "flagged": False, "flag_reason": None, "error": None} + if not force and random.random() < 0.3: + await page.wait_for_timeout(random.randint(8000, 20000)) + return {"success": True, "leads": [], "flagged": False, "flag_reason": None, "error": None} - all_posts = [] - tutoring = False - if query: - tl = query.lower() - tutoring = any(t in tl for t in ["tutor", "tutoring", "lessons", "homework", "teach", "learning", "child"]) - lang_pool = SA_TUTOR_QUERIES if tutoring else SA_WEBSITE_QUERIES - non_english = [q for q in lang_pool if q.strip().lower() != tl] - query_pool = _search_list_for_query(query) - supp_k = random.randint(3, 4) if tutoring else random.randint(2, 3) - supplement = random.sample(query_pool, k=supp_k) - searches = [query] + supplement + non_english - english_count = 1 + len(supplement) - else: - searches = random.sample(FB_SEARCHES + SA_WEBSITE_QUERIES + SA_TUTOR_QUERIES, k=random.randint(4, 7)) - for i, sq in enumerate(searches): - page, posts = await search_facebook(page, context, sq) if i == 0 else await _quick_search(page, context, sq) - all_posts.extend(posts) - if not posts: - continue - if i > 0: - await page.wait_for_timeout(random.uniform(5000, 12000)) - if random.random() < 0.2: - await random_idle(page) - continue - if random.random() < 0.4: - await page.evaluate(f"window.scrollBy(0, {random.randint(-300, 300)})") - delay = random.uniform(8, 25) - await page.wait_for_timeout(int(delay * 1000)) - if i == random.randint(0, 1) and random.random() < 0.15: - new_page = await context.new_page() - try: - await new_page.goto('https://www.facebook.com/groups/', wait_until='domcontentloaded', timeout=15000) - await new_page.wait_for_timeout(random.randint(3000, 8000)) - except Exception: - pass - await new_page.close() - page = await _ensure_page(page, context) + leads = await _run_phases(page, context, query) - if random.random() < 0.5: - await page.wait_for_timeout(random.randint(3000, 10000)) - - await context.close() - - seen = set() - deduped = [] - for p in all_posts: - key = p.get('content', '')[:100] - if key not in seen: - seen.add(key) - deduped.append(p) - - # Filter to last 2 days only - deduped = [p for p in deduped if _is_within_days(p.get('date', ''), 2)] - - leads = deduped[:20] - if leads: - leads = await classify_leads(leads, tutoring=tutoring) - - return {"success": True, "leads": leads[:15], "flagged": False, "flag_reason": None, "error": None} + return {"success": True, "leads": leads[:15], "flagged": False, "flag_reason": None, "error": None} + finally: + try: + await context.close() + except Exception: + pass except Exception as e: logger.error("Firefox scrape failed: %s", e) @@ -1617,85 +1639,35 @@ async def _scrape_with_chromium(profile_path: str, browser: str, force: bool = F except Exception: logger.warning("Google navigation failed, trying Facebook directly") - await page.goto('https://www.facebook.com/', wait_until='domcontentloaded', timeout=30000) - await page.wait_for_timeout(random.randint(3000, 8000)) + try: + await page.goto('https://www.facebook.com/', wait_until='domcontentloaded', timeout=30000) + await page.wait_for_timeout(random.randint(3000, 8000)) - url = page.url - page_text = await page.evaluate('document.body.innerText') if '/login' in url.lower() else '' - det = check_detection_signals(url, page_text) - if det or '/login' in url.lower(): - logger.warning("Facebook login page detected — flag: %s", det or "login_page") - await context.close() - return {"success": False, "leads": [], "flagged": True, "flag_reason": det or "login_page", "error": "Facebook login page detected"} + url = page.url + page_text = await page.evaluate('document.body.innerText') if '/login' in url.lower() else '' + det = check_detection_signals(url, page_text) + if det or '/login' in url.lower(): + logger.warning("Facebook login page detected — flag: %s", det or "login_page") + return {"success": False, "leads": [], "flagged": True, "flag_reason": det or "login_page", "error": "Facebook login page detected"} - await human_scroll(page, steps=random.randint(2, 4), total_delay=random.uniform(8, 20)) - if random.random() < 0.25: - await page.evaluate("window.scrollTo(0, 0)") - await page.wait_for_timeout(random.randint(2000, 5000)) - await human_scroll(page, steps=random.randint(1, 2)) + await human_scroll(page, steps=random.randint(2, 4), total_delay=random.uniform(8, 20)) + if random.random() < 0.25: + await page.evaluate("window.scrollTo(0, 0)") + await page.wait_for_timeout(random.randint(2000, 5000)) + await human_scroll(page, steps=random.randint(1, 2)) - if not force and random.random() < 0.3: - await page.wait_for_timeout(random.randint(8000, 20000)) - await context.close() - return {"success": True, "leads": [], "flagged": False, "flag_reason": None, "error": None} + if not force and random.random() < 0.3: + await page.wait_for_timeout(random.randint(8000, 20000)) + return {"success": True, "leads": [], "flagged": False, "flag_reason": None, "error": None} - all_posts = [] - tutoring = False - if query: - tl = query.lower() - tutoring = any(t in tl for t in ["tutor", "tutoring", "lessons", "homework", "teach", "learning", "child"]) - lang_pool = SA_TUTOR_QUERIES if tutoring else SA_WEBSITE_QUERIES - non_english = [q for q in lang_pool if q.strip().lower() != tl] - query_pool = _search_list_for_query(query) - supp_k = random.randint(3, 4) if tutoring else random.randint(2, 3) - supplement = random.sample(query_pool, k=supp_k) - searches = [query] + supplement + non_english - english_count = 1 + len(supplement) - else: - searches = random.sample(FB_SEARCHES + SA_WEBSITE_QUERIES + SA_TUTOR_QUERIES, k=random.randint(4, 7)) - for i, sq in enumerate(searches): - page, posts = await search_facebook(page, context, sq) if i == 0 else await _quick_search(page, context, sq) - all_posts.extend(posts) - if not posts: - continue - if i > 0: - await page.wait_for_timeout(random.uniform(5000, 12000)) - if random.random() < 0.2: - await random_idle(page) - continue - if random.random() < 0.4: - await page.evaluate(f"window.scrollBy(0, {random.randint(-300, 300)})") - delay = random.uniform(8, 25) - await page.wait_for_timeout(int(delay * 1000)) - if i == random.randint(0, 1) and random.random() < 0.15: - new_page = await context.new_page() - try: - await new_page.goto('https://www.facebook.com/groups/', wait_until='domcontentloaded', timeout=15000) - await new_page.wait_for_timeout(random.randint(3000, 8000)) - except Exception: - pass - await new_page.close() - page = await _ensure_page(page, context) + leads = await _run_phases(page, context, query) - if random.random() < 0.5: - await page.wait_for_timeout(random.randint(3000, 10000)) - - await context.close() - - seen = set() - deduped = [] - for p in all_posts: - key = p.get('content', '')[:100] - if key not in seen: - seen.add(key) - deduped.append(p) - - deduped = [p for p in deduped if _is_within_days(p.get('date', ''), 2)] - leads = deduped[:20] - if leads: - leads = await classify_leads(leads, tutoring=tutoring) - - return {"success": True, "leads": leads[:15], "flagged": False, "flag_reason": None, "error": None} + return {"success": True, "leads": leads[:15], "flagged": False, "flag_reason": None, "error": None} + finally: + try: + await context.close() + except Exception: + pass except Exception as e: logger.error("%s scrape failed: %s", browser, e) @@ -1715,7 +1687,7 @@ async def _scrape_with_chromium(profile_path: str, browser: str, force: bool = F # Uses Chromium headless with the same launch args as _scrape_with_chromium. # The Agent is prompted to extract structured post data and return JSON. -async def _scrape_with_agent(force: bool = False) -> dict: +async def _scrape_with_agent(force: bool = False, query: str | None = None) -> dict: """Fallback scraper — browser-use Agent + ChatOllama (free/local, Chromium).""" cleanup_chrome() profile_dir = None @@ -1734,7 +1706,13 @@ async def _scrape_with_agent(force: bool = False) -> dict: await browser.start() all_posts = [] - pool = FB_SEARCHES + random.sample(SA_WEBSITE_QUERIES, k=min(4, len(SA_WEBSITE_QUERIES))) + tutoring_agent = False + if query: + tl = query.lower() + tutoring_agent = any(t in tl for t in ["tutor", "tutoring", "lessons", "homework", "teach", "learning", "child"]) + sa_dict = SA_TUTOR_QUERIES if tutoring_agent else SA_WEBSITE_QUERIES + sa_all = sa_dict.get("afrikaans", []) + sa_dict.get("xhosa", []) + sa_dict.get("zulu", []) + pool = FB_SEARCHES + sa_all for query in random.sample(pool, k=random.randint(2, 4)): agent = _make_agent( task=f"""You are logged into Facebook. Do the following: @@ -1780,7 +1758,7 @@ When done, return the data as a JSON list with keys: content, author, url, date. leads = deduped[:20] if leads: - leads = await classify_leads(leads, tutoring=tutoring) + leads = await classify_leads(leads, tutoring=tutoring_agent) return {"success": True, "leads": leads[:15], "flagged": False, "flag_reason": None, "error": None} except Exception as e: @@ -1820,7 +1798,7 @@ async def classify_leads(results: list[dict], tutoring: bool = False) -> list[di return [] # ── 1. AI classification ───────────────────────────────────────── - briefs = [r["title"][:200] for r in results] + briefs = [(r.get("title") or r.get("content") or "")[:200] for r in results] if tutoring: lead_desc = "someone REQUESTING/LOOKING FOR/WANTING a tutor, teacher, or lessons for their child or themselves" lead_examples = '"Looking for a tutor for my child", "Need a math tutor for my son", "Need help with homework", "Looking for piano lessons for my daughter", "Need a reading tutor"' @@ -2021,7 +1999,7 @@ Return a JSON array like ["yes","no","yes"] matching the order above.""" 'website design ideas', 'website inspiration', ] for r in results: - t = r['title'].lower() + t = (r.get('title') or r.get('content') or '').lower() has_target = any(kw in t for kw in target_terms) has_request = any(kw in t for kw in request_terms) if not has_target or not has_request: @@ -2033,11 +2011,11 @@ Return a JSON array like ["yes","no","yes"] matching the order above.""" keyword_leads.append(r) # ── 3. Merge: prefer AI leads, supplement with keywords ── - seen_titles: set[int] = set() + seen_titles: set[str] = set() merged: list[dict] = [] for r in ai_leads + keyword_leads: - key = hash(r.get('title', '')) - if key not in seen_titles: + key = (r.get('title') or '').strip()[:200] + if key and key not in seen_titles: seen_titles.add(key) merged.append(r) # Final sweep: strip any remaining offers or group posts from merged diff --git a/data/ai/ai.md b/data/ai/ai.md index e8909e7..9d7af79 100644 --- a/data/ai/ai.md +++ b/data/ai/ai.md @@ -34,12 +34,14 @@ The scraper lives at `browser-use-service/main.py` port 3008. ### How It Works 1. **Browser detection** — tries Firefox profile first, then Chromium-based (Chrome/Opera/Edge), falls back to browser-use Agent 2. **Profile paths** — configured via env vars (`FX_PROFILE`, `CHROME_PROFILE`, `OPERA_PROFILE`, `EDGE_PROFILE`) or auto-detected on first run -3. **Search dispatch** — per scrape run: - - 1 English primary search (full scroll with human-like delays) - - 2-3 English supplementary searches (quick searches) - - 6-7 non-English quick searches (Afrikaans, isiXhosa, isiZulu — 2 queries each per category) - - Total: ~14 searches per scrape, completed in 2-4 minutes -4. **Quick searches** — load page, double-scroll, extract visible posts (~12-18s each) +3. **4-phase language pipeline** (English → Afrikaans → Xhosa → Zulu): + - **Phase 1 (English)**: User's selected query + 2-3 supplementary English searches from the English search pool. First query gets full human-like scroll, rest use quick search. This phase does the heavy lifting. + - **Phase 2 (Afrikaans)**: 2 Afrikaans queries targeting Afrikaans-speaking communities. + - **Phase 3 (isiXhosa)**: 2 Xhosa queries targeting Xhosa-speaking communities. + - **Phase 4 (isiZulu)**: 2 Zulu queries targeting Zulu-speaking communities. + - After all phases: pipeline check (date filter 2 days → AI + keyword classification → sort by freshness). Newest leads ranked first. + - Each phase extracts posts, deduplicates against all prior phases, then passes through a stealth delay (5-12s + mouse idle) before the next phase. +4. **Quick searches** — load page, double-scroll, extract visible posts (~12-18s each). Scroll-back behavior (35% chance to scroll up) and random return-to-top (25% chance) for stealth. 5. **Date filter** — only posts within **2 days** are considered. Anything older is discarded. Fresh leads only. 6. **Stealth mechanics**: - Random viewport dimensions (1280×800 to 1920×1080) — never the same size twice @@ -70,12 +72,12 @@ Two categories, selectable when starting a scrape: - Request terms: same as website category — must co-occur with a target keyword - Strict reject: people offering tutoring, educational products, homeschool programs, free trials, general study tips -### Multi-Language Support -Searches in 4 South African languages: -- English — 1 primary + 2-3 supplementary queries -- Afrikaans — 2 queries (e.g., "ek benodig n webwerf", "ek soek n privaat onderwyser") -- isiXhosa — 2 queries (e.g., "ndidinga iwebhusayithi yeshishini", "ndifuna utitshala womntwana wam") -- isiZulu — 2 queries (e.g., "ngidinga iwebhusayithi yebhizinisi", "ngifuna umfundisi wengane") +### Multi-Language Pipeline (Phase Order) +4 South African languages in structured phases: +- **Phase 1 (English)**: primary query + supplementary English searches +- **Phase 2 (Afrikaans)**: 2 queries targeting Afrikaans speakers +- **Phase 3 (isiXhosa)**: 2 queries targeting Xhosa speakers +- **Phase 4 (isiZulu)**: 2 queries targeting Zulu speakers ### Output Format Each lead returned includes: diff --git a/rust-ai/src/AI.md b/rust-ai/src/AI.md index 624ce7d..3be0eae 100644 --- a/rust-ai/src/AI.md +++ b/rust-ai/src/AI.md @@ -34,12 +34,14 @@ The scraper lives at `browser-use-service/main.py` port 3008. ### How It Works 1. **Browser detection** — tries Firefox profile first, then Chromium-based (Chrome/Opera/Edge), falls back to browser-use Agent 2. **Profile paths** — configured via env vars (`FX_PROFILE`, `CHROME_PROFILE`, `OPERA_PROFILE`, `EDGE_PROFILE`) or auto-detected on first run -3. **Search dispatch** — per scrape run: - - 1 English primary search (full scroll with human-like delays) - - 2-3 English supplementary searches (quick searches) - - 6-7 non-English quick searches (Afrikaans, isiXhosa, isiZulu — 2 queries each per category) - - Total: ~14 searches per scrape, completed in 2-4 minutes -4. **Quick searches** — load page, double-scroll, extract visible posts (~12-18s each) +3. **4-phase language pipeline** (English → Afrikaans → Xhosa → Zulu): + - **Phase 1 (English)**: User's selected query + 2-3 supplementary English searches from the English search pool. First query gets full human-like scroll, rest use quick search. This phase does the heavy lifting. + - **Phase 2 (Afrikaans)**: 2 Afrikaans queries targeting Afrikaans-speaking communities. + - **Phase 3 (isiXhosa)**: 2 Xhosa queries targeting Xhosa-speaking communities. + - **Phase 4 (isiZulu)**: 2 Zulu queries targeting Zulu-speaking communities. + - After all phases: pipeline check (date filter 2 days → AI + keyword classification → sort by freshness). Newest leads ranked first. + - Each phase extracts posts, deduplicates against all prior phases, then passes through a stealth delay (5-12s + mouse idle) before the next phase. +4. **Quick searches** — load page, double-scroll, extract visible posts (~12-18s each). Scroll-back behavior (35% chance to scroll up) and random return-to-top (25% chance) for stealth. 5. **Date filter** — only posts within **2 days** are considered. Anything older is discarded. Fresh leads only. 6. **Stealth mechanics**: - Random viewport dimensions (1280×800 to 1920×1080) — never the same size twice @@ -70,12 +72,12 @@ Two categories, selectable when starting a scrape: - Request terms: same as website category — must co-occur with a target keyword - Strict reject: people offering tutoring, educational products, homeschool programs, free trials, general study tips -### Multi-Language Support -Searches in 4 South African languages: -- English — 1 primary + 2-3 supplementary queries -- Afrikaans — 2 queries (e.g., "ek benodig n webwerf", "ek soek n privaat onderwyser") -- isiXhosa — 2 queries (e.g., "ndidinga iwebhusayithi yeshishini", "ndifuna utitshala womntwana wam") -- isiZulu — 2 queries (e.g., "ngidinga iwebhusayithi yebhizinisi", "ngifuna umfundisi wengane") +### Multi-Language Pipeline (Phase Order) +4 South African languages in structured phases: +- **Phase 1 (English)**: primary query + supplementary English searches +- **Phase 2 (Afrikaans)**: 2 queries targeting Afrikaans speakers +- **Phase 3 (isiXhosa)**: 2 queries targeting Xhosa speakers +- **Phase 4 (isiZulu)**: 2 queries targeting Zulu speakers ### Output Format Each lead returned includes: diff --git a/src/app/(dashboard)/ai-assistant/page.tsx b/src/app/(dashboard)/ai-assistant/page.tsx index ea2e5cf..dddb24c 100644 --- a/src/app/(dashboard)/ai-assistant/page.tsx +++ b/src/app/(dashboard)/ai-assistant/page.tsx @@ -17,7 +17,7 @@ export default function AIAssistantPage() { const handleSearch = useCallback(async (job: NonNullable) => { setSearching(true) - const keyword = job.keywords[0] + const keyword = job.keywords?.[0] || job.job_title aiChatRef.current?.addAssistantMessage(`🔍 Searching Facebook for **${job.job_title}** leads...`) const controller = new AbortController() @@ -34,9 +34,12 @@ export default function AIAssistantPage() { clearTimeout(statusId) const data = await res.json() if (data.success && data.leads?.length > 0) { - const leadsText = data.leads.map((lead: any, i: number) => - `**${i + 1}.** ${lead.author || "Unknown"}\n> ${(lead.content || "").slice(0, 300)}\n> 🔗 ${lead.url || "(no link available)"}` - ).join("\n\n") + const leadLines = data.leads + .filter(Boolean) + .map((lead: Record, i: number) => + `**${i + 1}.** ${lead?.author || "Unknown"}\n> ${(lead?.content || "").slice(0, 300)}\n> 🔗 ${lead?.url || "(no link available)"}` + ) + const leadsText = leadLines.join("\n\n") aiChatRef.current?.addAssistantMessage(`✅ Found **${data.leads.length}** leads:\n\n${leadsText}`) } else { const reason = data.error || data.flag_reason || "No leads found this time" diff --git a/src/components/ai/job-selector.tsx b/src/components/ai/job-selector.tsx index fa00790..0b771a3 100644 --- a/src/components/ai/job-selector.tsx +++ b/src/components/ai/job-selector.tsx @@ -62,7 +62,7 @@ export function JobSelector({ onSelect, onSearch, searching }: JobSelectorProps) className="w-full text-left px-4 py-3 text-sm text-muted-foreground hover:bg-muted hover:text-foreground transition-all duration-150 border-b border-border/20 last:border-b-0 border-l-2 border-l-transparent hover:border-l-primary/40" >
{job.job_title}
-
{job.industry} — {job.description}
+
{job.industry} — {job.description}
))} {jobs.length === 0 && !loading && (