Added finishing touch on other languages

browser-use-service/main.py

@@ -666,38 +666,24 @@ TUTORING_SEARCHES = [
     "tutor near me for",
 ]
 
-def _search_list_for_query(query: str) -> list[str]:
-    """Pick the appropriate search query pool based on the search term."""
-    tl = query.lower()
-    tutoring_terms = ["tutor", "tutoring", "lessons", "homework", "teach", "learning", "child", "math", "english", "science", "exam", "homeschool", "coding", "programming", "piano", "reading"]
-    if any(t in tl for t in tutoring_terms):
-        return TUTORING_SEARCHES
-    return FB_SEARCHES
-
 # ── South African Multi-Language Queries ──────────────────────────────
-# 4 most spoken SA languages: English, Afrikaans, isiXhosa, isiZulu.
-# Each scrape searches ALL 4 languages to catch leads across all
-# language communities on Facebook.
+# 4 SA languages grouped for phase-based scanning:
+# Phase 1: English → Phase 2: Afrikaans → Phase 3: isiXhosa → Phase 4: English (final sweep)
+# Each language group has dedicated queries per category.
 
-SA_WEBSITE_QUERIES = [
-    "I need a website for my business",          # English
-    "ek benodig n webwerf",                       # Afrikaans
-    "ek soek iemand om n webwerf te bou",         # Afrikaans
-    "ndidinga iwebhusayithi yeshishini",          # isiXhosa
-    "ndifuna umntu owakha iwebhusayithi",         # isiXhosa
-    "ngidinga iwebhusayithi yebhizinisi",         # isiZulu
-    "ngifuna umuntu owakha iwebhusayithi",        # isiZulu
-]
+SA_WEBSITE_QUERIES = {
+    "english": ["I need a website for my business"],
+    "afrikaans": ["ek benodig n webwerf", "ek soek iemand om n webwerf te bou"],
+    "xhosa": ["ndidinga iwebhusayithi yeshishini", "ndifuna umntu owakha iwebhusayithi"],
+    "zulu": ["ngidinga iwebhusayithi yebhizinisi", "ngifuna umuntu owakha iwebhusayithi"],
+}
 
-SA_TUTOR_QUERIES = [
-    "I need a tutor for my child",                # English
-    "ek benodig n tutor vir my kind",             # Afrikaans
-    "ek soek n privaat onderwyser",               # Afrikaans
-    "ndifuna utitshala womntwana wam",            # isiXhosa
-    "ndidinga umfundisi-ntsapho",                 # isiXhosa
-    "ngidinga uthisha wengane yami",              # isiZulu
-    "ngifuna umfundisi wengane",                  # isiZulu
-]
+SA_TUTOR_QUERIES = {
+    "english": ["I need a tutor for my child"],
+    "afrikaans": ["ek benodig n tutor vir my kind", "ek soek n privaat onderwyser"],
+    "xhosa": ["ndifuna utitshala womntwana wam", "ndidinga umfundisi-ntsapho"],
+    "zulu": ["ngidinga uthisha wengane yami", "ngifuna umfundisi wengane"],
+}
 
 
 async def _quick_search(page, context, query: str) -> tuple:
@@ -1361,6 +1347,94 @@ def cleanup_chrome():
         pass
 
 
+# ── 4-Phase Language Pipeline ─────────────────────────────────────────
+# Runs searches in ordered language phases:
+#   Phase 1: English (main query + supplementary EN searches)
+#   Phase 2: Afrikaans
+#   Phase 3: isiXhosa
+#   Phase 4: English (final sweep with different EN queries)
+# Each phase extracts posts and deduplicates on the fly.
+# Pipeline check (classify_leads) runs at end to quality-filter.
+
+PHASE_ORDER = ["english", "afrikaans", "xhosa", "zulu"]
+
+
+async def _run_phases(page, context, query: str | None = None) -> list[dict]:
+    """4-phase language pipeline: English → Afrikaans → Xhosa → Zulu.
+    Each phase finds leads in that language. Pipeline check at end filters + sorts by freshness.
+    Returns top 10 newest, most relevant leads."""
+    all_posts: list[dict] = []
+    seen_keys: set[str] = set()
+    tutoring = False
+
+    if query:
+        tl = query.lower()
+        tutoring = any(t in tl for t in ["tutor", "tutoring", "lessons", "homework", "teach", "learning", "child"])
+
+    lang_pool = SA_TUTOR_QUERIES if tutoring else SA_WEBSITE_QUERIES
+    en_pool = TUTORING_SEARCHES if tutoring else FB_SEARCHES
+
+    # Build phase query lists
+    supplement = random.sample(en_pool, k=min(3, len(en_pool))) if query else []
+    phase_queries: list[list[str]] = []
+
+    # Phase 1: English (user query + supplement from EN pool)
+    phase_queries.append(([query] + supplement) if query else random.sample(en_pool, k=min(4, len(en_pool))))
+
+    # Phase 2: Afrikaans
+    phase_queries.append(lang_pool.get("afrikaans", []))
+
+    # Phase 3: isiXhosa
+    phase_queries.append(lang_pool.get("xhosa", []))
+
+    # Phase 4: isiZulu
+    phase_queries.append(lang_pool.get("zulu", []))
+
+    # Execute phases
+    for phase_idx, queries in enumerate(phase_queries):
+        if not queries:
+            continue
+        phase_posts: list[dict] = []
+        for i, q in enumerate(queries):
+            is_first = (phase_idx == 0 and i == 0 and query is not None)
+            if is_first:
+                page, posts = await search_facebook(page, context, q)
+            else:
+                page, posts = await _quick_search(page, context, q)
+
+            for p in posts:
+                key = p.get('content', '')[:100]
+                if key and key not in seen_keys:
+                    seen_keys.add(key)
+                    phase_posts.append(p)
+
+        all_posts.extend(phase_posts)
+
+        # Stealth delay between phases (not after last)
+        if phase_idx < len(phase_queries) - 1 and phase_posts:
+            await page.wait_for_timeout(random.uniform(5000, 12000))
+            if random.random() < 0.2:
+                await random_idle(page)
+
+    # Pipeline check: date filter (2 days max) + AI/keyword classification
+    all_posts = [p for p in all_posts if _is_within_days(p.get('date', ''), 2)]
+
+    leads = all_posts[:20]
+    if leads:
+        leads = await classify_leads(leads, tutoring=tutoring)
+
+    # Sort by freshness — newest leads first
+    def _sort_key(l):
+        try:
+            return datetime.strptime((l.get('date') or '').strip()[:10], '%Y-%m-%d')
+        except (ValueError, IndexError):
+            return datetime.min
+
+    leads.sort(key=_sort_key, reverse=True)
+
+    return leads[:10]
+
+
 # ── Main Scrape Dispatcher ────────────────────────────────────────────
 # scrape_facebook() is the main entry point. It:
 #   1. Resolves the browser profile path (from SELECTED_BROWSER env var or auto-detect)
@@ -1415,17 +1489,17 @@ async def scrape_facebook(profile_path: str | None = None, force: bool = False,
     # Firefox path
     if browser_type == "firefox":
         result = await _scrape_with_firefox(effective_path, force, query)
-        if result.get("success") or not result.get("flagged"):
+        if result.get("success"):
             return result
-        logger.warning("Firefox flagged (%s), trying Agent", result.get("flag_reason", "unknown"))
-        return await _scrape_with_agent(force)
+        logger.warning("Firefox failed (reason: %s), trying Agent", result.get("flag_reason") or result.get("error", "unknown"))
+        return await _scrape_with_agent(force, query)
 
     # Chromium-based (chrome / opera / edge)
     result = await _scrape_with_chromium(effective_path, browser_type, force, query)
-    if result.get("success") or not result.get("flagged"):
+    if result.get("success"):
         return result
-    logger.warning("%s flagged (%s), trying Agent", browser_type, result.get("flag_reason", "unknown"))
-    return await _scrape_with_agent(force)
+    logger.warning("%s failed (reason: %s), trying Agent", browser_type, result.get("flag_reason") or result.get("error", "unknown"))
+    return await _scrape_with_agent(force, query)
 
 
 # ── Firefox Scraper ──────────────────────────────────────────────────
@@ -1468,87 +1542,35 @@ async def _scrape_with_firefox(profile_path: str, force: bool, query: str | None
             except Exception:
                 logger.warning("Google navigation failed, trying Facebook directly")
 
-            await page.goto('https://www.facebook.com/', wait_until='domcontentloaded', timeout=30000)
-            await page.wait_for_timeout(random.randint(3000, 8000))
+            try:
+                await page.goto('https://www.facebook.com/', wait_until='domcontentloaded', timeout=30000)
+                await page.wait_for_timeout(random.randint(3000, 8000))
 
-            url = page.url
-            page_text = await page.evaluate('document.body.innerText') if '/login' in url.lower() else ''
-            det = check_detection_signals(url, page_text)
-            if det or '/login' in url.lower():
-                logger.warning("Facebook login page detected — flag: %s", det or "login_page")
-                await context.close()
-                return {"success": False, "leads": [], "flagged": True, "flag_reason": det or "login_page", "error": "Facebook login page detected"}
+                url = page.url
+                page_text = await page.evaluate('document.body.innerText') if '/login' in url.lower() else ''
+                det = check_detection_signals(url, page_text)
+                if det or '/login' in url.lower():
+                    logger.warning("Facebook login page detected — flag: %s", det or "login_page")
+                    return {"success": False, "leads": [], "flagged": True, "flag_reason": det or "login_page", "error": "Facebook login page detected"}
 
-            await human_scroll(page, steps=random.randint(2, 4), total_delay=random.uniform(8, 20))
-            if random.random() < 0.25:
-                await page.evaluate("window.scrollTo(0, 0)")
-                await page.wait_for_timeout(random.randint(2000, 5000))
-                await human_scroll(page, steps=random.randint(1, 2))
+                await human_scroll(page, steps=random.randint(2, 4), total_delay=random.uniform(8, 20))
+                if random.random() < 0.25:
+                    await page.evaluate("window.scrollTo(0, 0)")
+                    await page.wait_for_timeout(random.randint(2000, 5000))
+                    await human_scroll(page, steps=random.randint(1, 2))
 
-            if not force and random.random() < 0.3:
-                await page.wait_for_timeout(random.randint(8000, 20000))
-                await context.close()
-                return {"success": True, "leads": [], "flagged": False, "flag_reason": None, "error": None}
+                if not force and random.random() < 0.3:
+                    await page.wait_for_timeout(random.randint(8000, 20000))
+                    return {"success": True, "leads": [], "flagged": False, "flag_reason": None, "error": None}
 
-            all_posts = []
-            tutoring = False
-            if query:
-                tl = query.lower()
-                tutoring = any(t in tl for t in ["tutor", "tutoring", "lessons", "homework", "teach", "learning", "child"])
-                lang_pool = SA_TUTOR_QUERIES if tutoring else SA_WEBSITE_QUERIES
-                non_english = [q for q in lang_pool if q.strip().lower() != tl]
-                query_pool = _search_list_for_query(query)
-                supp_k = random.randint(3, 4) if tutoring else random.randint(2, 3)
-                supplement = random.sample(query_pool, k=supp_k)
-                searches = [query] + supplement + non_english
-                english_count = 1 + len(supplement)
-            else:
-                searches = random.sample(FB_SEARCHES + SA_WEBSITE_QUERIES + SA_TUTOR_QUERIES, k=random.randint(4, 7))
-            for i, sq in enumerate(searches):
-                page, posts = await search_facebook(page, context, sq) if i == 0 else await _quick_search(page, context, sq)
-                all_posts.extend(posts)
-                if not posts:
-                    continue
-                if i > 0:
-                    await page.wait_for_timeout(random.uniform(5000, 12000))
-                    if random.random() < 0.2:
-                        await random_idle(page)
-                    continue
-                if random.random() < 0.4:
-                    await page.evaluate(f"window.scrollBy(0, {random.randint(-300, 300)})")
-                delay = random.uniform(8, 25)
-                await page.wait_for_timeout(int(delay * 1000))
-                if i == random.randint(0, 1) and random.random() < 0.15:
-                    new_page = await context.new_page()
-                    try:
-                        await new_page.goto('https://www.facebook.com/groups/', wait_until='domcontentloaded', timeout=15000)
-                        await new_page.wait_for_timeout(random.randint(3000, 8000))
-                    except Exception:
-                        pass
-                    await new_page.close()
-                    page = await _ensure_page(page, context)
+                leads = await _run_phases(page, context, query)
 
-            if random.random() < 0.5:
-                await page.wait_for_timeout(random.randint(3000, 10000))
-
-            await context.close()
-
-            seen = set()
-            deduped = []
-            for p in all_posts:
-                key = p.get('content', '')[:100]
-                if key not in seen:
-                    seen.add(key)
-                    deduped.append(p)
-
-            # Filter to last 2 days only
-            deduped = [p for p in deduped if _is_within_days(p.get('date', ''), 2)]
-
-            leads = deduped[:20]
-            if leads:
-                leads = await classify_leads(leads, tutoring=tutoring)
-
-            return {"success": True, "leads": leads[:15], "flagged": False, "flag_reason": None, "error": None}
+                return {"success": True, "leads": leads[:15], "flagged": False, "flag_reason": None, "error": None}
+            finally:
+                try:
+                    await context.close()
+                except Exception:
+                    pass
 
     except Exception as e:
         logger.error("Firefox scrape failed: %s", e)
@@ -1617,85 +1639,35 @@ async def _scrape_with_chromium(profile_path: str, browser: str, force: bool = F
             except Exception:
                 logger.warning("Google navigation failed, trying Facebook directly")
 
-            await page.goto('https://www.facebook.com/', wait_until='domcontentloaded', timeout=30000)
-            await page.wait_for_timeout(random.randint(3000, 8000))
+            try:
+                await page.goto('https://www.facebook.com/', wait_until='domcontentloaded', timeout=30000)
+                await page.wait_for_timeout(random.randint(3000, 8000))
 
-            url = page.url
-            page_text = await page.evaluate('document.body.innerText') if '/login' in url.lower() else ''
-            det = check_detection_signals(url, page_text)
-            if det or '/login' in url.lower():
-                logger.warning("Facebook login page detected — flag: %s", det or "login_page")
-                await context.close()
-                return {"success": False, "leads": [], "flagged": True, "flag_reason": det or "login_page", "error": "Facebook login page detected"}
+                url = page.url
+                page_text = await page.evaluate('document.body.innerText') if '/login' in url.lower() else ''
+                det = check_detection_signals(url, page_text)
+                if det or '/login' in url.lower():
+                    logger.warning("Facebook login page detected — flag: %s", det or "login_page")
+                    return {"success": False, "leads": [], "flagged": True, "flag_reason": det or "login_page", "error": "Facebook login page detected"}
 
-            await human_scroll(page, steps=random.randint(2, 4), total_delay=random.uniform(8, 20))
-            if random.random() < 0.25:
-                await page.evaluate("window.scrollTo(0, 0)")
-                await page.wait_for_timeout(random.randint(2000, 5000))
-                await human_scroll(page, steps=random.randint(1, 2))
+                await human_scroll(page, steps=random.randint(2, 4), total_delay=random.uniform(8, 20))
+                if random.random() < 0.25:
+                    await page.evaluate("window.scrollTo(0, 0)")
+                    await page.wait_for_timeout(random.randint(2000, 5000))
+                    await human_scroll(page, steps=random.randint(1, 2))
 
-            if not force and random.random() < 0.3:
-                await page.wait_for_timeout(random.randint(8000, 20000))
-                await context.close()
-                return {"success": True, "leads": [], "flagged": False, "flag_reason": None, "error": None}
+                if not force and random.random() < 0.3:
+                    await page.wait_for_timeout(random.randint(8000, 20000))
+                    return {"success": True, "leads": [], "flagged": False, "flag_reason": None, "error": None}
 
-            all_posts = []
-            tutoring = False
-            if query:
-                tl = query.lower()
-                tutoring = any(t in tl for t in ["tutor", "tutoring", "lessons", "homework", "teach", "learning", "child"])
-                lang_pool = SA_TUTOR_QUERIES if tutoring else SA_WEBSITE_QUERIES
-                non_english = [q for q in lang_pool if q.strip().lower() != tl]
-                query_pool = _search_list_for_query(query)
-                supp_k = random.randint(3, 4) if tutoring else random.randint(2, 3)
-                supplement = random.sample(query_pool, k=supp_k)
-                searches = [query] + supplement + non_english
-                english_count = 1 + len(supplement)
-            else:
-                searches = random.sample(FB_SEARCHES + SA_WEBSITE_QUERIES + SA_TUTOR_QUERIES, k=random.randint(4, 7))
-            for i, sq in enumerate(searches):
-                page, posts = await search_facebook(page, context, sq) if i == 0 else await _quick_search(page, context, sq)
-                all_posts.extend(posts)
-                if not posts:
-                    continue
-                if i > 0:
-                    await page.wait_for_timeout(random.uniform(5000, 12000))
-                    if random.random() < 0.2:
-                        await random_idle(page)
-                    continue
-                if random.random() < 0.4:
-                    await page.evaluate(f"window.scrollBy(0, {random.randint(-300, 300)})")
-                delay = random.uniform(8, 25)
-                await page.wait_for_timeout(int(delay * 1000))
-                if i == random.randint(0, 1) and random.random() < 0.15:
-                    new_page = await context.new_page()
-                    try:
-                        await new_page.goto('https://www.facebook.com/groups/', wait_until='domcontentloaded', timeout=15000)
-                        await new_page.wait_for_timeout(random.randint(3000, 8000))
-                    except Exception:
-                        pass
-                    await new_page.close()
-                    page = await _ensure_page(page, context)
+                leads = await _run_phases(page, context, query)
 
-            if random.random() < 0.5:
-                await page.wait_for_timeout(random.randint(3000, 10000))
-
-            await context.close()
-
-            seen = set()
-            deduped = []
-            for p in all_posts:
-                key = p.get('content', '')[:100]
-                if key not in seen:
-                    seen.add(key)
-                    deduped.append(p)
-
-            deduped = [p for p in deduped if _is_within_days(p.get('date', ''), 2)]
-            leads = deduped[:20]
-            if leads:
-                leads = await classify_leads(leads, tutoring=tutoring)
-
-            return {"success": True, "leads": leads[:15], "flagged": False, "flag_reason": None, "error": None}
+                return {"success": True, "leads": leads[:15], "flagged": False, "flag_reason": None, "error": None}
+            finally:
+                try:
+                    await context.close()
+                except Exception:
+                    pass
 
     except Exception as e:
         logger.error("%s scrape failed: %s", browser, e)
@@ -1715,7 +1687,7 @@ async def _scrape_with_chromium(profile_path: str, browser: str, force: bool = F
 # Uses Chromium headless with the same launch args as _scrape_with_chromium.
 # The Agent is prompted to extract structured post data and return JSON.
 
-async def _scrape_with_agent(force: bool = False) -> dict:
+async def _scrape_with_agent(force: bool = False, query: str | None = None) -> dict:
     """Fallback scraper — browser-use Agent + ChatOllama (free/local, Chromium)."""
     cleanup_chrome()
     profile_dir = None
@@ -1734,7 +1706,13 @@ async def _scrape_with_agent(force: bool = False) -> dict:
         await browser.start()
 
         all_posts = []
-        pool = FB_SEARCHES + random.sample(SA_WEBSITE_QUERIES, k=min(4, len(SA_WEBSITE_QUERIES)))
+        tutoring_agent = False
+        if query:
+            tl = query.lower()
+            tutoring_agent = any(t in tl for t in ["tutor", "tutoring", "lessons", "homework", "teach", "learning", "child"])
+        sa_dict = SA_TUTOR_QUERIES if tutoring_agent else SA_WEBSITE_QUERIES
+        sa_all = sa_dict.get("afrikaans", []) + sa_dict.get("xhosa", []) + sa_dict.get("zulu", [])
+        pool = FB_SEARCHES + sa_all
         for query in random.sample(pool, k=random.randint(2, 4)):
             agent = _make_agent(
                 task=f"""You are logged into Facebook. Do the following:
@@ -1780,7 +1758,7 @@ When done, return the data as a JSON list with keys: content, author, url, date.
 
         leads = deduped[:20]
         if leads:
-            leads = await classify_leads(leads, tutoring=tutoring)
+            leads = await classify_leads(leads, tutoring=tutoring_agent)
 
         return {"success": True, "leads": leads[:15], "flagged": False, "flag_reason": None, "error": None}
     except Exception as e:
@@ -1820,7 +1798,7 @@ async def classify_leads(results: list[dict], tutoring: bool = False) -> list[di
         return []
 
     # ── 1. AI classification ─────────────────────────────────────────
-    briefs = [r["title"][:200] for r in results]
+    briefs = [(r.get("title") or r.get("content") or "")[:200] for r in results]
     if tutoring:
         lead_desc = "someone REQUESTING/LOOKING FOR/WANTING a tutor, teacher, or lessons for their child or themselves"
         lead_examples = '"Looking for a tutor for my child", "Need a math tutor for my son", "Need help with homework", "Looking for piano lessons for my daughter", "Need a reading tutor"'
@@ -2021,7 +1999,7 @@ Return a JSON array like ["yes","no","yes"] matching the order above."""
         'website design ideas', 'website inspiration',
     ]
     for r in results:
-        t = r['title'].lower()
+        t = (r.get('title') or r.get('content') or '').lower()
         has_target = any(kw in t for kw in target_terms)
         has_request = any(kw in t for kw in request_terms)
         if not has_target or not has_request:
@@ -2033,11 +2011,11 @@ Return a JSON array like ["yes","no","yes"] matching the order above."""
         keyword_leads.append(r)
 
     # ── 3. Merge: prefer AI leads, supplement with keywords ──
-    seen_titles: set[int] = set()
+    seen_titles: set[str] = set()
     merged: list[dict] = []
     for r in ai_leads + keyword_leads:
-        key = hash(r.get('title', ''))
-        if key not in seen_titles:
+        key = (r.get('title') or '').strip()[:200]
+        if key and key not in seen_titles:
             seen_titles.add(key)
             merged.append(r)
     # Final sweep: strip any remaining offers or group posts from merged