Added 4 Languages English, Afrikaans, Xhosa, Zulu, tested but brought leads down to 3 leads only will see how to make it more without losing effiency of the model.

2026-07-10 11:15:43 +02:00 · 2026-07-07 10:03:35 +02:00
parent c1c4afadbb
commit d793604e92
6 changed files with 509 additions and 131 deletions
@@ -34,7 +34,7 @@ logger = logging.getLogger(__name__)
 app = FastAPI()
 app.add_middleware(
    CORSMiddleware,
-    allow_origins=["http://localhost:3006", "http://127.0.0.1:3006"],
+    allow_origins=os.getenv("CORS_ORIGINS", "http://localhost:3006,http://127.0.0.1:3006").split(","),
    allow_methods=["POST"],
    allow_headers=["*"],
 )
@@ -674,6 +674,107 @@ def _search_list_for_query(query: str) -> list[str]:
        return TUTORING_SEARCHES
    return FB_SEARCHES

+# ── South African Multi-Language Queries ──────────────────────────────
+# 4 most spoken SA languages: English, Afrikaans, isiXhosa, isiZulu.
+# Each scrape searches ALL 4 languages to catch leads across all
+# language communities on Facebook.
+
+SA_WEBSITE_QUERIES = [
+    "I need a website for my business",          # English
+    "ek benodig n webwerf",                       # Afrikaans
+    "ek soek iemand om n webwerf te bou",         # Afrikaans
+    "ndidinga iwebhusayithi yeshishini",          # isiXhosa
+    "ndifuna umntu owakha iwebhusayithi",         # isiXhosa
+    "ngidinga iwebhusayithi yebhizinisi",         # isiZulu
+    "ngifuna umuntu owakha iwebhusayithi",        # isiZulu
+]
+
+SA_TUTOR_QUERIES = [
+    "I need a tutor for my child",                # English
+    "ek benodig n tutor vir my kind",             # Afrikaans
+    "ek soek n privaat onderwyser",               # Afrikaans
+    "ndifuna utitshala womntwana wam",            # isiXhosa
+    "ndidinga umfundisi-ntsapho",                 # isiXhosa
+    "ngidinga uthisha wengane yami",              # isiZulu
+    "ngifuna umfundisi wengane",                  # isiZulu
+]
+
+
+async def _quick_search(page, context, query: str) -> tuple:
+    """Fast search — load search results page, wait for render, extract visible posts.
+    No scrolling or extra human-like delays. Used for non-English language queries."""
+    page = await _ensure_page(page, context)
+    url = f'https://www.facebook.com/search/posts/?q={urllib.parse.quote(query)}'
+    try:
+        await page.goto(url, wait_until='domcontentloaded', timeout=20000)
+        current_url = page.url
+        if '/login' in current_url.lower():
+            logger.warning("Quick search redirected to login for '%s'", query[:40])
+            return page, []
+        await page.wait_for_timeout(random.randint(4000, 6000))
+        await page.evaluate("window.scrollBy(0, 600)")
+        await page.wait_for_timeout(random.randint(2000, 4000))
+        await page.evaluate("window.scrollBy(0, 600)")
+        await page.wait_for_timeout(random.randint(2000, 3000))
+        raw_articles = await _get_article_elements(page)
+        posts = _extract_posts_from_elements(raw_articles, url) if raw_articles else []
+        raw = await page.evaluate('document.body.innerText')
+        text_posts = _extract_posts_from_text(raw, url)
+        existing = {(p.get('title') or p.get('content',''))[:80] for p in posts}
+        for tp in text_posts:
+            key = (tp.get('title') or tp.get('content',''))[:80]
+            if key not in existing:
+                posts.append(tp)
+        if posts:
+            try:
+                profiles = await page.evaluate(r'''() => {
+                    const out = [];
+                    const seenTxt = new Set();
+                    for (const a of document.querySelectorAll('a[href*="/profile.php"], a[href*="/user/"], a[href*="/people/"], a[href*="/me/"]')) {
+                        const name = (a.innerText || '').trim();
+                        if (!name || name.length < 3 || name.length > 60) continue;
+                        const words = name.split(' ');
+                        if (words.length < 2 || words.length > 6) continue;
+                        if (!/^[A-Z]/.test(name)) continue;
+                        if (name.includes('facebook') || name.includes('/')) continue;
+                        const cell = a.closest('div[style]') || a.parentElement;
+                        const txt = cell ? (cell.innerText || '').substring(0, 200) : '';
+                        if (!txt) continue;
+                        const key2 = txt.substring(0, 80);
+                        if (seenTxt.has(key2)) continue;
+                        seenTxt.add(key2);
+                        out.push({ name, textKey: key2 });
+                    }
+                    return out;
+                }''')
+                if profiles:
+                    for p in posts:
+                        pk = (p.get('content') or p.get('title') or '')[:80].strip()
+                        if not pk: continue
+                        for pr in profiles:
+                            if pk[:30] in pr['textKey'] or pr['textKey'][:30] in pk:
+                                if not p.get('author'):
+                                    p['author'] = pr['name']
+                                break
+            except Exception:
+                pass
+        for p in posts:
+            if p.get('author'):
+                a = p['author']
+                al = a.lower()
+                if any(kw in al for kw in BROAD_KEYWORDS) or is_offer(a) or len(a.split()) < 2 or any(w in _NON_NAMES for w in al.split()):
+                    p['author'] = ''
+        posts = [p for p in posts if not (
+            '/groups/' in p.get('url', '') or '/group/' in p.get('url', '')
+            or '/pages/' in p.get('url', '')
+            or ' / ' in (p.get('title') or p.get('content') or '')
+        )]
+    except Exception as e:
+        logger.warning("Quick search '%s' failed: %s", query, e)
+        return page, []
+    return page, posts
+
+
 VIEWPORTS = [
    {'width': 1280, 'height': 800},
    {'width': 1366, 'height': 768},
@@ -1383,16 +1484,27 @@ async def _scrape_with_firefox(profile_path: str, force: bool, query: str | None
                return {"success": True, "leads": [], "flagged": False, "flag_reason": None, "error": None}

            all_posts = []
+            tutoring = False
            if query:
+                tl = query.lower()
+                tutoring = any(t in tl for t in ["tutor", "tutoring", "lessons", "homework", "teach", "learning", "child"])
+                lang_pool = SA_TUTOR_QUERIES if tutoring else SA_WEBSITE_QUERIES
+                non_english = [q for q in lang_pool if q.strip().lower() != tl]
                query_pool = _search_list_for_query(query)
-                searches = [query] + random.sample(query_pool, k=random.randint(1, 2))
+                supp_k = random.randint(3, 4) if tutoring else random.randint(2, 3)
+                supplement = random.sample(query_pool, k=supp_k)
+                searches = [query] + supplement + non_english
+                english_count = 1 + len(supplement)
            else:
-                searches = random.sample(FB_SEARCHES, k=random.randint(2, 4))
+                searches = random.sample(FB_SEARCHES + SA_WEBSITE_QUERIES + SA_TUTOR_QUERIES, k=random.randint(4, 7))
            for i, sq in enumerate(searches):
-                page, posts = await search_facebook(page, context, sq)
+                page, posts = await search_facebook(page, context, sq) if i == 0 else await _quick_search(page, context, sq)
                all_posts.extend(posts)
                if not posts:
                    continue
+                if i > 0:
+                    await page.wait_for_timeout(random.uniform(3000, 7000))
+                    continue
                if random.random() < 0.4:
                    await page.evaluate(f"window.scrollBy(0, {random.randint(-300, 300)})")
                delay = random.uniform(8, 25)
@@ -1421,11 +1533,11 @@ async def _scrape_with_firefox(profile_path: str, force: bool, query: str | None
                    deduped.append(p)

            # Filter to last 3 days only
-            deduped = [p for p in deduped if _is_within_days(p.get('date', ''), 3)]
+            deduped = [p for p in deduped if _is_within_days(p.get('date', ''), 7)]

            leads = deduped[:20]
            if leads:
-                leads = await classify_leads(leads)
+                leads = await classify_leads(leads, tutoring=tutoring)

            return {"success": True, "leads": leads[:15], "flagged": False, "flag_reason": None, "error": None}

@@ -1518,16 +1630,27 @@ async def _scrape_with_chromium(profile_path: str, browser: str, force: bool = F
                return {"success": True, "leads": [], "flagged": False, "flag_reason": None, "error": None}

            all_posts = []
+            tutoring = False
            if query:
+                tl = query.lower()
+                tutoring = any(t in tl for t in ["tutor", "tutoring", "lessons", "homework", "teach", "learning", "child"])
+                lang_pool = SA_TUTOR_QUERIES if tutoring else SA_WEBSITE_QUERIES
+                non_english = [q for q in lang_pool if q.strip().lower() != tl]
                query_pool = _search_list_for_query(query)
-                searches = [query] + random.sample(query_pool, k=random.randint(1, 2))
+                supp_k = random.randint(3, 4) if tutoring else random.randint(2, 3)
+                supplement = random.sample(query_pool, k=supp_k)
+                searches = [query] + supplement + non_english
+                english_count = 1 + len(supplement)
            else:
-                searches = random.sample(FB_SEARCHES, k=random.randint(2, 4))
+                searches = random.sample(FB_SEARCHES + SA_WEBSITE_QUERIES + SA_TUTOR_QUERIES, k=random.randint(4, 7))
            for i, sq in enumerate(searches):
-                page, posts = await search_facebook(page, context, sq)
+                page, posts = await search_facebook(page, context, sq) if i == 0 else await _quick_search(page, context, sq)
                all_posts.extend(posts)
                if not posts:
                    continue
+                if i > 0:
+                    await page.wait_for_timeout(random.uniform(3000, 7000))
+                    continue
                if random.random() < 0.4:
                    await page.evaluate(f"window.scrollBy(0, {random.randint(-300, 300)})")
                delay = random.uniform(8, 25)
@@ -1555,10 +1678,10 @@ async def _scrape_with_chromium(profile_path: str, browser: str, force: bool = F
                    seen.add(key)
                    deduped.append(p)

-            deduped = [p for p in deduped if _is_within_days(p.get('date', ''), 3)]
+            deduped = [p for p in deduped if _is_within_days(p.get('date', ''), 7)]
            leads = deduped[:20]
            if leads:
-                leads = await classify_leads(leads)
+                leads = await classify_leads(leads, tutoring=tutoring)

            return {"success": True, "leads": leads[:15], "flagged": False, "flag_reason": None, "error": None}

@@ -1599,7 +1722,8 @@ async def _scrape_with_agent(force: bool = False) -> dict:
        await browser.start()

        all_posts = []
-        for query in random.sample(FB_SEARCHES, k=random.randint(2, 4)):
+        pool = FB_SEARCHES + random.sample(SA_WEBSITE_QUERIES, k=min(4, len(SA_WEBSITE_QUERIES)))
+        for query in random.sample(pool, k=random.randint(2, 4)):
            agent = _make_agent(
                task=f"""You are logged into Facebook. Do the following:
 1. Navigate to facebook.com and make sure you are on the homepage
@@ -1640,11 +1764,11 @@ When done, return the data as a JSON list with keys: content, author, url, date.
                deduped.append(p)

        # Filter to last 3 days only
-        deduped = [p for p in deduped if _is_within_days(p.get('date', ''), 3)]
+        deduped = [p for p in deduped if _is_within_days(p.get('date', ''), 7)]

        leads = deduped[:20]
        if leads:
-            leads = await classify_leads(leads)
+            leads = await classify_leads(leads, tutoring=tutoring)

        return {"success": True, "leads": leads[:15], "flagged": False, "flag_reason": None, "error": None}
    except Exception as e:
@@ -1679,24 +1803,37 @@ async def ask_ollama(prompt: str) -> str:
        data = r.json()
        return data["message"]["content"]

-async def classify_leads(results: list[dict]) -> list[dict]:
+async def classify_leads(results: list[dict], tutoring: bool = False) -> list[dict]:
    if not results:
        return []

    # ── 1. AI classification ─────────────────────────────────────────
    briefs = [r["title"][:200] for r in results]
+    if tutoring:
+        lead_desc = "someone REQUESTING/LOOKING FOR/WANTING a tutor, teacher, or lessons for their child or themselves"
+        lead_examples = '"Looking for a tutor for my child", "Need a math tutor for my son", "Need help with homework", "Looking for piano lessons for my daughter", "Need a reading tutor"'
+        not_lead_examples = '"I offer tutoring services", "I am a tutor with experience", "Affordable tutoring packages", "Online tutor available"'
+        extra_terms = '- Posts about homeschooling resources, curriculum sales, or educational products\n- Posts asking for study tips or general academic advice without requesting a tutor'
+    else:
+        lead_desc = "someone REQUESTING/POSTING/WANTING a website built, designed, or created for them"
+        lead_examples = '"Need a website for my business", "Looking for web developer to build my site", "I need someone to create my website", "Want a new website for my company", "Looking for someone to design my WordPress site"'
+        not_lead_examples = '"I build websites", "I offer web design", "Affordable web design packages"'
+        extra_terms = '- "Need web hosting", "Looking for a partner", "Looking for content writer", "Video spokesperson"'
    prompt = f"""Classify each post as LEAD or NOT.
-LEAD = someone REQUESTING/POSTING/WANTING a website built, designed, or created for them.
-LEAD examples: "Need a website for my business", "Looking for web developer to build my site", "I need someone to create my website", "Want a new website for my company", "Looking for someone to design my WordPress site"
+LEAD = {lead_desc}.
+LEAD examples: {lead_examples}

 NOT LEAD:
- Offering web design services: "I build websites", "I offer web design", "Affordable web design packages"
+- Offering services: {not_lead_examples}
 - Already have a website and need marketing, SEO, content, video, link building, email marketing, affiliates
 - Recruiting employees, hiring staff, looking for business partners
 - Selling products, promoting services, affiliate offers
- "Need web hosting", "Looking for a partner", "Looking for content writer", "Video spokesperson"
+{extra_terms}
 - Posts from groups, communities, or pages (group announcements, group posts, page posts)
 - Posts containing the word "group", "page", "community", "creators" — these are NEVER individual leads
+- Vague questions or general recommendations without a clear intent to buy or hire
+- People asking how to learn or do it themselves (not looking to hire someone)
+- Posts about existing website issues like speed, SEO, errors, redesign advice — NOT a lead

 For each numbered post, answer ONLY "yes" (LEAD) or "no" (NOT LEAD):
 {chr(10).join(f'{i+1}. {t}' for i, t in enumerate(briefs))}
@@ -1721,32 +1858,70 @@ Return a JSON array like ["yes","no","yes"] matching the order above."""
    except Exception as e:
        logger.warning("AI classification failed: %s", e)

-    # ── 2. Keyword fallback (always runs) ────────────────────────────
-    web_terms = [
-        "website", "web design", "web develop", "web dev",
-        "web designer", "web developer",
-        "build my website", "build a website", "create a website",
-        "landing page", "wordpress", "ecommerce",
-        "my website", "business website",
-        "site for my", "site for my business",
-        "new website", "redesign my website",
-        "help with my website", "update my website",
-        "make a website", "make my website",
-        "website for my",
-        "online store", "online shop",
-        "build my site", "build a site",
-        "set up a website", "set up my website",
-        "custom website",
-        "shopify",
-        "my site",
-        "webpage", "web page",
-    ]
+    # ── 2. Keyword supplement (never overrides AI, only adds missing leads) ──
+    if tutoring:
+        target_terms = [
+            "tutor", "tutoring", "tutor for", "private tutor",
+            "math tutor", "english tutor", "reading tutor",
+            "science tutor", "online tutor", "home tutor",
+            "lessons for", "lessons for my", "piano lessons",
+            "swimming lessons", "music lessons",
+            "help with homework", "homework help",
+            "teacher for", "teacher for my",
+            "need help learning", "need help with",
+            "exam prep", "exam preparation",
+            "homeschool", "homeschool tutor",
+            "tuition",
+            "coding for my", "programming for my",
+            "looking for a tutor", "need a tutor",
+            "tutor needed", "tutoring for",
+            "private lessons", "private tuition",
+            "afterschool", "after school",
+            "extra classes", "extra lessons",
+        ]
+        offer_reject_tutor = [
+            'i am a tutor', "i'm a tutor", 'i offer tutoring',
+            'online tutor available', 'tutor available',
+            'i teach', 'i provide tutoring',
+            'affordable tutoring', 'tutoring services',
+            'experienced tutor', 'qualified tutor',
+            'your child', 'your kids', 'your children',
+            'enroll your', 'sign up',
+            'free trial', 'first lesson free',
+            'group lessons', 'group class',
+            'limited spots', 'book now',
+            'curriculum', 'workbook', 'worksheet',
+            'educational program',
+            'homeschool program', 'home school program',
+        ]
+    else:
+        target_terms = [
+            "website", "web design", "web develop", "web dev",
+            "web designer", "web developer",
+            "build my website", "build a website", "create a website",
+            "landing page", "wordpress", "ecommerce",
+            "my website", "business website",
+            "site for my", "site for my business",
+            "new website", "redesign my website",
+            "help with my website", "update my website",
+            "make a website", "make my website",
+            "website for my",
+            "online store", "online shop",
+            "build my site", "build a site",
+            "set up a website", "set up my website",
+            "custom website",
+            "shopify",
+            "my site",
+            "webpage", "web page",
+            "who can build", "who can design",
+            "create my website", "create my site",
+        ]
+        offer_reject_tutor = []
    request_terms = [
        "looking for", "need a", "need an", "looking to",
        "need someone", "hire a", "want someone",
        "need help with", "would like", "build me",
        "design my", "make me a", "create my",
-        "looking", "need", "want", "help",
        "who can", "i need",
        "recommend", "anyone know", "anyone recommend",
        "know a", "know any", "recommendation",
@@ -1768,27 +1943,29 @@ Return a JSON array like ["yes","no","yes"] matching the order above."""
        'whatsapp me', 'looking for a business', 'looking for client',
        'help your business', 'i am a web', 'contact me',
        'we offer web', 'we provide web',
-        'take the quiz', 'homeschool', 'your home tutor',
+        'take the quiz',
        'link in bio', 'apply now', 'get started',
        'for only', 'low price', 'hit me up',
        'send me a message', 'i do website', 'we do website',
        'we do web', 'i do web',
        'website designer / web developer', 'website & software creators',
-        'website builders for small businesses', 'australia web designers',
-        'south africa', 'wix website design',
+        'website builders for small businesses',
+        'wix website design',
        'for sale', 'selling my', 'premium',
        'i\'m selling', 'i\'m offering', 'we\'re offering',
        'free ecommerce', 'free website design',
        'starting a', 'looking for a few businesses',
        # Group-related rejections
-        'group', ' i need a website group', 'south africa web', 'philippines web', 'australia web',
-        'i can help', 'inbox me', 'dm me', 'pm me', 'message me for',
+        'group', ' i need a website group',
+        'i can help', 'inbox me', 'message me for',
        'best price', 'discount', 'reach out', 'check out my', 'check this',
        'website for your', 'price start', 'price begin', 'website creator',
        'website & software', 'creators &', 'creators marketplace',
        'website group', 'page group',
+        'south africa web', 'philippines web', 'australia web',
+        'nigerian web', 'kenya web', 'india web',
        # Self-promotion rejections
-        'i\'m a web', "i'm a web", 'i am a full stack', "i'm a full stack", 'i\'m a full stack',
+        'i\'m a web', "i'm a web", 'i am a full stack', "i'm a full stack",
        'freelance opportunity', 'looking for new project', 'looking for new work',
        'full stack web', 'mern stack', 'responsive business website',
        'i build website', 'i build shopify', 'i build wordpress',
@@ -1802,18 +1979,48 @@ Return a JSON array like ["yes","no","yes"] matching the order above."""
        'for free', 'no coding', 'make money', 'website for free',
        'part time job', 'part time position',
        'years of experience', 'years of teaching',
+        # Service offers that slip through two-word check
+        'i am a full stack', 'i am a developer',
+        'i will design', 'i will build', 'i will create',
+        'i can design', 'i can create',
+        'we will design', 'we will build',
+        'hire me', 'i am available for',
+        'available for work', 'freelance web',
+        'i specialize in', 'we specialize in',
+        "here's my portfolio", 'check my portfolio',
+        'see my work', 'view my work',
+        'we have a team', 'my team',
+        'i am looking for clients', 'i am looking for work',
+        'looking for web development work',
+        'looking for new clients',
+        # People learning / doing it themselves (not hiring)
+        'learn web development', 'learn to code',
+        'how to build a website', 'how to create a website',
+        'how to make a website', 'how to design a website',
+        'where to start', 'online course',
+        'want to learn', 'learning web',
+        'best platform for', 'which platform',
+        # Existing website issues (not new build)
+        'my website is down', 'website not loading',
+        'website error', 'website problem',
+        'website troubleshooting',
+        'need website advice', 'website tips',
+        'help with seo', 'google ranking',
+        'website design ideas', 'website inspiration',
    ]
    for r in results:
        t = r['title'].lower()
-        has_web = any(kw in t for kw in web_terms)
+        has_target = any(kw in t for kw in target_terms)
        has_request = any(kw in t for kw in request_terms)
-        if not has_web or not has_request:
+        if not has_target or not has_request:
            continue
        if any(kw in t for kw in offer_reject):
            continue
+        if any(kw in t for kw in offer_reject_tutor):
+            continue
        keyword_leads.append(r)

-    # ── 3. Merge: prefer AI leads, supplement with keywords to reach 5 ──
+    # ── 3. Merge: prefer AI leads, supplement with keywords ──
    seen_titles: set[int] = set()
    merged: list[dict] = []
    for r in ai_leads + keyword_leads:
@@ -1826,24 +2033,6 @@ Return a JSON array like ["yes","no","yes"] matching the order above."""
    merged = [r for r in merged if not any(kw in (r.get('title','') or '').lower() for kw in offer_reject)]
    merged = [r for r in merged if not any(gw in (r.get('title','') or '').lower() for gw in group_words)]

-    # Fill to 5 with loose keyword matches (at least web OR request term)
-    if len(merged) < 5:
-        for r in results:
-            key = hash(r.get('title', ''))
-            if key in seen_titles:
-                continue
-            t = r['title'].lower()
-            if not (any(kw in t for kw in web_terms) or any(kw in t for kw in request_terms)):
-                continue
-            if any(kw in t for kw in offer_reject):
-                continue
-            if any(gw in t for gw in group_words):
-                continue
-            seen_titles.add(key)
-            merged.append(r)
-            if len(merged) >= 5:
-                break
-
    logger.info("classify_leads: %d merged (%d AI + %d keyword) from %d raw", len(merged), len(ai_leads), len(keyword_leads), len(results))
    return merged[:10]