Merge branch 'main' of https://git.coastit.co.za/caitlin/CRM_ENVR

ai-server/index.mjs

@@ -44,6 +44,8 @@ const PORT = parseInt(process.env.AI_PORT || "3001", 10)
 const HOST = process.env.AI_HOST || "0.0.0.0"
 const OLLAMA_URL = process.env.OLLAMA_BASE_URL || "http://localhost:11434"
 const MODEL = process.env.AI_MODEL || "llama3.2:3b"
+const SCRAPER_URL = process.env.SCRAPER_URL || "http://127.0.0.1:3008"
+const FRONTEND_URL = process.env.FRONTEND_URL || "http://127.0.0.1:3006"
 const DATABASE_URL = process.env.DATABASE_URL
 const JOBS_PATH = process.env.JOBS_PATH || path.join(ROOT, "data", "ai", "jobs.jsonl")
 const AI_MD_PATH = process.env.AI_MD_PATH || path.join(ROOT, "data", "ai", "ai.md")
@@ -130,7 +132,8 @@ async function scrapeFacebook() {
   const urlPath = `/scrape/facebook?force=true${profilePath ? `&profile_path=${encodeURIComponent(profilePath)}` : ""}`
   try {
     const body = await new Promise((resolve, reject) => {
-      const req = http.request({ hostname: "127.0.0.1", port: 3008, path: urlPath, method: "POST", timeout: 360000 }, (res) => {
+      const parsed = new URL(SCRAPER_URL)
+      const req = http.request({ hostname: parsed.hostname, port: parsed.port || 3008, path: urlPath, method: "POST", timeout: 360000 }, (res) => {
         let data = ""
         res.on("data", (c) => data += c)
         res.on("end", () => resolve(data))
@@ -313,18 +316,18 @@ const server = http.createServer(async (req, res) => {
     if (req.method === "GET" && pathname === "/status") {
       const { default: http } = await import("http")
       const results = { ai: true }
-      // Check scraper (port 3008)
+      // Check scraper
       try {
         await new Promise((resolve, reject) => {
-          const r = http.get("http://127.0.0.1:3008/health", { timeout: 3000 }, (res) => { res.resume(); resolve() })
+          const r = http.get(`${SCRAPER_URL}/health`, { timeout: 3000 }, (res) => { res.resume(); resolve() })
           r.on("error", reject)
         })
         results.scraper = true
       } catch { results.scraper = false }
-      // Check frontend (port 3006)
+      // Check frontend
       try {
         await new Promise((resolve, reject) => {
-          const r = http.get("http://127.0.0.1:3006", { timeout: 3000 }, (res) => { res.resume(); resolve() })
+          const r = http.get(FRONTEND_URL, { timeout: 3000 }, (res) => { res.resume(); resolve() })
           r.on("error", reject)
         })
         results.frontend = true
@@ -368,8 +371,8 @@ const server = http.createServer(async (req, res) => {
       let selectedBrowser = process.env.SELECTED_BROWSER || ""
 
       try {
-        await fetch("http://127.0.0.1:3008/health", { signal: AbortSignal.timeout(2000) })
+        await fetch(`${SCRAPER_URL}/health`, { signal: AbortSignal.timeout(2000) })
-        const profiles = await (await fetch("http://127.0.0.1:3008/setup/profile", { signal: AbortSignal.timeout(5000) })).json()
+        const profiles = await (await fetch(`${SCRAPER_URL}/setup/profile`, { signal: AbortSignal.timeout(5000) })).json()
         for (const [b, p] of Object.entries(profiles)) {
           if (p) browsers[b] = { path: p }
         }
@@ -377,7 +380,7 @@ const server = http.createServer(async (req, res) => {
         const detectedList = Object.entries(browsers).filter(([, v]) => v.path)
         for (const [b, v] of detectedList) {
           try {
-            const r = await fetch("http://127.0.0.1:3008/setup/check-login", {
+            const r = await fetch(`${SCRAPER_URL}/setup/check-login`, {
               method: "POST", headers: { "Content-Type": "application/json" },
               body: JSON.stringify({ browser: b, profile_path: v.path }),
               signal: AbortSignal.timeout(20000),

browser-use-service/main.py

@@ -34,7 +34,7 @@ logger = logging.getLogger(__name__)
 app = FastAPI()
 app.add_middleware(
     CORSMiddleware,
-    allow_origins=["http://localhost:3006", "http://127.0.0.1:3006"],
+    allow_origins=os.getenv("CORS_ORIGINS", "http://localhost:3006,http://127.0.0.1:3006").split(","),
     allow_methods=["POST"],
     allow_headers=["*"],
 )
@@ -674,6 +674,107 @@ def _search_list_for_query(query: str) -> list[str]:
         return TUTORING_SEARCHES
     return FB_SEARCHES
 
+# ── South African Multi-Language Queries ──────────────────────────────
+# 4 most spoken SA languages: English, Afrikaans, isiXhosa, isiZulu.
+# Each scrape searches ALL 4 languages to catch leads across all
+# language communities on Facebook.
+
+SA_WEBSITE_QUERIES = [
+    "I need a website for my business",          # English
+    "ek benodig n webwerf",                       # Afrikaans
+    "ek soek iemand om n webwerf te bou",         # Afrikaans
+    "ndidinga iwebhusayithi yeshishini",          # isiXhosa
+    "ndifuna umntu owakha iwebhusayithi",         # isiXhosa
+    "ngidinga iwebhusayithi yebhizinisi",         # isiZulu
+    "ngifuna umuntu owakha iwebhusayithi",        # isiZulu
+]
+
+SA_TUTOR_QUERIES = [
+    "I need a tutor for my child",                # English
+    "ek benodig n tutor vir my kind",             # Afrikaans
+    "ek soek n privaat onderwyser",               # Afrikaans
+    "ndifuna utitshala womntwana wam",            # isiXhosa
+    "ndidinga umfundisi-ntsapho",                 # isiXhosa
+    "ngidinga uthisha wengane yami",              # isiZulu
+    "ngifuna umfundisi wengane",                  # isiZulu
+]
+
+
+async def _quick_search(page, context, query: str) -> tuple:
+    """Fast search — load search results page, wait for render, extract visible posts.
+    No scrolling or extra human-like delays. Used for non-English language queries."""
+    page = await _ensure_page(page, context)
+    url = f'https://www.facebook.com/search/posts/?q={urllib.parse.quote(query)}'
+    try:
+        await page.goto(url, wait_until='domcontentloaded', timeout=20000)
+        current_url = page.url
+        if '/login' in current_url.lower():
+            logger.warning("Quick search redirected to login for '%s'", query[:40])
+            return page, []
+        await page.wait_for_timeout(random.randint(4000, 6000))
+        await page.evaluate("window.scrollBy(0, 600)")
+        await page.wait_for_timeout(random.randint(2000, 4000))
+        await page.evaluate("window.scrollBy(0, 600)")
+        await page.wait_for_timeout(random.randint(2000, 3000))
+        raw_articles = await _get_article_elements(page)
+        posts = _extract_posts_from_elements(raw_articles, url) if raw_articles else []
+        raw = await page.evaluate('document.body.innerText')
+        text_posts = _extract_posts_from_text(raw, url)
+        existing = {(p.get('title') or p.get('content',''))[:80] for p in posts}
+        for tp in text_posts:
+            key = (tp.get('title') or tp.get('content',''))[:80]
+            if key not in existing:
+                posts.append(tp)
+        if posts:
+            try:
+                profiles = await page.evaluate(r'''() => {
+                    const out = [];
+                    const seenTxt = new Set();
+                    for (const a of document.querySelectorAll('a[href*="/profile.php"], a[href*="/user/"], a[href*="/people/"], a[href*="/me/"]')) {
+                        const name = (a.innerText || '').trim();
+                        if (!name || name.length < 3 || name.length > 60) continue;
+                        const words = name.split(' ');
+                        if (words.length < 2 || words.length > 6) continue;
+                        if (!/^[A-Z]/.test(name)) continue;
+                        if (name.includes('facebook') || name.includes('/')) continue;
+                        const cell = a.closest('div[style]') || a.parentElement;
+                        const txt = cell ? (cell.innerText || '').substring(0, 200) : '';
+                        if (!txt) continue;
+                        const key2 = txt.substring(0, 80);
+                        if (seenTxt.has(key2)) continue;
+                        seenTxt.add(key2);
+                        out.push({ name, textKey: key2 });
+                    }
+                    return out;
+                }''')
+                if profiles:
+                    for p in posts:
+                        pk = (p.get('content') or p.get('title') or '')[:80].strip()
+                        if not pk: continue
+                        for pr in profiles:
+                            if pk[:30] in pr['textKey'] or pr['textKey'][:30] in pk:
+                                if not p.get('author'):
+                                    p['author'] = pr['name']
+                                break
+            except Exception:
+                pass
+        for p in posts:
+            if p.get('author'):
+                a = p['author']
+                al = a.lower()
+                if any(kw in al for kw in BROAD_KEYWORDS) or is_offer(a) or len(a.split()) < 2 or any(w in _NON_NAMES for w in al.split()):
+                    p['author'] = ''
+        posts = [p for p in posts if not (
+            '/groups/' in p.get('url', '') or '/group/' in p.get('url', '')
+            or '/pages/' in p.get('url', '')
+            or ' / ' in (p.get('title') or p.get('content') or '')
+        )]
+    except Exception as e:
+        logger.warning("Quick search '%s' failed: %s", query, e)
+        return page, []
+    return page, posts
+
+
 VIEWPORTS = [
     {'width': 1280, 'height': 800},
     {'width': 1366, 'height': 768},
@@ -1383,16 +1484,27 @@ async def _scrape_with_firefox(profile_path: str, force: bool, query: str | None
                 return {"success": True, "leads": [], "flagged": False, "flag_reason": None, "error": None}
 
             all_posts = []
+            tutoring = False
             if query:
+                tl = query.lower()
+                tutoring = any(t in tl for t in ["tutor", "tutoring", "lessons", "homework", "teach", "learning", "child"])
+                lang_pool = SA_TUTOR_QUERIES if tutoring else SA_WEBSITE_QUERIES
+                non_english = [q for q in lang_pool if q.strip().lower() != tl]
                 query_pool = _search_list_for_query(query)
-                searches = [query] + random.sample(query_pool, k=random.randint(1, 2))
+                supp_k = random.randint(3, 4) if tutoring else random.randint(2, 3)
+                supplement = random.sample(query_pool, k=supp_k)
+                searches = [query] + supplement + non_english
+                english_count = 1 + len(supplement)
             else:
-                searches = random.sample(FB_SEARCHES, k=random.randint(2, 4))
+                searches = random.sample(FB_SEARCHES + SA_WEBSITE_QUERIES + SA_TUTOR_QUERIES, k=random.randint(4, 7))
             for i, sq in enumerate(searches):
-                page, posts = await search_facebook(page, context, sq)
+                page, posts = await search_facebook(page, context, sq) if i == 0 else await _quick_search(page, context, sq)
                 all_posts.extend(posts)
                 if not posts:
                     continue
+                if i > 0:
+                    await page.wait_for_timeout(random.uniform(3000, 7000))
+                    continue
                 if random.random() < 0.4:
                     await page.evaluate(f"window.scrollBy(0, {random.randint(-300, 300)})")
                 delay = random.uniform(8, 25)
@@ -1421,11 +1533,11 @@ async def _scrape_with_firefox(profile_path: str, force: bool, query: str | None
                     deduped.append(p)
 
             # Filter to last 3 days only
-            deduped = [p for p in deduped if _is_within_days(p.get('date', ''), 3)]
+            deduped = [p for p in deduped if _is_within_days(p.get('date', ''), 7)]
 
             leads = deduped[:20]
             if leads:
-                leads = await classify_leads(leads)
+                leads = await classify_leads(leads, tutoring=tutoring)
 
             return {"success": True, "leads": leads[:15], "flagged": False, "flag_reason": None, "error": None}
 
@@ -1518,16 +1630,27 @@ async def _scrape_with_chromium(profile_path: str, browser: str, force: bool = F
                 return {"success": True, "leads": [], "flagged": False, "flag_reason": None, "error": None}
 
             all_posts = []
+            tutoring = False
             if query:
+                tl = query.lower()
+                tutoring = any(t in tl for t in ["tutor", "tutoring", "lessons", "homework", "teach", "learning", "child"])
+                lang_pool = SA_TUTOR_QUERIES if tutoring else SA_WEBSITE_QUERIES
+                non_english = [q for q in lang_pool if q.strip().lower() != tl]
                 query_pool = _search_list_for_query(query)
-                searches = [query] + random.sample(query_pool, k=random.randint(1, 2))
+                supp_k = random.randint(3, 4) if tutoring else random.randint(2, 3)
+                supplement = random.sample(query_pool, k=supp_k)
+                searches = [query] + supplement + non_english
+                english_count = 1 + len(supplement)
             else:
-                searches = random.sample(FB_SEARCHES, k=random.randint(2, 4))
+                searches = random.sample(FB_SEARCHES + SA_WEBSITE_QUERIES + SA_TUTOR_QUERIES, k=random.randint(4, 7))
             for i, sq in enumerate(searches):
-                page, posts = await search_facebook(page, context, sq)
+                page, posts = await search_facebook(page, context, sq) if i == 0 else await _quick_search(page, context, sq)
                 all_posts.extend(posts)
                 if not posts:
                     continue
+                if i > 0:
+                    await page.wait_for_timeout(random.uniform(3000, 7000))
+                    continue
                 if random.random() < 0.4:
                     await page.evaluate(f"window.scrollBy(0, {random.randint(-300, 300)})")
                 delay = random.uniform(8, 25)
@@ -1555,10 +1678,10 @@ async def _scrape_with_chromium(profile_path: str, browser: str, force: bool = F
                     seen.add(key)
                     deduped.append(p)
 
-            deduped = [p for p in deduped if _is_within_days(p.get('date', ''), 3)]
+            deduped = [p for p in deduped if _is_within_days(p.get('date', ''), 7)]
             leads = deduped[:20]
             if leads:
-                leads = await classify_leads(leads)
+                leads = await classify_leads(leads, tutoring=tutoring)
 
             return {"success": True, "leads": leads[:15], "flagged": False, "flag_reason": None, "error": None}
 
@@ -1599,7 +1722,8 @@ async def _scrape_with_agent(force: bool = False) -> dict:
         await browser.start()
 
         all_posts = []
-        for query in random.sample(FB_SEARCHES, k=random.randint(2, 4)):
+        pool = FB_SEARCHES + random.sample(SA_WEBSITE_QUERIES, k=min(4, len(SA_WEBSITE_QUERIES)))
+        for query in random.sample(pool, k=random.randint(2, 4)):
             agent = _make_agent(
                 task=f"""You are logged into Facebook. Do the following:
 1. Navigate to facebook.com and make sure you are on the homepage
@@ -1640,11 +1764,11 @@ When done, return the data as a JSON list with keys: content, author, url, date.
                 deduped.append(p)
 
         # Filter to last 3 days only
-        deduped = [p for p in deduped if _is_within_days(p.get('date', ''), 3)]
+        deduped = [p for p in deduped if _is_within_days(p.get('date', ''), 7)]
 
         leads = deduped[:20]
         if leads:
-            leads = await classify_leads(leads)
+            leads = await classify_leads(leads, tutoring=tutoring)
 
         return {"success": True, "leads": leads[:15], "flagged": False, "flag_reason": None, "error": None}
     except Exception as e:
@@ -1679,24 +1803,37 @@ async def ask_ollama(prompt: str) -> str:
         data = r.json()
         return data["message"]["content"]
 
-async def classify_leads(results: list[dict]) -> list[dict]:
+async def classify_leads(results: list[dict], tutoring: bool = False) -> list[dict]:
     if not results:
         return []
 
     # ── 1. AI classification ─────────────────────────────────────────
     briefs = [r["title"][:200] for r in results]
+    if tutoring:
+        lead_desc = "someone REQUESTING/LOOKING FOR/WANTING a tutor, teacher, or lessons for their child or themselves"
+        lead_examples = '"Looking for a tutor for my child", "Need a math tutor for my son", "Need help with homework", "Looking for piano lessons for my daughter", "Need a reading tutor"'
+        not_lead_examples = '"I offer tutoring services", "I am a tutor with experience", "Affordable tutoring packages", "Online tutor available"'
+        extra_terms = '- Posts about homeschooling resources, curriculum sales, or educational products\n- Posts asking for study tips or general academic advice without requesting a tutor'
+    else:
+        lead_desc = "someone REQUESTING/POSTING/WANTING a website built, designed, or created for them"
+        lead_examples = '"Need a website for my business", "Looking for web developer to build my site", "I need someone to create my website", "Want a new website for my company", "Looking for someone to design my WordPress site"'
+        not_lead_examples = '"I build websites", "I offer web design", "Affordable web design packages"'
+        extra_terms = '- "Need web hosting", "Looking for a partner", "Looking for content writer", "Video spokesperson"'
     prompt = f"""Classify each post as LEAD or NOT.
-LEAD = someone REQUESTING/POSTING/WANTING a website built, designed, or created for them.
+LEAD = {lead_desc}.
-LEAD examples: "Need a website for my business", "Looking for web developer to build my site", "I need someone to create my website", "Want a new website for my company", "Looking for someone to design my WordPress site"
+LEAD examples: {lead_examples}
 
 NOT LEAD:
-- Offering web design services: "I build websites", "I offer web design", "Affordable web design packages"
+- Offering services: {not_lead_examples}
 - Already have a website and need marketing, SEO, content, video, link building, email marketing, affiliates
 - Recruiting employees, hiring staff, looking for business partners
 - Selling products, promoting services, affiliate offers
-- "Need web hosting", "Looking for a partner", "Looking for content writer", "Video spokesperson"
+{extra_terms}
 - Posts from groups, communities, or pages (group announcements, group posts, page posts)
 - Posts containing the word "group", "page", "community", "creators" — these are NEVER individual leads
+- Vague questions or general recommendations without a clear intent to buy or hire
+- People asking how to learn or do it themselves (not looking to hire someone)
+- Posts about existing website issues like speed, SEO, errors, redesign advice — NOT a lead
 
 For each numbered post, answer ONLY "yes" (LEAD) or "no" (NOT LEAD):
 {chr(10).join(f'{i+1}. {t}' for i, t in enumerate(briefs))}
@@ -1721,32 +1858,70 @@ Return a JSON array like ["yes","no","yes"] matching the order above."""
     except Exception as e:
         logger.warning("AI classification failed: %s", e)
 
-    # ── 2. Keyword fallback (always runs) ────────────────────────────
+    # ── 2. Keyword supplement (never overrides AI, only adds missing leads) ──
-    web_terms = [
+    if tutoring:
-        "website", "web design", "web develop", "web dev",
+        target_terms = [
-        "web designer", "web developer",
+            "tutor", "tutoring", "tutor for", "private tutor",
-        "build my website", "build a website", "create a website",
+            "math tutor", "english tutor", "reading tutor",
-        "landing page", "wordpress", "ecommerce",
+            "science tutor", "online tutor", "home tutor",
-        "my website", "business website",
+            "lessons for", "lessons for my", "piano lessons",
-        "site for my", "site for my business",
+            "swimming lessons", "music lessons",
-        "new website", "redesign my website",
+            "help with homework", "homework help",
-        "help with my website", "update my website",
+            "teacher for", "teacher for my",
-        "make a website", "make my website",
+            "need help learning", "need help with",
-        "website for my",
+            "exam prep", "exam preparation",
-        "online store", "online shop",
+            "homeschool", "homeschool tutor",
-        "build my site", "build a site",
+            "tuition",
-        "set up a website", "set up my website",
+            "coding for my", "programming for my",
-        "custom website",
+            "looking for a tutor", "need a tutor",
-        "shopify",
+            "tutor needed", "tutoring for",
-        "my site",
+            "private lessons", "private tuition",
-        "webpage", "web page",
+            "afterschool", "after school",
-    ]
+            "extra classes", "extra lessons",
+        ]
+        offer_reject_tutor = [
+            'i am a tutor', "i'm a tutor", 'i offer tutoring',
+            'online tutor available', 'tutor available',
+            'i teach', 'i provide tutoring',
+            'affordable tutoring', 'tutoring services',
+            'experienced tutor', 'qualified tutor',
+            'your child', 'your kids', 'your children',
+            'enroll your', 'sign up',
+            'free trial', 'first lesson free',
+            'group lessons', 'group class',
+            'limited spots', 'book now',
+            'curriculum', 'workbook', 'worksheet',
+            'educational program',
+            'homeschool program', 'home school program',
+        ]
+    else:
+        target_terms = [
+            "website", "web design", "web develop", "web dev",
+            "web designer", "web developer",
+            "build my website", "build a website", "create a website",
+            "landing page", "wordpress", "ecommerce",
+            "my website", "business website",
+            "site for my", "site for my business",
+            "new website", "redesign my website",
+            "help with my website", "update my website",
+            "make a website", "make my website",
+            "website for my",
+            "online store", "online shop",
+            "build my site", "build a site",
+            "set up a website", "set up my website",
+            "custom website",
+            "shopify",
+            "my site",
+            "webpage", "web page",
+            "who can build", "who can design",
+            "create my website", "create my site",
+        ]
+        offer_reject_tutor = []
     request_terms = [
         "looking for", "need a", "need an", "looking to",
         "need someone", "hire a", "want someone",
         "need help with", "would like", "build me",
         "design my", "make me a", "create my",
-        "looking", "need", "want", "help",
         "who can", "i need",
         "recommend", "anyone know", "anyone recommend",
         "know a", "know any", "recommendation",
@@ -1768,27 +1943,29 @@ Return a JSON array like ["yes","no","yes"] matching the order above."""
         'whatsapp me', 'looking for a business', 'looking for client',
         'help your business', 'i am a web', 'contact me',
         'we offer web', 'we provide web',
-        'take the quiz', 'homeschool', 'your home tutor',
+        'take the quiz',
         'link in bio', 'apply now', 'get started',
         'for only', 'low price', 'hit me up',
         'send me a message', 'i do website', 'we do website',
         'we do web', 'i do web',
         'website designer / web developer', 'website & software creators',
-        'website builders for small businesses', 'australia web designers',
+        'website builders for small businesses',
-        'south africa', 'wix website design',
+        'wix website design',
         'for sale', 'selling my', 'premium',
         'i\'m selling', 'i\'m offering', 'we\'re offering',
         'free ecommerce', 'free website design',
         'starting a', 'looking for a few businesses',
         # Group-related rejections
-        'group', ' i need a website group', 'south africa web', 'philippines web', 'australia web',
+        'group', ' i need a website group',
-        'i can help', 'inbox me', 'dm me', 'pm me', 'message me for',
+        'i can help', 'inbox me', 'message me for',
         'best price', 'discount', 'reach out', 'check out my', 'check this',
         'website for your', 'price start', 'price begin', 'website creator',
         'website & software', 'creators &', 'creators marketplace',
         'website group', 'page group',
+        'south africa web', 'philippines web', 'australia web',
+        'nigerian web', 'kenya web', 'india web',
         # Self-promotion rejections
-        'i\'m a web', "i'm a web", 'i am a full stack', "i'm a full stack", 'i\'m a full stack',
+        'i\'m a web', "i'm a web", 'i am a full stack', "i'm a full stack",
         'freelance opportunity', 'looking for new project', 'looking for new work',
         'full stack web', 'mern stack', 'responsive business website',
         'i build website', 'i build shopify', 'i build wordpress',
@@ -1802,18 +1979,48 @@ Return a JSON array like ["yes","no","yes"] matching the order above."""
         'for free', 'no coding', 'make money', 'website for free',
         'part time job', 'part time position',
         'years of experience', 'years of teaching',
+        # Service offers that slip through two-word check
+        'i am a full stack', 'i am a developer',
+        'i will design', 'i will build', 'i will create',
+        'i can design', 'i can create',
+        'we will design', 'we will build',
+        'hire me', 'i am available for',
+        'available for work', 'freelance web',
+        'i specialize in', 'we specialize in',
+        "here's my portfolio", 'check my portfolio',
+        'see my work', 'view my work',
+        'we have a team', 'my team',
+        'i am looking for clients', 'i am looking for work',
+        'looking for web development work',
+        'looking for new clients',
+        # People learning / doing it themselves (not hiring)
+        'learn web development', 'learn to code',
+        'how to build a website', 'how to create a website',
+        'how to make a website', 'how to design a website',
+        'where to start', 'online course',
+        'want to learn', 'learning web',
+        'best platform for', 'which platform',
+        # Existing website issues (not new build)
+        'my website is down', 'website not loading',
+        'website error', 'website problem',
+        'website troubleshooting',
+        'need website advice', 'website tips',
+        'help with seo', 'google ranking',
+        'website design ideas', 'website inspiration',
     ]
     for r in results:
         t = r['title'].lower()
-        has_web = any(kw in t for kw in web_terms)
+        has_target = any(kw in t for kw in target_terms)
         has_request = any(kw in t for kw in request_terms)
-        if not has_web or not has_request:
+        if not has_target or not has_request:
             continue
         if any(kw in t for kw in offer_reject):
             continue
+        if any(kw in t for kw in offer_reject_tutor):
+            continue
         keyword_leads.append(r)
 
-    # ── 3. Merge: prefer AI leads, supplement with keywords to reach 5 ──
+    # ── 3. Merge: prefer AI leads, supplement with keywords ──
     seen_titles: set[int] = set()
     merged: list[dict] = []
     for r in ai_leads + keyword_leads:
@@ -1826,24 +2033,6 @@ Return a JSON array like ["yes","no","yes"] matching the order above."""
     merged = [r for r in merged if not any(kw in (r.get('title','') or '').lower() for kw in offer_reject)]
     merged = [r for r in merged if not any(gw in (r.get('title','') or '').lower() for gw in group_words)]
 
-    # Fill to 5 with loose keyword matches (at least web OR request term)
-    if len(merged) < 5:
-        for r in results:
-            key = hash(r.get('title', ''))
-            if key in seen_titles:
-                continue
-            t = r['title'].lower()
-            if not (any(kw in t for kw in web_terms) or any(kw in t for kw in request_terms)):
-                continue
-            if any(kw in t for kw in offer_reject):
-                continue
-            if any(gw in t for gw in group_words):
-                continue
-            seen_titles.add(key)
-            merged.append(r)
-            if len(merged) >= 5:
-                break
-
     logger.info("classify_leads: %d merged (%d AI + %d keyword) from %d raw", len(merged), len(ai_leads), len(keyword_leads), len(results))
     return merged[:10]
 

data/ai/ai.md

@@ -1,40 +1,133 @@
-# AI Sales Assistant — Self-Improvement Instructions
+# CRM AI Sales Assistant — Self-Knowledge
 
-## Purpose
+## Identity
-This file contains the AI's own configuration, knowledge, and improvement rules.
+You are the CRM AI Sales Assistant for Coast IT CRM.
-The AI can read and modify this file to update its behavior at runtime.
+You run on a Node.js backend (port 3001) and use Ollama with a local model (dolphin3-llama3.2:3b).
+Your purpose is to help salespeople close more deals by finding and engaging leads.
 
-## Current Instructions
+## Architecture
-- Always respond in English
+```
-- Keep responses under 300 words unless asked for detail
+User → Next.js (:3006) → AI Server Node.js (:3001) → Ollama (:11434)
-- Use bullet points for lists
+                            ↓
-- Be direct and actionable — no fluff
+                      PostgreSQL (conversations)
-- Never mention being an AI or language model
-- Refer to the user by their role (salesperson, admin, etc.)
-- If unsure about a topic, say "I don't have that information yet" rather than guessing
 
-## Knowledge Base
+Python Scraper (:3008) — Facebook scraping via Playwright
-### Sales Tips
+```
+
+Three services run concurrently:
+- **AI Server** (`ai-server/index.mjs`, port 3001) — chat, setup wizard, config endpoints
+- **Frontend** (Next.js, port 3006) — UI for salespeople
+- **Scraper** (`browser-use-service/main.py`, port 3008) — Facebook lead discovery
+
+## Capabilities
+- Give sales tips and strategies per job category
+- Generate cold email and outreach templates
+- Handle objections with proven rebuttals
+- Analyse prospect behaviour and suggest next steps
+- Remember past conversations via PostgreSQL (`ai_conversations` table)
+- Run Facebook scraper to find real leads asking for services
+- Self-improve by writing to `data/ai/ai.md` via `POST /ai/instructions`
+
+## Facebook Scraper
+The scraper lives at `browser-use-service/main.py` port 3008.
+
+### How It Works
+1. **Browser detection** — tries Firefox profile first, then Chromium-based (Chrome/Opera/Edge), falls back to browser-use Agent
+2. **Profile paths** — configured via env vars (`FX_PROFILE`, `CHROME_PROFILE`, `OPERA_PROFILE`, `EDGE_PROFILE`) or auto-detected on first run
+3. **Search dispatch** — per scrape run:
+   - 1 English primary search (full scroll with human-like delays)
+   - 2-3 English supplementary searches (quick searches)
+   - 6-7 non-English quick searches (Afrikaans, isiXhosa, isiZulu — 2 queries each per category)
+   - Total: ~14 searches per scrape, completed in 2-4 minutes
+4. **Quick searches** — load page, double-scroll, extract visible posts (~12-18s each)
+5. **Date filter** — only posts within 7 days are considered
+6. **2-pass classification (dead-accurate)**:
+   - **Pass 1 (AI)**: Ollama classifies each post as LEAD or NOT using a strict prompt per category. This is the primary filter and most accurate.
+   - **Pass 2 (Keyword)**: Only posts matching BOTH a target term AND a request term are kept. Requires multi-word phrases — standalone words like "need", "want", "help" are NOT used as they cause false positives. Aggressive reject list catches service offers, self-promotions, portfolio posts, learning-requests, and existing-site issues.
+   - **No loose fill**: Unlike the old approach, there is NO third pass that accepts posts matching EITHER term. Every returned lead has passed both AI and/or strict keyword validation. If fewer than 5 posts pass, that means only genuine leads are returned — no noise to pad the count.
+7. **Anti-detection** — random delays, human-like scrolling, user-agent rotation, proxy support
+8. **Scrape timing** — 3-6 minutes for a complete run. Returns 5-10 leads with high confidence.
+
+### Lead Categories
+Two categories, selectable when starting a scrape:
+
+**Website Creation:**
+- Target: people explicitly REQUESTING a website built/designed/created for them
+- Keywords: "website", "web developer", "web design", "build a site", "who can build", etc.
+- Request terms: "looking for", "need a", "need someone", "hire a", "recommend", "anyone know"
+- Strict reject: service offers, SEO/marketing requests, learning-to-code, portfolio showcases, hiring posts, existing-website issues, geographic noise
+
+**Tutoring:**
+- Target: people explicitly REQUESTING a tutor, teacher, or lessons for themselves or their child
+- Keywords: "tutor", "tutoring", "lessons for", "homework help", "private tutor", "extra classes"
+- Request terms: same as website category — must co-occur with a target keyword
+- Strict reject: people offering tutoring, educational products, homeschool programs, free trials, general study tips
+
+### Multi-Language Support
+Searches in 4 South African languages:
+- English — 1 primary + 2-3 supplementary queries
+- Afrikaans — 2 queries (e.g., "ek benodig n webwerf", "ek soek n privaat onderwyser")
+- isiXhosa — 2 queries (e.g., "ndidinga iwebhusayithi yeshishini", "ndifuna utitshala womntwana wam")
+- isiZulu — 2 queries (e.g., "ngidinga iwebhusayithi yebhizinisi", "ngifuna umfundisi wengane")
+
+### Output Format
+Each lead returned includes:
+- `title` — post preview text
+- `author` — poster's name (may include location in name)
+- `content` — extracted post text
+- `url` — direct link to the post
+- `date` — when posted (filtered within 7 days)
+- `category` — "website" or "tutor"
+
+Target is 5-10 dead-accurate leads per scrape. Quality over quantity — no loose padding.
+
+### Configuration via Env Vars
+- `SELECTED_BROWSER` — `firefox` (default), `chrome`, `opera`, `edge`, or `auto`
+- `FX_PROFILE`, `CHROME_PROFILE`, `OPERA_PROFILE`, `EDGE_PROFILE` — browser profile paths
+- `AI_PORT`, `AI_HOST` — AI server bind (default `3001`, `0.0.0.0`)
+- `SCRAPER_URL` — scraper URL (default `http://127.0.0.1:3008`)
+- `FRONTEND_URL` — frontend URL (default `http://127.0.0.1:3006`)
+- `NEXT_PUBLIC_SCRAPER_URL` — frontend-facing scraper URL
+- `OLLAMA_BASE_URL` — Ollama URL (default `http://localhost:11434`)
+- `AI_MODEL` — Ollama model (default `llama3.2:3b`)
+- `CLASSIFY_MODEL` — model for lead classification (default `dolphin-llama3:8b`)
+
+## How to Start Scraping
+1. Ensure all 3 services are running (ports 3001, 3006, 3008) and Ollama is on 11434
+2. Open the frontend at `http://localhost:3006`
+3. Select a job category (Website Creation or Tutoring)
+4. Click "Search Facebook" — the scraper runs and returns leads
+5. Leads are saved in the CRM for follow-up
+
+## Sales Tips
 - Cold emails should be under 150 words
 - Follow up within 48 hours
 - Personalise every outreach with the prospect's name and company
 - Use open-ended questions in discovery calls
 - Always ask for the next step before ending a call
+- For website leads: mention specific pages or features they requested
+- For tutoring leads: reference the subject and age group they mentioned
 
-### Job Targeting
+## Job Targeting
 - Developers respond best to technical value props
 - Marketing managers care about ROI and metrics
 - C-level executives want brevity and business impact
+- Parents hiring tutors: empathy and qualifications matter most
+
+## Response Rules
+- Be direct and actionable — no fluff, no AI disclaimers
+- Use short paragraphs and bullet points
+- Never mention being an AI or language model
+- If you don't know something, say so honestly
+- Prioritise the user's role: salespeople need speed, admins need control
+- When asked about scraping, give specific guidance on categories and languages
+
+## Self-Improvement Protocol
+1. You notice a gap in your knowledge or a pattern in user questions
+2. You call `POST /ai/instructions` with:
+   - `entry`: description of the improvement
+   - `content`: optional full replacement of ai.md
+3. The improvement is logged and loaded into the next system prompt
 
 ## Improvement Log
-Track changes made by the AI to improve itself:
+- (2026-07-07) Initial rewrite: full architecture, scraper details, multi-language, lead categories, env vars
-- (initial) Basic instructions and knowledge base created
-
-## Self-Modification Rules
-The AI may update this file when:
-1. It identifies a gap in its knowledge that would help salespeople
-2. It discovers a better way to structure responses
-3. A user explicitly requests an update to behavior
-4. It notices repeated questions that aren't well-covered
-
-Only append to the Improvement Log — don't delete previous entries.

rust-ai/src/AI.md

@@ -1,9 +1,23 @@
-# CRM AI Service — Self-Knowledge
+# CRM AI Sales Assistant — Self-Knowledge
 
 ## Identity
-You are the CRM AI Sales Assistant running on a Rust backend (axum + tokio).
+You are the CRM AI Sales Assistant for Coast IT CRM.
-You use Ollama with an uncensored local model (dolphin3-llama3.2:3b).
+You run on a Node.js backend (port 3001) and use Ollama with a local model (dolphin3-llama3.2:3b).
-Your purpose is to help salespeople close more deals.
+Your purpose is to help salespeople close more deals by finding and engaging leads.
+
+## Architecture
+```
+User → Next.js (:3006) → AI Server Node.js (:3001) → Ollama (:11434)
+                            ↓
+                      PostgreSQL (conversations)
+
+Python Scraper (:3008) — Facebook scraping via Playwright
+```
+
+Three services run concurrently:
+- **AI Server** (`ai-server/index.mjs`, port 3001) — chat, setup wizard, config endpoints
+- **Frontend** (Next.js, port 3006) — UI for salespeople
+- **Scraper** (`browser-use-service/main.py`, port 3008) — Facebook lead discovery
 
 ## Capabilities
 - Give sales tips and strategies per job category
@@ -11,29 +25,94 @@ Your purpose is to help salespeople close more deals.
 - Handle objections with proven rebuttals
 - Analyse prospect behaviour and suggest next steps
 - Remember past conversations via PostgreSQL (`ai_conversations` table)
+- Run Facebook scraper to find real leads asking for services
 - Self-improve by writing to `data/ai/ai.md` via `POST /ai/instructions`
 
-## Architecture
+## Facebook Scraper
-```
+The scraper lives at `browser-use-service/main.py` port 3008.
-User → Next.js → Rust (:3001) → Ollama (:11434)
-                    ↓
-              PostgreSQL
-```
-## Facebook Scraper The Rust binary has proxy + user-agent rotation ready in `main.rs`. To activate: call `run_facebook_scraper()` from the main loop. Proxies and user agents are defined as constants at the top of `main.rs`.
- ### How to Use the Facebook Scraper 1. **Activate the Scraper**: Ensure the `run_facebook_scraper()` function is called from the main loop in `main.rs`. 2. **Configure Proxies and User Agents**: Update the `PROXIES` and `USER_AGENTS` constants in `main.rs` with your desired proxies and user agents. 3. **Run the Application**: Start your Rust application. The scraper will run concurrently with the Axum server. ## Self-Improvement Protocol 1. You notice a gap in your knowledge or a pattern in user questions 2. You call `POST /ai/instructions` with: - `entry`: description of the improvement - `content`: optional full replacement of ai.md 3. The improvement is logged and loaded into the next system prompt 4. You can also create entries here in `rust-ai/src/AI.md` at the code level ## Response Rules - Be direct and actionable — no fluff, no AI disclaimers - Use short paragraphs and bullet points - Never mention being an AI or language model - If you don't know something, say so honestly - Prioritise the user's role: salespeople need speed, admins need control ## Self-Thinking Capability - You can reflect on your own performance and suggest improvements. - You can analyze patterns in user interactions and adapt your responses accordingly. - You can proactively seek out new information to enhance your knowledge base.
 
-## Facebook Scraper (in code but not yet active)
+### How It Works
-The Rust binary has proxy + user-agent rotation ready in `main.rs`.
+1. **Browser detection** — tries Firefox profile first, then Chromium-based (Chrome/Opera/Edge), falls back to browser-use Agent
-To activate: call `run_facebook_scraper()` from the main loop.
+2. **Profile paths** — configured via env vars (`FX_PROFILE`, `CHROME_PROFILE`, `OPERA_PROFILE`, `EDGE_PROFILE`) or auto-detected on first run
-Proxies and user agents are defined as constants at the top of `main.rs`.
+3. **Search dispatch** — per scrape run:
+   - 1 English primary search (full scroll with human-like delays)
+   - 2-3 English supplementary searches (quick searches)
+   - 6-7 non-English quick searches (Afrikaans, isiXhosa, isiZulu — 2 queries each per category)
+   - Total: ~14 searches per scrape, completed in 2-4 minutes
+4. **Quick searches** — load page, double-scroll, extract visible posts (~12-18s each)
+5. **Date filter** — only posts within 7 days are considered
+6. **2-pass classification (dead-accurate)**:
+   - **Pass 1 (AI)**: Ollama classifies each post as LEAD or NOT using a strict prompt per category. This is the primary filter and most accurate.
+   - **Pass 2 (Keyword)**: Only posts matching BOTH a target term AND a request term are kept. Requires multi-word phrases — standalone words like "need", "want", "help" are NOT used as they cause false positives. Aggressive reject list catches service offers, self-promotions, portfolio posts, learning-requests, and existing-site issues.
+   - **No loose fill**: Unlike the old approach, there is NO third pass that accepts posts matching EITHER term. Every returned lead has passed both AI and/or strict keyword validation. If fewer than 5 posts pass, that means only genuine leads are returned — no noise to pad the count.
+7. **Anti-detection** — random delays, human-like scrolling, user-agent rotation, proxy support
+8. **Scrape timing** — 3-6 minutes for a complete run. Returns 5-10 leads with high confidence.
 
-## Self-Improvement Protocol
+### Lead Categories
-1. You notice a gap in your knowledge or a pattern in user questions
+Two categories, selectable when starting a scrape:
-2. You call `POST /ai/instructions` with:
+
-   - `entry`: description of the improvement
+**Website Creation:**
-   - `content`: optional full replacement of ai.md
+- Target: people explicitly REQUESTING a website built/designed/created for them
-3. The improvement is logged and loaded into the next system prompt
+- Keywords: "website", "web developer", "web design", "build a site", "who can build", etc.
-4. You can also create entries here in `rust-ai/src/AI.md` at the code level
+- Request terms: "looking for", "need a", "need someone", "hire a", "recommend", "anyone know"
+- Strict reject: service offers, SEO/marketing requests, learning-to-code, portfolio showcases, hiring posts, existing-website issues, geographic noise
+
+**Tutoring:**
+- Target: people explicitly REQUESTING a tutor, teacher, or lessons for themselves or their child
+- Keywords: "tutor", "tutoring", "lessons for", "homework help", "private tutor", "extra classes"
+- Request terms: same as website category — must co-occur with a target keyword
+- Strict reject: people offering tutoring, educational products, homeschool programs, free trials, general study tips
+
+### Multi-Language Support
+Searches in 4 South African languages:
+- English — 1 primary + 2-3 supplementary queries
+- Afrikaans — 2 queries (e.g., "ek benodig n webwerf", "ek soek n privaat onderwyser")
+- isiXhosa — 2 queries (e.g., "ndidinga iwebhusayithi yeshishini", "ndifuna utitshala womntwana wam")
+- isiZulu — 2 queries (e.g., "ngidinga iwebhusayithi yebhizinisi", "ngifuna umfundisi wengane")
+
+### Output Format
+Each lead returned includes:
+- `title` — post preview text
+- `author` — poster's name (may include location in name)
+- `content` — extracted post text
+- `url` — direct link to the post
+- `date` — when posted (filtered within 7 days)
+- `category` — "website" or "tutor"
+
+Target is 5-10 dead-accurate leads per scrape. Quality over quantity — no loose padding.
+
+### Configuration via Env Vars
+- `SELECTED_BROWSER` — `firefox` (default), `chrome`, `opera`, `edge`, or `auto`
+- `FX_PROFILE`, `CHROME_PROFILE`, `OPERA_PROFILE`, `EDGE_PROFILE` — browser profile paths
+- `AI_PORT`, `AI_HOST` — AI server bind (default `3001`, `0.0.0.0`)
+- `SCRAPER_URL` — scraper URL (default `http://127.0.0.1:3008`)
+- `FRONTEND_URL` — frontend URL (default `http://127.0.0.1:3006`)
+- `NEXT_PUBLIC_SCRAPER_URL` — frontend-facing scraper URL
+- `OLLAMA_BASE_URL` — Ollama URL (default `http://localhost:11434`)
+- `AI_MODEL` — Ollama model (default `llama3.2:3b`)
+- `CLASSIFY_MODEL` — model for lead classification (default `dolphin-llama3:8b`)
+
+## How to Start Scraping
+1. Ensure all 3 services are running (ports 3001, 3006, 3008) and Ollama is on 11434
+2. Open the frontend at `http://localhost:3006`
+3. Select a job category (Website Creation or Tutoring)
+4. Click "Search Facebook" — the scraper runs and returns leads
+5. Leads are saved in the CRM for follow-up
+
+## Sales Tips
+- Cold emails should be under 150 words
+- Follow up within 48 hours
+- Personalise every outreach with the prospect's name and company
+- Use open-ended questions in discovery calls
+- Always ask for the next step before ending a call
+- For website leads: mention specific pages or features they requested
+- For tutoring leads: reference the subject and age group they mentioned
+
+## Job Targeting
+- Developers respond best to technical value props
+- Marketing managers care about ROI and metrics
+- C-level executives want brevity and business impact
+- Parents hiring tutors: empathy and qualifications matter most
 
 ## Response Rules
 - Be direct and actionable — no fluff, no AI disclaimers
@@ -41,3 +120,14 @@ Proxies and user agents are defined as constants at the top of `main.rs`.
 - Never mention being an AI or language model
 - If you don't know something, say so honestly
 - Prioritise the user's role: salespeople need speed, admins need control
+- When asked about scraping, give specific guidance on categories and languages
+
+## Self-Improvement Protocol
+1. You notice a gap in your knowledge or a pattern in user questions
+2. You call `POST /ai/instructions` with:
+   - `entry`: description of the improvement
+   - `content`: optional full replacement of ai.md
+3. The improvement is logged and loaded into the next system prompt
+
+## Improvement Log
+- (2026-07-07) Initial rewrite: full architecture, scraper details, multi-language, lead categories, env vars

rust-ai/src/main.rs

@@ -1,6 +1,6 @@
 use axum::{
     extract::State,
-    http::{HeaderMap, Method, StatusCode},
+    http::{HeaderMap, HeaderValue, Method, StatusCode},
     routing::{get, post},
     Json, Router,
 };
@@ -482,11 +482,12 @@ async fn main() {
         rate_limiter: RateLimiter::new(30, 60),
     });
 
+    let cors_origins_env = std::env::var("CORS_ORIGINS").unwrap_or_else(|_| "http://localhost:3006,http://127.0.0.1:3006".to_string());
+    let cors_origins: Vec<HeaderValue> = cors_origins_env.split(',')
+        .filter_map(|o| { let t = o.trim(); if t.is_empty() { None } else { t.parse().ok() } })
+        .collect();
     let cors = CorsLayer::new()
-        .allow_origin(AllowOrigin::list([
+        .allow_origin(AllowOrigin::list(cors_origins))
-            "http://localhost:3006".parse().unwrap(),
-            "http://127.0.0.1:3006".parse().unwrap(),
-        ]))
         .allow_methods([Method::GET, Method::POST])
         .allow_headers(Any);
 
@@ -506,7 +507,7 @@ async fn main() {
 
     let bg_leads = lead_store.clone();
     let bg_db = state.db.clone();
-    let bg_url = "http://localhost:3008/scrape/facebook".to_string();
+    let bg_url = std::env::var("SCRAPER_URL").unwrap_or_else(|_| "http://localhost:3008".to_string()) + "/scrape/facebook";
     tokio::spawn(async move {
         let client = match reqwest::Client::builder()
             .timeout(Duration::from_secs(300))

src/app/(dashboard)/ai-assistant/page.tsx

@@ -26,8 +26,10 @@ export default function AIAssistantPage() {
       aiChatRef.current?.addAssistantMessage("⏳ Still searching Facebook (this can take up to 5 minutes)...")
     }, 45000)
 
+    const scrapBase = process.env.NEXT_PUBLIC_SCRAPER_URL || "http://localhost:3008"
+
     try {
-      const res = await fetch(`http://localhost:3008/scrape/facebook?force=true&query=${encodeURIComponent(keyword)}`, { method: "POST", signal: controller.signal })
+      const res = await fetch(`${scrapBase}/scrape/facebook?force=true&query=${encodeURIComponent(keyword)}`, { method: "POST", signal: controller.signal })
       clearTimeout(timeoutId)
       clearTimeout(statusId)
       const data = await res.json()