Merge branch 'main' of https://git.coastit.co.za/caitlin/CRM_ENVR

browser-use-service/main.py

@@ -534,6 +534,19 @@ FB_SEARCHES = [
     "who can design a website for my",
     "looking for someone to create a website",
     "recommendations for a web designer",
+    "I need a website",
+    "I need a website built",
+    "need wordpress site",
+    "need a custom website",
+    "I need a new website",
+    "need an online store",
+    "need an online shop",
+    "need website help",
+    "who can help me with my website",
+    "looking for a website designer",
+    "need a shopify website",
+    "looking for website design",
+    "want to build a website for my",
 ]
 
 VIEWPORTS = [
@@ -1361,6 +1374,8 @@ async def ask_ollama(prompt: str) -> str:
 async def classify_leads(results: list[dict]) -> list[dict]:
     if not results:
         return []
+
+    # ── 1. AI classification ─────────────────────────────────────────
     briefs = [r["title"][:200] for r in results]
     prompt = f"""Classify each post as LEAD or NOT.
 LEAD = someone REQUESTING/POSTING/WANTING a website built, designed, or created for them.
@@ -1376,7 +1391,7 @@ NOT LEAD:
 For each numbered post, answer ONLY "yes" (LEAD) or "no" (NOT LEAD):
 {chr(10).join(f'{i+1}. {t}' for i, t in enumerate(briefs))}
 Return a JSON array like ["yes","no","yes"] matching the order above."""
-    ai_succeeded = False
+    ai_leads: list[dict] = []
     try:
         raw = await ask_ollama(prompt)
         raw = raw.strip()
@@ -1389,55 +1404,87 @@ Return a JSON array like ["yes","no","yes"] matching the order above."""
             raw = raw.strip()
         answers = json.loads(raw)
         if isinstance(answers, list) and len(answers) == len(results):
-            ai_succeeded = True
-            filtered = []
             for i, ans in enumerate(answers):
                 if isinstance(ans, str) and ans.lower() == 'yes':
-                    filtered.append(results[i])
-            if filtered:
-                return filtered[:10]
-            logger.info("AI classified all %d items as NOT LEAD — returning empty", len(results))
-            return []
+                    ai_leads.append(results[i])
+            logger.info("AI classified %d/%d as LEAD", len(ai_leads), len(results))
     except Exception as e:
-        logger.warning("AI classification failed, falling back to keyword filter: %s", e)
+        logger.warning("AI classification failed: %s", e)
 
-    if not ai_succeeded:
-        web_terms = [
-            "website", "web design", "web develop", "web dev",
-            "web designer", "web developer",
-            "build my website", "build a website", "create a website",
-            "landing page", "wordpress", "ecommerce",
-            "my website", "business website",
-            "site for my", "site for my business",
-            "new website", "redesign my website",
-            "help with my website", "update my website",
-        ]
-        request_terms = [
-            "looking for", "need a", "need an", "looking to",
-            "need someone", "hire a", "want someone",
-            "need help with", "would like", "build me",
-            "design my", "make me a", "create my",
-            "looking", "need", "want", "help",
-            "who can", "i need",
-            "recommend", "anyone know", "anyone recommend",
-            "know a", "know any", "recommendation",
-            "suggest", "looking for recommendations",
-            "can anyone", "does anyone",
-        ]
-        filtered = []
+    # ── 2. Keyword fallback (always runs) ────────────────────────────
+    web_terms = [
+        "website", "web design", "web develop", "web dev",
+        "web designer", "web developer",
+        "build my website", "build a website", "create a website",
+        "landing page", "wordpress", "ecommerce",
+        "my website", "business website",
+        "site for my", "site for my business",
+        "new website", "redesign my website",
+        "help with my website", "update my website",
+        "make a website", "make my website",
+        "website for my",
+        "online store", "online shop",
+        "build my site", "build a site",
+        "set up a website", "set up my website",
+        "custom website",
+        "shopify",
+        "my site",
+        "webpage", "web page",
+    ]
+    request_terms = [
+        "looking for", "need a", "need an", "looking to",
+        "need someone", "hire a", "want someone",
+        "need help with", "would like", "build me",
+        "design my", "make me a", "create my",
+        "looking", "need", "want", "help",
+        "who can", "i need",
+        "recommend", "anyone know", "anyone recommend",
+        "know a", "know any", "recommendation",
+        "suggest", "looking for recommendations",
+        "can anyone", "does anyone",
+        "dm me", "pm me", "message me",
+        "quote for",
+        "can you help",
+        "how much",
+        "price for",
+    ]
+    keyword_leads: list[dict] = []
+    for r in results:
+        t = r['title'].lower()
+        has_web = any(kw in t for kw in web_terms)
+        has_request = any(kw in t for kw in request_terms)
+        if not has_web or not has_request:
+            continue
+        if any(kw in t for kw in ['i build website', 'i offer web', 'i am a web developer',
+                                   'affordable web design package', 'web hosting',
+                                   'i do web design', 'i develop websites']):
+            continue
+        keyword_leads.append(r)
+
+    # ── 3. Merge: prefer AI leads, supplement with keywords to reach 5 ──
+    seen_titles: set[int] = set()
+    merged: list[dict] = []
+    for r in ai_leads + keyword_leads:
+        key = hash(r.get('title', ''))
+        if key not in seen_titles:
+            seen_titles.add(key)
+            merged.append(r)
+
+    # Fill to 5 with loose keyword matches (at least web OR request term)
+    if len(merged) < 5:
         for r in results:
+            key = hash(r.get('title', ''))
+            if key in seen_titles:
+                continue
             t = r['title'].lower()
-            has_web = any(kw in t for kw in web_terms)
-            has_request = any(kw in t for kw in request_terms)
-            if not has_web or not has_request:
-                continue
-            if any(kw in t for kw in ['i build website', 'i offer web', 'i am a web developer',
-                                       'affordable web design package', 'web hosting',
-                                       'i do web design', 'i develop websites']):
-                continue
-            filtered.append(r)
-        return filtered[:10]
-    return []
+            if any(kw in t for kw in web_terms) or any(kw in t for kw in request_terms):
+                seen_titles.add(key)
+                merged.append(r)
+                if len(merged) >= 5:
+                    break
+
+    logger.info("classify_leads: %d merged (%d AI + %d keyword) from %d raw", len(merged), len(ai_leads), len(keyword_leads), len(results))
+    return merged[:10]
 
 # ══════════════════════════════════════════════════════════════════════
 # FastAPI Endpoints