Updated

browser-use-service/main.py

@@ -37,10 +37,72 @@ BROAD_KEYWORDS = [
     "want someone", "need help with",
 ]
 
+OFFER_PATTERNS = [
+    r"\bfree\s+domain\b",
+    r"\bweb\s+hosting\b",
+    r"\bfree\s+website\b",
+    r"\bprofessional\s+website",
+    r"\baffordable\s+web\b",
+    r"\bget\s+a\s+free\b",
+    r"\bsee\s+more\b",
+    r"\bbook\s+your\b",
+    r"\bhire\s+a\s+freelancer\b",
+    r"\bcontent\s+strategy\b",
+    r"\bdigital\s+marketing\b",
+    r"\bseo\s+service",
+    r"\bsocial\s+media\b",
+    r"\blimited\s+time\s+offer\b",
+    r"\bspecial\s+offer\b",
+    r"\bsign\s+up\s+now\b",
+    r"\br\d[\d,.]",
+    r"\bstarting\s+at\b",
+    r"\bper\s+month\b",
+    r"\bmonthly\b",
+    r"\bfreelancer\s+marketplace\b",
+    r"\bfirst\s+order\b",
+    r"\bdomain\s+name\b",
+    r"\bregister\s+your\s+domain\b",
+    r"\bsponsored\b",
+    r"\bpromoted\b",
+    r"\badvertisement\b",
+]
+
+REQUEST_PATTERNS = [
+    r"\bi\s+need\b",
+    r"\bi\s+need\s+someone\b",
+    r"\bi\s+need\s+a\b",
+    r"\bi\s+need\s+an\b",
+    r"\bi\s+want\b",
+    r"\bi\s+would\s+like\b",
+    r"\bi'm\s+looking\b",
+    r"\bi\s+am\s+looking\b",
+    r"\blooking\s+for\b",
+    r"\blooking\s+to\b",
+    r"\bwho\s+can\b",
+    r"\bcan\s+someone\b",
+    r"\bhelp\s+me\b",
+    r"\bneed\s+help\b",
+    r"\bwould\s+like\s+to\b",
+    r"\bwant\s+to\b",
+    r"\banyone\s+know\b",
+    r"\bdoes\s+anyone\b",
+    r"\brecommend\b",
+    r"\bsuggestion\b",
+    r"\bquote\s+(please|for|to)\b",
+]
+
 def kw_match(text: str) -> bool:
     t = text.lower()
     return any(kw in t for kw in BROAD_KEYWORDS)
 
+def is_request(text: str) -> bool:
+    t = text.lower()
+    return any(re.search(p, t) for p in REQUEST_PATTERNS)
+
+def is_offer(text: str) -> bool:
+    t = text.lower()
+    return any(re.search(p, t) for p in OFFER_PATTERNS)
+
 FB_SEARCHES = [
     "looking for web developer",
     "need a website designed",
@@ -50,10 +112,19 @@ FB_SEARCHES = [
     "looking for someone to create website",
     "need ecommerce website built",
     "want to hire web developer",
-    "website quote please",
     "need wordpress website",
     "I need a website for my business",
     "need a site for my business",
+    "looking for website designer South Africa",
+    "need website for my small business",
+    "who can build me a website",
+    "want to create a website for my business",
+    "looking for affordable web design",
+    "need a web developer South Africa",
+    "help me build a website",
+    "need someone to design my website",
+    "looking for web designer near me",
+    "need a website for my startup",
 ]
 
 VIEWPORTS = [
@@ -136,52 +207,96 @@ def _parse_fb_date(block: list[str]) -> str:
                 pass
     return datetime.now().strftime('%Y-%m-%d')
 
-def _extract_posts_from_text(raw: str, url: str) -> list[dict]:
-    lines = [l.strip() for l in raw.split('\n')]
-    blocks = []
-    cur = []
-    fb_run = 0
-    for l in lines:
-        if 'facebook' in l.lower():
-            fb_run += 1
-            if cur and fb_run >= 2:
-                blocks.append(cur)
-                cur = []
-        else:
-            fb_run = 0
-            if l and len(l) >= 15:
-                words = re.findall(r'[A-Za-z]{2,}', l)
-                if len(words) >= 2:
-                    cur.append(l)
-    if cur:
-        blocks.append(cur)
+def _clean_fb_text(text: str) -> str:
+    cleaned_lines = []
+    for l in text.split('\n'):
+        stripped = l.strip()
+        if not stripped:
+            continue
+        if len(stripped) < 3:
+            continue
+        if all(not c.isalpha() for c in stripped):
+            continue
+        cleaned_lines.append(stripped)
+    return '\n'.join(cleaned_lines)
 
+def _extract_posts_from_elements(elements: list[dict], base_url: str) -> list[dict]:
     posts = []
     seen_texts = set()
-    for block in blocks:
-        if not block:
+    for el in elements:
+        raw_text = el.get('text', '')
+        if len(raw_text) < 40:
             continue
-        kw_indices = [i for i, l in enumerate(block) if kw_match(l)]
-        if not kw_indices:
+        text = _clean_fb_text(raw_text)
+        if len(text) < 40:
             continue
-        i = kw_indices[0]
-        start = max(0, i - 2)
-        end = min(len(block), i + 5)
-        snippet = ' '.join(block[start:end])
-        if len(snippet) < 40:
+        if is_offer(text):
             continue
-        dekey = snippet[:80]
+        lines = text.split('\n')
+        dekey = text[:80]
         if dekey in seen_texts:
             continue
         seen_texts.add(dekey)
+        request_score = 2 if is_request(text) else 0
+        post_url = el.get('url') or base_url
+        posts.append({
+            "title": text[:300],
+            "content": text[:1000],
+            "author": el.get('author', ''),
+            "url": post_url,
+            "source": "facebook",
+            "date": _parse_fb_date(lines),
+            "_score": request_score,
+        })
+    posts.sort(key=lambda p: p.get('_score', 0), reverse=True)
+    for p in posts:
+        p.pop('_score', None)
+    return posts
+
+def _extract_posts_from_text(raw: str, url: str) -> list[dict]:
+    lines = [l.strip() for l in raw.split('\n') if l.strip() and len(l.strip()) >= 15]
+    posts = []
+    seen_texts = set()
+    cur = []
+    for l in lines:
+        words = re.findall(r'[A-Za-z]{2,}', l)
+        if len(words) < 2:
+            continue
+        lower = l.lower()
+        if any(kw in lower for kw in BROAD_KEYWORDS) and not is_offer(l):
+            if cur:
+                snippet = ' '.join(cur[-3:] + [l])
+                if len(snippet) >= 40 and not is_offer(snippet):
+                    dekey = snippet[:80]
+                    if dekey not in seen_texts:
+                        seen_texts.add(dekey)
                         posts.append({
                             "title": snippet[:300],
                             "content": snippet[:1000],
-            "author": block[start] if start < i else '',
+                            "author": '',
                             "url": url,
                             "source": "facebook",
-            "date": _parse_fb_date(block),
+                            "date": _parse_fb_date(cur + [l]),
                         })
+                cur = []
+            else:
+                if not is_offer(l):
+                    snippet = l
+                    if len(snippet) >= 40:
+                        dekey = snippet[:80]
+                        if dekey not in seen_texts:
+                            seen_texts.add(dekey)
+                            posts.append({
+                                "title": snippet[:300],
+                                "content": snippet[:1000],
+                                "author": '',
+                                "url": url,
+                                "source": "facebook",
+                                "date": _parse_fb_date([l]),
+                            })
+        cur.append(l)
+        if len(cur) > 10:
+            cur.pop(0)
     return posts
 
 async def human_scroll(page, steps: int = None, total_delay: float = None):
@@ -212,15 +327,60 @@ async def random_idle(page):
         except Exception:
             pass
 
+async def _get_article_elements(page) -> list[dict]:
+    return await page.evaluate('''() => {
+        const results = [];
+        const seenTexts = new Set();
+        const selectors = [
+            'div[role="article"]',
+            'div[role="feed"] > div',
+            'div.x1yztbdb',
+            'div[data-pagelet]',
+        ];
+        for (const sel of selectors) {
+            const els = document.querySelectorAll(sel);
+            for (const el of els) {
+                const text = (el.innerText || '').trim();
+                if (text.length < 40) continue;
+                const key = text.substring(0, 80);
+                if (seenTexts.has(key)) continue;
+                seenTexts.add(key);
+                const links = el.querySelectorAll('a');
+                let author = '';
+                for (const a of links) {
+                    const t = (a.innerText || '').trim();
+                    if (t && t.length > 2 && t.length < 80 && /^[a-zA-ZÀ-ÿ][a-zA-ZÀ-ÿ .'-]+$/.test(t)) {
+                        author = t;
+                        break;
+                    }
+                }
+                let postUrl = '';
+                for (const a of links) {
+                    const h = (a.getAttribute('href') || '');
+                    if (h.includes('/posts/') || h.includes('/photo/') || h.includes('/video/') || h.includes('/groups/')) {
+                        postUrl = h.startsWith('http') ? h : 'https://www.facebook.com' + h;
+                        break;
+                    }
+                }
+                results.push({ text, author, url: postUrl });
+            }
+            if (results.length > 0) break;
+        }
+        return results;
+    }''')
+
 async def search_facebook(page, query: str) -> list[dict]:
     url = f'https://www.facebook.com/search/posts/?q={urllib.parse.quote(query)}'
     try:
         await page.goto(url, wait_until='domcontentloaded', timeout=30000)
+        try:
+            await page.wait_for_selector('div[role="article"], div[role="feed"]', timeout=15000)
+        except Exception:
+            pass
         await page.wait_for_timeout(random.randint(3000, 8000))
 
         await human_scroll(page, steps=random.randint(2, 4), total_delay=random.uniform(6, 15))
 
-        # Accidental overscroll: 10% chance to jump to bottom and back
         if random.random() < 0.1:
             await page.evaluate("window.scrollTo(0, document.body.scrollHeight)")
             await page.wait_for_timeout(random.randint(1000, 3000))
@@ -234,11 +394,15 @@ async def search_facebook(page, query: str) -> list[dict]:
         if random.random() < 0.3:
             await random_idle(page)
 
+        raw_articles = await _get_article_elements(page)
+        posts = _extract_posts_from_elements(raw_articles, url) if raw_articles else []
+        if not posts:
             raw = await page.evaluate('document.body.innerText')
+            posts = _extract_posts_from_text(raw, url)
     except Exception as e:
         logger.warning("Facebook search failed: %s", e)
         return []
-    return _extract_posts_from_text(raw, url)
+    return posts
 
 def cleanup_chrome():
     import subprocess, signal
@@ -306,7 +470,7 @@ async def scrape_facebook(profile_path: str | None = None, force: bool = False)
                 return {"success": True, "leads": [], "flagged": False, "flag_reason": None, "error": None}
 
             all_posts = []
-            searches = random.sample(FB_SEARCHES, k=random.randint(3, 6))
+            searches = random.sample(FB_SEARCHES, k=random.randint(5, 8))
             for i, query in enumerate(searches):
                 try:
                     posts = await search_facebook(page, query)
@@ -417,16 +581,18 @@ Return a JSON array like ["yes","no","yes"] matching the order above."""
             "need a website", "my website", "business website",
             "need a designer", "help with my website",
             "redesign", "update my website",
+            "looking for", "need a", "need an", "looking to",
+            "need someone", "hire a", "want someone",
+            "need help with", "would like", "build me",
+            "design my", "make me a", "create my",
         ]
         filtered = []
         for r in results:
             t = r['title'].lower()
             if not any(kw in t for kw in strict_keywords):
                 continue
-            if any(kw in t for kw in ['i build', 'i offer', 'i create', 'i am a', 'web developer available',
-                                       'affordable web', 'web hosting', 'free website',
-                                       'limited time', 'special offer', 'sign up now',
-                                       'offer']):
+            if any(kw in t for kw in ['i build website', 'i offer web', 'i am a web developer',
+                                       'affordable web design package', 'web hosting']):
                 continue
             filtered.append(r)
         return filtered[:10]