diff --git a/browser-use-service/main.py b/browser-use-service/main.py index 5b113b3..f1d701f 100644 --- a/browser-use-service/main.py +++ b/browser-use-service/main.py @@ -1,5 +1,10 @@ -import os, json, asyncio, re, shutil, sqlite3, urllib.parse, random, logging +import os, json, asyncio, re, shutil, sqlite3, urllib.parse, random, logging, httpx from datetime import datetime, timedelta +from dotenv import load_dotenv + +dotenv_path = os.path.join(os.path.dirname(__file__), '..', '.env.local') +if os.path.exists(dotenv_path): + load_dotenv(dotenv_path) from fastapi import FastAPI, Query from fastapi.middleware.cors import CORSMiddleware import uvicorn @@ -19,8 +24,10 @@ PORT = int(os.getenv("PORT", "3008")) OLLAMA_URL = os.getenv("OLLAMA_URL", "http://localhost:11434") CLASSIFY_MODEL = os.getenv("CLASSIFY_MODEL", "dolphin-llama3:8b") - FX_PROFILE = os.getenv('FX_PROFILE', '') +BU_API_KEY = os.getenv('BU_API_KEY', '') + +BU_API_BASE = "https://api.browser-use.com/api/v3" BROAD_KEYWORDS = [ "website", "web design", "web develop", "web dev", @@ -38,107 +45,164 @@ BROAD_KEYWORDS = [ ] OFFER_PATTERNS = [ - r"\bfree\s+domain\b", - r"\bweb\s+hosting\b", - r"\bfree\s+website\b", - r"\bprofessional\s+website", - r"\baffordable\s+web\b", - r"\bget\s+a\s+free\b", - r"\bsee\s+more\b", - r"\bbook\s+your\b", - r"\bhire\s+a\s+freelancer\b", - r"\bcontent\s+strategy\b", - r"\bdigital\s+marketing\b", - r"\bseo\s+service", - r"\bsocial\s+media\b", - r"\blimited\s+time\s+offer\b", - r"\bspecial\s+offer\b", - r"\bsign\s+up\s+now\b", - r"\br\d[\d,.]", - r"\bstarting\s+at\b", - r"\bper\s+month\b", - r"\bmonthly\b", - r"\bfreelancer\s+marketplace\b", - r"\bfirst\s+order\b", - r"\bdomain\s+name\b", - r"\bregister\s+your\s+domain\b", - r"\bsponsored\b", - r"\bpromoted\b", - r"\badvertisement\b", + r"\bfree\s+domain\b", r"\bweb\s+hosting\b", r"\bfree\s+website\b", + r"\bprofessional\s+website", r"\baffordable\s+web\b", r"\bget\s+a\s+free\b", + r"\bsee\s+more\b", r"\bbook\s+your\b", r"\bhire\s+a\s+freelancer\b", + r"\bcontent\s+strategy\b", r"\bdigital\s+marketing\b", r"\bseo\s+service", + r"\bsocial\s+media\b", r"\blimited\s+time\s+offer\b", r"\bspecial\s+offer\b", + r"\bsign\s+up\s+now\b", r"\br\d[\d,.]", r"\bstarting\s+at\b", r"\bper\s+month\b", + r"\bmonthly\b", r"\bfreelancer\s+marketplace\b", r"\bfirst\s+order\b", + r"\bdomain\s+name\b", r"\bregister\s+your\s+domain\b", r"\bsponsored\b", + r"\bpromoted\b", r"\badvertisement\b", ] REQUEST_PATTERNS = [ - r"\bi\s+need\b", - r"\bi\s+need\s+someone\b", - r"\bi\s+need\s+a\b", - r"\bi\s+need\s+an\b", - r"\bi\s+want\b", - r"\bi\s+would\s+like\b", - r"\bi'm\s+looking\b", - r"\bi\s+am\s+looking\b", - r"\blooking\s+for\b", - r"\blooking\s+to\b", - r"\bwho\s+can\b", - r"\bcan\s+someone\b", - r"\bhelp\s+me\b", - r"\bneed\s+help\b", - r"\bwould\s+like\s+to\b", - r"\bwant\s+to\b", - r"\banyone\s+know\b", - r"\bdoes\s+anyone\b", - r"\brecommend\b", - r"\bsuggestion\b", + r"\bi\s+need\b", r"\bi\s+need\s+someone\b", r"\bi\s+need\s+a\b", r"\bi\s+need\s+an\b", + r"\bi\s+want\b", r"\bi\s+would\s+like\b", r"\bi'm\s+looking\b", r"\bi\s+am\s+looking\b", + r"\blooking\s+for\b", r"\blooking\s+to\b", r"\bwho\s+can\b", r"\bcan\s+someone\b", + r"\bhelp\s+me\b", r"\bneed\s+help\b", r"\bwould\s+like\s+to\b", r"\bwant\s+to\b", + r"\banyone\s+know\b", r"\bdoes\s+anyone\b", r"\brecommend\b", r"\bsuggestion\b", r"\bquote\s+(please|for|to)\b", ] -def kw_match(text: str) -> bool: - t = text.lower() - return any(kw in t for kw in BROAD_KEYWORDS) - -def is_request(text: str) -> bool: - t = text.lower() - return any(re.search(p, t) for p in REQUEST_PATTERNS) - -def is_offer(text: str) -> bool: - t = text.lower() - return any(re.search(p, t) for p in OFFER_PATTERNS) - FB_SEARCHES = [ - "looking for web developer", - "need a website designed", - "need website built South Africa", - "need someone to build my website", - "need web designer", - "looking for someone to create website", - "need ecommerce website built", - "want to hire web developer", - "need wordpress website", - "I need a website for my business", - "need a site for my business", - "looking for website designer South Africa", - "need website for my small business", - "who can build me a website", - "want to create a website for my business", - "looking for affordable web design", - "need a web developer South Africa", - "help me build a website", - "need someone to design my website", - "looking for web designer near me", + "looking for web developer", "need a website designed", + "need website built South Africa", "need someone to build my website", + "need web designer", "looking for someone to create website", + "need ecommerce website built", "want to hire web developer", + "need wordpress website", "I need a website for my business", + "need a site for my business", "looking for website designer South Africa", + "need website for my small business", "who can build me a website", + "want to create a website for my business", "looking for affordable web design", + "need a web developer South Africa", "help me build a website", + "need someone to design my website", "looking for web designer near me", "need a website for my startup", ] -VIEWPORTS = [ - {'width': 1280, 'height': 800}, - {'width': 1366, 'height': 768}, - {'width': 1440, 'height': 900}, - {'width': 1536, 'height': 864}, - {'width': 1920, 'height': 1080}, -] +def is_offer(text: str) -> bool: + return any(re.search(p, text.lower()) for p in OFFER_PATTERNS) + +def is_request(text: str) -> bool: + return any(re.search(p, text.lower()) for p in REQUEST_PATTERNS) + +WEEKDAY_ORDER = ['monday', 'tuesday', 'wednesday', 'thursday', 'friday', 'saturday', 'sunday'] + +def _parse_fb_date(block: list[str]) -> str: + for l in block: + l = l.strip() + m = re.match(r'(\d+)\s*(h|hr|hrs|hour|hours)\s*ago', l) + if m: + d = datetime.now() - timedelta(hours=int(m.group(1))) + return d.strftime('%Y-%m-%d') + if l.lower() == 'yesterday': + return (datetime.now() - timedelta(days=1)).strftime('%Y-%m-%d') + low = l.lower() + if low in WEEKDAY_ORDER: + day_idx = WEEKDAY_ORDER.index(low) + today_idx = datetime.now().weekday() + d = datetime.now() - timedelta(days=(today_idx - day_idx) % 7) + return d.strftime('%Y-%m-%d') + for fmt in ['%Y-%m-%d', '%d %B %Y', '%B %d %Y', '%d %b %Y', '%b %d %Y', '%d %b', '%b %d']: + try: + return datetime.strptime(l, fmt).strftime('%Y-%m-%d') + except ValueError: + pass + return datetime.now().strftime('%Y-%m-%d') + +def _clean_fb_text(text: str) -> str: + cleaned = [] + for l in text.split('\n'): + s = l.strip() + if len(s) < 3 or all(not c.isalpha() for c in s): + continue + cleaned.append(s) + return '\n'.join(cleaned) + +def _extract_posts_from_elements(elements: list[dict], base_url: str) -> list[dict]: + posts = [] + seen = set() + for el in elements: + raw = el.get('text', '') + if len(raw) < 40: + continue + text = _clean_fb_text(raw) + if len(text) < 40 or is_offer(text): + continue + dekey = text[:80] + if dekey in seen: + continue + seen.add(dekey) + score = 2 if is_request(text) else 0 + posts.append({ + "title": text[:300], "content": text[:1000], + "author": el.get('author', ''), "url": el.get('url') or base_url, + "source": "facebook", "date": _parse_fb_date(text.split('\n')), + "_score": score, + }) + posts.sort(key=lambda p: p['_score'], reverse=True) + for p in posts: + p.pop('_score', None) + return posts + +def _extract_posts_from_text(raw: str, url: str) -> list[dict]: + lines = [l.strip() for l in raw.split('\n') if l.strip() and len(l.strip()) >= 15] + posts, seen = [], set() + cur = [] + for l in lines: + words = re.findall(r'[A-Za-z]{2,}', l) + if len(words) < 2: + continue + lower = l.lower() + if any(kw in lower for kw in BROAD_KEYWORDS) and not is_offer(l): + snippet = ' '.join(cur[-3:] + [l]) if cur else l + if len(snippet) >= 40 and not is_offer(snippet) and snippet[:80] not in seen: + seen.add(snippet[:80]) + posts.append({"title": snippet[:300], "content": snippet[:1000], + "author": '', "url": url, "source": "facebook", + "date": _parse_fb_date(cur + [l] if cur else [l])}) + cur = [] + cur.append(l) + if len(cur) > 10: + cur.pop(0) + return posts + +async def _get_article_elements(page) -> list[dict]: + return await page.evaluate('''() => { + const results = []; + const seenTexts = new Set(); + const selectors = ['div[role="article"]', 'div[role="feed"] > div', 'div.x1yztbdb', 'div[data-pagelet]']; + for (const sel of selectors) { + const els = document.querySelectorAll(sel); + for (const el of els) { + const text = (el.innerText || '').trim(); + if (text.length < 40) continue; + const key = text.substring(0, 80); + if (seenTexts.has(key)) continue; + seenTexts.add(key); + const links = el.querySelectorAll('a'); + let author = '', postUrl = ''; + for (const a of links) { + const t = (a.innerText || '').trim(); + if (t && t.length > 2 && t.length < 80 && /^[a-zA-ZÀ-ÿ][a-zA-ZÀ-ÿ .'-]+$/.test(t)) { + author = t; break; + } + } + for (const a of links) { + const h = (a.getAttribute('href') || ''); + if (h.includes('/posts/') || h.includes('/photo/') || h.includes('/video/') || h.includes('/groups/')) { + postUrl = h.startsWith('http') ? h : 'https://www.facebook.com' + h; break; + } + } + results.push({ text, author, url: postUrl }); + } + if (results.length > 0) break; + } + return results; + }''') async def get_fb_cookies(profile_path: str | None = None): cookie_db_path = profile_path or FX_PROFILE if not cookie_db_path: - logger.warning("No profile path provided") return [] cookie_db = os.path.join(cookie_db_path, 'cookies.sqlite') if not os.path.exists(cookie_db): @@ -153,367 +217,108 @@ async def get_fb_cookies(profile_path: str | None = None): if attempt < 2: await asyncio.sleep(0.5) else: - logger.error("Failed to copy cookie DB after 3 attempts") return [] try: conn = sqlite3.connect(tmp) - c = conn.cursor() - c.execute("SELECT name, value, host, path FROM moz_cookies WHERE host LIKE '%facebook.com'") - rows = c.fetchall() + rows = conn.cursor().execute( + "SELECT name, value, host, path FROM moz_cookies WHERE host LIKE '%facebook.com'" + ).fetchall() conn.close() - try: - os.remove(tmp) - except Exception: - pass - return [{ - "name": name, "value": value, - "domain": host if host.startswith('.') else f'.{host}', - "path": path, - "httpOnly": True, "secure": True, "sameSite": "Lax", - } for name, value, host, path in rows] + os.remove(tmp) + return [{"name": n, "value": v, "domain": h if h.startswith('.') else f'.{h}', + "path": p, "httpOnly": True, "secure": True, "sameSite": "Lax"} + for n, v, h, p in rows] except Exception as e: - logger.error("Cookie DB read error: %s", e) - try: - os.remove(tmp) - except Exception: - pass + logger.error("Cookie DB error: %s", e) + try: os.remove(tmp) + except: pass return [] -WEEKDAY_ORDER = ['monday', 'tuesday', 'wednesday', 'thursday', 'friday', 'saturday', 'sunday'] +async def create_browser_session(): + if not BU_API_KEY: + raise RuntimeError("BU_API_KEY not set") + async with httpx.AsyncClient(timeout=30) as client: + r = await client.post( + f"{BU_API_BASE}/browsers", + headers={"X-Browser-Use-API-Key": BU_API_KEY}, + json={"timeout": 60, "proxyCountryCode": "za"} + ) + r.raise_for_status() + data = r.json() + return data["id"], data["cdpUrl"] -def _parse_fb_date(block: list[str]) -> str: - for l in block: - l = l.strip() - m = re.match(r'(\d+)\s*(h|hr|hrs|hour|hours)\s*ago', l) - if m: - hours = int(m.group(1)) - d = datetime.now() - timedelta(hours=hours) - return d.strftime('%Y-%m-%d') - if l.lower() == 'yesterday': - d = datetime.now() - timedelta(days=1) - return d.strftime('%Y-%m-%d') - low = l.lower() - if low in WEEKDAY_ORDER: - day_idx = WEEKDAY_ORDER.index(low) - today_idx = datetime.now().weekday() - days_ago = (today_idx - day_idx) % 7 - d = datetime.now() - timedelta(days=days_ago) - return d.strftime('%Y-%m-%d') - for fmt in ['%Y-%m-%d', '%d %B %Y', '%B %d %Y', '%d %b %Y', '%b %d %Y', '%d %b', '%b %d']: - try: - dt = datetime.strptime(l, fmt) - return dt.strftime('%Y-%m-%d') - except ValueError: - pass - return datetime.now().strftime('%Y-%m-%d') - -def _clean_fb_text(text: str) -> str: - cleaned_lines = [] - for l in text.split('\n'): - stripped = l.strip() - if not stripped: - continue - if len(stripped) < 3: - continue - if all(not c.isalpha() for c in stripped): - continue - cleaned_lines.append(stripped) - return '\n'.join(cleaned_lines) - -def _extract_posts_from_elements(elements: list[dict], base_url: str) -> list[dict]: - posts = [] - seen_texts = set() - for el in elements: - raw_text = el.get('text', '') - if len(raw_text) < 40: - continue - text = _clean_fb_text(raw_text) - if len(text) < 40: - continue - if is_offer(text): - continue - lines = text.split('\n') - dekey = text[:80] - if dekey in seen_texts: - continue - seen_texts.add(dekey) - request_score = 2 if is_request(text) else 0 - post_url = el.get('url') or base_url - posts.append({ - "title": text[:300], - "content": text[:1000], - "author": el.get('author', ''), - "url": post_url, - "source": "facebook", - "date": _parse_fb_date(lines), - "_score": request_score, - }) - posts.sort(key=lambda p: p.get('_score', 0), reverse=True) - for p in posts: - p.pop('_score', None) - return posts - -def _extract_posts_from_text(raw: str, url: str) -> list[dict]: - lines = [l.strip() for l in raw.split('\n') if l.strip() and len(l.strip()) >= 15] - posts = [] - seen_texts = set() - cur = [] - for l in lines: - words = re.findall(r'[A-Za-z]{2,}', l) - if len(words) < 2: - continue - lower = l.lower() - if any(kw in lower for kw in BROAD_KEYWORDS) and not is_offer(l): - if cur: - snippet = ' '.join(cur[-3:] + [l]) - if len(snippet) >= 40 and not is_offer(snippet): - dekey = snippet[:80] - if dekey not in seen_texts: - seen_texts.add(dekey) - posts.append({ - "title": snippet[:300], - "content": snippet[:1000], - "author": '', - "url": url, - "source": "facebook", - "date": _parse_fb_date(cur + [l]), - }) - cur = [] - else: - if not is_offer(l): - snippet = l - if len(snippet) >= 40: - dekey = snippet[:80] - if dekey not in seen_texts: - seen_texts.add(dekey) - posts.append({ - "title": snippet[:300], - "content": snippet[:1000], - "author": '', - "url": url, - "source": "facebook", - "date": _parse_fb_date([l]), - }) - cur.append(l) - if len(cur) > 10: - cur.pop(0) - return posts - -async def human_scroll(page, steps: int = None, total_delay: float = None): - steps = steps or random.randint(2, 5) - total_delay = total_delay or random.uniform(6, 18) - step_delay = total_delay / steps - for i in range(steps): - scroll_dist = random.randint(200, 600) - await page.evaluate(f"window.scrollBy(0, {scroll_dist})") - await page.wait_for_timeout(int(step_delay * 1000)) - if random.random() < 0.3: - await page.evaluate(f"window.scrollBy(0, -{random.randint(100, 300)})") - await page.wait_for_timeout(random.randint(1000, 3000)) - -async def random_idle(page): - action = random.random() - if action < 0.15: - try: - elems = await page.query_selector_all('a, span, div[role="button"]') - if elems: - el = random.choice(elems[:20]) - box = await el.bounding_box() - if box: - x = box['x'] + box['width'] / 2 + random.randint(-20, 20) - y = box['y'] + box['height'] / 2 + random.randint(-10, 10) - await page.mouse.move(x, y, steps=random.randint(20, 40)) - await page.wait_for_timeout(random.randint(500, 2000)) - except Exception: - pass - -async def _get_article_elements(page) -> list[dict]: - return await page.evaluate('''() => { - const results = []; - const seenTexts = new Set(); - const selectors = [ - 'div[role="article"]', - 'div[role="feed"] > div', - 'div.x1yztbdb', - 'div[data-pagelet]', - ]; - for (const sel of selectors) { - const els = document.querySelectorAll(sel); - for (const el of els) { - const text = (el.innerText || '').trim(); - if (text.length < 40) continue; - const key = text.substring(0, 80); - if (seenTexts.has(key)) continue; - seenTexts.add(key); - const links = el.querySelectorAll('a'); - let author = ''; - for (const a of links) { - const t = (a.innerText || '').trim(); - if (t && t.length > 2 && t.length < 80 && /^[a-zA-ZÀ-ÿ][a-zA-ZÀ-ÿ .'-]+$/.test(t)) { - author = t; - break; - } - } - let postUrl = ''; - for (const a of links) { - const h = (a.getAttribute('href') || ''); - if (h.includes('/posts/') || h.includes('/photo/') || h.includes('/video/') || h.includes('/groups/')) { - postUrl = h.startsWith('http') ? h : 'https://www.facebook.com' + h; - break; - } - } - results.push({ text, author, url: postUrl }); - } - if (results.length > 0) break; - } - return results; - }''') - -async def _ensure_page(page, context): +async def stop_browser_session(session_id: str): + if not BU_API_KEY: + return try: - await page.evaluate('1') - return page - except Exception: - logger.warning("Page was closed mid-scrape, creating a fresh page") - page = await context.new_page() - try: - await page.goto('https://www.google.com/', wait_until='domcontentloaded', timeout=15000) - await page.wait_for_timeout(random.randint(1000, 3000)) - except Exception: - logger.warning("Google navigation failed during page recreation") - await page.goto('https://www.facebook.com/', wait_until='domcontentloaded', timeout=30000) - await page.wait_for_timeout(random.randint(3000, 8000)) - return page + async with httpx.AsyncClient(timeout=10) as client: + await client.delete( + f"{BU_API_BASE}/browsers/{session_id}", + headers={"X-Browser-Use-API-Key": BU_API_KEY} + ) + except Exception as e: + logger.warning("Failed to stop BU session: %s", e) -async def search_facebook(page, context, query: str): - page = await _ensure_page(page, context) +async def search_facebook(page, query: str) -> list[dict]: url = f'https://www.facebook.com/search/posts/?q={urllib.parse.quote(query)}' try: await page.goto(url, wait_until='domcontentloaded', timeout=30000) try: - await page.wait_for_selector('div[role="article"], div[role="feed"]', timeout=15000) + await page.wait_for_selector('div[role="article"], div[role="feed"]', timeout=10000) except Exception: pass - await page.wait_for_timeout(random.randint(3000, 8000)) - - await human_scroll(page, steps=random.randint(2, 4), total_delay=random.uniform(6, 15)) - - if random.random() < 0.1: - await page.evaluate("window.scrollTo(0, document.body.scrollHeight)") - await page.wait_for_timeout(random.randint(1000, 3000)) - await page.evaluate("window.scrollBy(0, -300)") - await page.wait_for_timeout(random.randint(1000, 3000)) - - if random.random() < 0.2: - await page.evaluate("window.scrollTo(0, 0)") - await page.wait_for_timeout(random.randint(2000, 5000)) - - if random.random() < 0.3: - await random_idle(page) - + await page.wait_for_timeout(2000) + await page.evaluate("window.scrollBy(0, 600)") + await page.wait_for_timeout(1500) raw_articles = await _get_article_elements(page) posts = _extract_posts_from_elements(raw_articles, url) if raw_articles else [] if not posts: raw = await page.evaluate('document.body.innerText') posts = _extract_posts_from_text(raw, url) + return posts except Exception as e: logger.warning("Facebook search '%s' failed: %s", query, e) - return page, [] - return page, posts - -def cleanup_chrome(): - import subprocess, signal - try: - subprocess.run(["taskkill", "/F", "/IM", "chrome-headless-shell.exe"], capture_output=True, timeout=5) - except Exception: - pass + return [] async def scrape_facebook(profile_path: str | None = None, force: bool = False) -> dict: - cleanup_chrome() fb_cookies = await get_fb_cookies(profile_path) if not fb_cookies: - logger.warning("No Facebook cookies available") return {"success": False, "leads": [], "flagged": False, "flag_reason": None, "error": "No cookies available"} + + session_id = None + try: + session_id, cdp_url = await create_browser_session() + logger.info("BU session %s created, connecting via CDP", session_id) + except Exception as e: + logger.error("Failed to create BU session: %s", e) + return {"success": False, "leads": [], "flagged": False, "flag_reason": None, "error": f"BU session failed: {e}"} + try: async with async_playwright() as pw: - browser = await pw.chromium.launch( - headless=True, - args=['--disable-blink-features=AutomationControlled'] - ) - viewport = random.choice(VIEWPORTS) - context = await browser.new_context( - user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36', - viewport=viewport, - ) + browser = await pw.chromium.connect_over_cdp(cdp_url) + ctx = browser.contexts[0] if browser.contexts else await browser.new_context() + if not browser.contexts: + for c in fb_cookies: + try: await ctx.add_cookies([c]) + except Exception: pass - await context.add_init_script(""" - Object.defineProperty(navigator, 'webdriver', { get: () => false }); - Object.defineProperty(navigator, 'languages', { get: () => ['en-US', 'en'] }); - Object.defineProperty(navigator, 'platform', { get: () => 'Win32' }); - window.chrome = { runtime: {} }; - Object.defineProperty(navigator, 'hardwareConcurrency', { get: () => 8 }); - Object.defineProperty(navigator, 'deviceMemory', { get: () => 8 }); - """) - for c in fb_cookies: - try: - await context.add_cookies([c]) - except Exception as e: - logger.warning("Failed to inject cookie %s: %s", c.get('name'), e) - - page = await context.new_page() - # Navigate through google first for a legitimate Referer header - try: - await page.goto('https://www.google.com/', wait_until='domcontentloaded', timeout=15000) - await page.wait_for_timeout(random.randint(1000, 3000)) - except Exception: - logger.warning("Google navigation failed (IP may be blocked), trying Facebook directly") + page = ctx.pages[0] if ctx.pages else await ctx.new_page() await page.goto('https://www.facebook.com/', wait_until='domcontentloaded', timeout=30000) - await page.wait_for_timeout(random.randint(3000, 8000)) + await page.wait_for_timeout(2000) url = page.url if '/login' in url.lower(): - logger.warning("Facebook login page detected — account flagged") + logger.warning("Facebook login page detected") await browser.close() + await stop_browser_session(session_id) return {"success": False, "leads": [], "flagged": True, "flag_reason": "login_page", "error": "Facebook login page detected"} - # Browse feed like a human before searching - await human_scroll(page, steps=random.randint(2, 4), total_delay=random.uniform(8, 20)) - if random.random() < 0.25: - await page.evaluate("window.scrollTo(0, 0)") - await page.wait_for_timeout(random.randint(2000, 5000)) - await human_scroll(page, steps=random.randint(1, 2)) - - # False start: 30% chance to idle and leave without scraping (skipped when force=true) - if not force and random.random() < 0.3: - await page.wait_for_timeout(random.randint(8000, 20000)) - await browser.close() - return {"success": True, "leads": [], "flagged": False, "flag_reason": None, "error": None} - all_posts = [] searches = random.sample(FB_SEARCHES, k=random.randint(5, 8)) - for i, query in enumerate(searches): - page, posts = await search_facebook(page, context, query) + for query in searches: + posts = await search_facebook(page, query) all_posts.extend(posts) - if not posts: - continue - # Between searches: random break with possible small scroll - if random.random() < 0.4: - await page.evaluate(f"window.scrollBy(0, {random.randint(-300, 300)})") - delay = random.uniform(8, 25) - await page.wait_for_timeout(int(delay * 1000)) - # Tab switch: 15% chance after 2nd-3rd search - if i == random.randint(1, 2) and random.random() < 0.15: - new_page = await context.new_page() - await new_page.goto('https://www.messenger.com/', wait_until='domcontentloaded', timeout=15000) - await new_page.wait_for_timeout(random.randint(3000, 8000)) - await new_page.close() - page = await _ensure_page(page, context) - - # Idle before closing — human would pause - if random.random() < 0.5: - await page.wait_for_timeout(random.randint(3000, 10000)) - - await browser.close() + await page.wait_for_timeout(1000) seen = set() deduped = [] @@ -527,13 +332,17 @@ async def scrape_facebook(profile_path: str | None = None, force: bool = False) if leads: leads = await classify_leads(leads) + await browser.close() + await stop_browser_session(session_id) + return {"success": True, "leads": leads[:15], "flagged": False, "flag_reason": None, "error": None} + except Exception as e: logger.error("Facebook scrape failed: %s", e) + await stop_browser_session(session_id) return {"success": False, "leads": [], "flagged": False, "flag_reason": None, "error": str(e)} async def ask_ollama(prompt: str) -> str: - import httpx async with httpx.AsyncClient(timeout=120) as c: r = await c.post(f"{OLLAMA_URL}/api/chat", json={ "model": CLASSIFY_MODEL, @@ -545,8 +354,7 @@ async def ask_ollama(prompt: str) -> str: "options": {"temperature": 0.05, "num_predict": 1024}, }) r.raise_for_status() - data = r.json() - return data["message"]["content"] + return r.json()["message"]["content"] async def classify_leads(results: list[dict]) -> list[dict]: if not results: @@ -568,8 +376,7 @@ For each numbered post, answer ONLY "yes" (LEAD) or "no" (NOT LEAD): Return a JSON array like ["yes","no","yes"] matching the order above.""" ai_succeeded = False try: - raw = await ask_ollama(prompt) - raw = raw.strip() + raw = (await ask_ollama(prompt)).strip() if raw.startswith("```"): first_nl = raw.find('\n') if first_nl != -1: @@ -580,35 +387,24 @@ Return a JSON array like ["yes","no","yes"] matching the order above.""" answers = json.loads(raw) if isinstance(answers, list) and len(answers) == len(results): ai_succeeded = True - filtered = [] - for i, ans in enumerate(answers): - if isinstance(ans, str) and ans.lower() == 'yes': - filtered.append(results[i]) + filtered = [results[i] for i, ans in enumerate(answers) if isinstance(ans, str) and ans.lower() == 'yes'] if filtered: return filtered[:10] - logger.info("AI classified all %d items as NOT LEAD — returning empty", len(results)) - return [] except Exception as e: logger.warning("AI classification failed, falling back to keyword filter: %s", e) if not ai_succeeded: - strict_keywords = [ - "website", "web design", "web develop", "web dev", - "build my website", "build a website", "create a website", - "need web", "looking for web", "new website", - "landing page", "wordpress", - "need a website", "my website", "business website", - "need a designer", "help with my website", - "redesign", "update my website", - "looking for", "need a", "need an", "looking to", - "need someone", "hire a", "want someone", - "need help with", "would like", "build me", - "design my", "make me a", "create my", - ] + strict = ["website", "web design", "web develop", "web dev", "build my website", + "build a website", "create a website", "need web", "looking for web", "new website", + "landing page", "wordpress", "need a website", "my website", "business website", + "need a designer", "help with my website", "redesign", "update my website", + "looking for", "need a", "need an", "looking to", "need someone", "hire a", + "want someone", "need help with", "would like", "build me", "design my", + "make me a", "create my"] filtered = [] for r in results: t = r['title'].lower() - if not any(kw in t for kw in strict_keywords): + if not any(kw in t for kw in strict): continue if any(kw in t for kw in ['i build website', 'i offer web', 'i am a web developer', 'affordable web design package', 'web hosting']):