added browser

browser-use-service/main.py

@@ -1,9 +1,21 @@
-import os, json, asyncio, re, shutil, sqlite3, urllib.parse, random, logging
+import os, json, asyncio, re, shutil, sqlite3, urllib.parse, random, logging, tempfile
 from datetime import datetime, timedelta
-from fastapi import FastAPI, Query
+from fastapi import FastAPI, Query, Body
 from fastapi.middleware.cors import CORSMiddleware
 import uvicorn
 from playwright.async_api import async_playwright
+from browser_use import Agent, Browser
+from langchain_ollama import ChatOllama
+
+
+def make_ollama(model: str | None = None, **kwargs) -> ChatOllama:
+    """Create ChatOllama with required attrs for browser-use Agent compatibility."""
+    llm = ChatOllama(model=model or CLASSIFY_MODEL, **kwargs)
+    object.__setattr__(llm, 'provider', 'ollama')
+    object.__setattr__(llm, 'name', 'ChatOllama')
+    object.__setattr__(llm, 'model_name', llm.model)
+    return llm
+
 
 logging.basicConfig(level=logging.INFO, format='%(asctime)s [%(levelname)s] %(message)s')
 logger = logging.getLogger(__name__)
@@ -20,7 +32,137 @@ PORT = int(os.getenv("PORT", "3008"))
 OLLAMA_URL = os.getenv("OLLAMA_URL", "http://localhost:11434")
 CLASSIFY_MODEL = os.getenv("CLASSIFY_MODEL", "dolphin-llama3:8b")
 
+
 FX_PROFILE = os.getenv('FX_PROFILE', '')
+CHROME_PROFILE = os.getenv('CHROME_PROFILE', '')
+
+CHROME_LAUNCH_ARGS = [
+    '--headless=new',
+    '--disable-blink-features=AutomationControlled',
+    '--disable-background-networking',
+    '--no-first-run',
+    '--disable-sync',
+    '--disable-field-trial-config',
+    '--disable-background-timer-throttling',
+    '--disable-backgrounding-occluded-windows',
+    '--disable-breakpad',
+    '--disable-component-update',
+    '--disable-default-apps',
+    '--disable-dev-shm-usage',
+    '--disable-popup-blocking',
+    '--disable-prompt-on-repost',
+    '--disable-renderer-backgrounding',
+    '--metrics-recording-only',
+    '--no-default-browser-check',
+    '--disable-extensions-http-throttling',
+    '--disable-hang-monitor',
+    '--disable-ipc-flooding-protection',
+]
+
+
+def detect_browser_from_profile(profile_path: str) -> str | None:
+    """Detect browser type from profile path. Returns 'firefox', 'chromium', or None."""
+    if not profile_path:
+        return None
+    p = profile_path.lower().replace('\\', '/')
+    if 'firefox' in p or '.default' in p or '.dev-edition' in p:
+        return 'firefox'
+    if 'chrome' in p or 'chromium' in p or 'edge' in p:
+        return 'chromium'
+    return None
+
+
+def copy_firefox_profile(src_path: str) -> str:
+    """Copy essential Firefox profile files to a temp dir for Playwright."""
+    essential = ['cookies.sqlite', 'webappsstore.sqlite', 'permissions.sqlite']
+    dst = tempfile.mkdtemp(prefix='fb_fx_profile_')
+    for f_name in essential:
+        s = os.path.join(src_path, f_name)
+        if os.path.exists(s):
+            shutil.copy2(s, os.path.join(dst, f_name))
+    with open(os.path.join(dst, 'profiles.ini'), 'w') as fh:
+        fh.write("[Profile0]\nName=default\nIsRelative=0\nPath=.\nDefault=yes\n")
+    logger.info("Copied Firefox profile to %s", dst)
+    return dst
+
+
+def copy_chrome_profile(user_data_dir: str, profile_dir: str = 'Default') -> str:
+    """Copy Chrome profile to temp dir for launch_persistent_context."""
+    essential = ['Cookies', 'Login Data', 'Bookmarks', 'Web Data']
+    dst = tempfile.mkdtemp(prefix='fb_chrome_udir_')
+    src_profile = os.path.join(user_data_dir, profile_dir)
+    os.makedirs(dst, exist_ok=True)
+    os.makedirs(os.path.join(dst, profile_dir), exist_ok=True)
+    dst_profile = os.path.join(dst, profile_dir)
+    for f_name in essential:
+        s = os.path.join(src_profile, f_name)
+        if os.path.exists(s):
+            shutil.copy2(s, os.path.join(dst_profile, f_name))
+    for sub in ['Local Storage', 'Session Storage']:
+        s = os.path.join(src_profile, sub)
+        if os.path.isdir(s):
+            shutil.copytree(s, os.path.join(dst_profile, sub), dirs_exist_ok=True)
+    local_state_src = os.path.join(user_data_dir, 'Local State')
+    local_state_dst = os.path.join(dst, 'Local State')
+    if os.path.exists(local_state_src):
+        shutil.copy2(local_state_src, local_state_dst)
+    logger.info("Copied Chrome profile from %s to %s", src_profile, dst)
+    return dst
+
+
+def ensure_ublock_extension() -> str | None:
+    """Download and extract uBlock Origin extension for ad blocking."""
+    ext_dir = os.path.join(tempfile.gettempdir(), 'fb_ublock_origin')
+    manifest = os.path.join(ext_dir, 'manifest.json')
+    if os.path.exists(manifest):
+        return ext_dir
+    import urllib.request, zipfile
+    crx_url = ('https://clients2.google.com/service/update2/crx'
+               '?response=redirect&prodversion=133&acceptformat=crx3'
+               '&x=id%3Dddkjiahejlhfcafbddmgiahcphecmpfh%26uc')
+    crx_path = os.path.join(tempfile.gettempdir(), 'ublock_origin.crx')
+    try:
+        if not os.path.exists(crx_path):
+            logger.info("Downloading uBlock Origin extension...")
+            urllib.request.urlretrieve(crx_url, crx_path)
+        with zipfile.ZipFile(crx_path, 'r') as z:
+            z.extractall(ext_dir)
+        logger.info("uBlock Origin extension ready at %s", ext_dir)
+        return ext_dir
+    except Exception as e:
+        logger.warning("Failed to load uBlock Origin extension: %s", e)
+        return None
+
+
+def extract_agent_posts(agent_result: str, page_text: str) -> list[dict]:
+    """Parse posts from agent output or page text fallback."""
+    posts = []
+    # Try JSON from agent output
+    json_match = re.search(r'\[.*?\]', agent_result, re.DOTALL)
+    if json_match:
+        try:
+            parsed = json.loads(json_match.group())
+            if isinstance(parsed, list):
+                for item in parsed:
+                    if isinstance(item, dict) and item.get('content'):
+                        posts.append({
+                            'content': item.get('content', ''),
+                            'author': item.get('author', ''),
+                            'url': item.get('url', ''),
+                            'date': item.get('date', ''),
+                        })
+        except json.JSONDecodeError:
+            pass
+
+    if not posts:
+        lines = [l.strip() for l in page_text.split('\n') if l.strip()]
+        for line in lines:
+            if len(line) > 30 and not any(skip in line.lower() for skip in
+                    ['facebook', 'messenger', 'notification', 'comment', 'like', 'share']):
+                posts.append({'content': line[:500], 'author': '', 'url': '', 'date': ''})
+
+    return posts
+
 
 BROAD_KEYWORDS = [
     "website", "web design", "web develop", "web dev",
@@ -91,6 +233,100 @@ REQUEST_PATTERNS = [
     r"\bquote\s+(please|for|to)\b",
 ]
 
+def firefox_init_script() -> str:
+    return r"""// Firefox Anti-Detection
+Object.defineProperty(navigator, 'webdriver', { get: () => undefined });
+Object.defineProperty(navigator, 'languages', { get: () => ['en-US', 'en'] });
+Object.defineProperty(navigator, 'platform', { get: () => 'Win32' });
+Object.defineProperty(navigator, 'hardwareConcurrency', { get: () => 8 });
+Object.defineProperty(navigator, 'maxTouchPoints', { get: () => 0 });
+Object.defineProperty(navigator, 'oscpu', { get: () => 'Windows NT 10.0; Win64; x64' });
+Object.defineProperty(navigator, 'vendor', { get: () => '' });
+Object.defineProperty(navigator, 'vendorSub', { get: () => '' });
+Object.defineProperty(navigator, 'productSub', { get: () => '20100101' });
+Object.defineProperty(navigator, 'product', { get: () => 'Gecko' });
+Object.defineProperty(navigator, 'appCodeName', { get: () => 'Mozilla' });
+Object.defineProperty(navigator, 'appName', { get: () => 'Netscape' });
+Object.defineProperty(navigator, 'appVersion', { get: () => '5.0 (Windows NT 10.0; Win64; x64; rv:130.0) Gecko/20100101 Firefox/130.0' });
+Object.defineProperty(navigator, 'userAgent', { get: () => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:130.0) Gecko/20100101 Firefox/130.0' });
+Object.defineProperty(navigator, 'buildID', { get: () => '20250101000000' });
+if (window.chrome !== undefined) { window.chrome = undefined; }
+if (navigator.permissions) {
+    const origQ = navigator.permissions.query.bind(navigator.permissions);
+    navigator.permissions.query = (p) => origQ(p).then(r => (['camera','microphone','geolocation','notifications'].includes(p.name) ? Object.assign(r, { state: 'prompt' }) : r));
+}
+const _gEC = HTMLCanvasElement.prototype.getContext;
+HTMLCanvasElement.prototype.getContext = function(t, ...a) {
+    const ctx = _gEC.call(this, t, ...a);
+    if (ctx && (t === 'webgl' || t === 'webgl2')) {
+        const _gP = ctx.getParameter.bind(ctx);
+        ctx.getParameter = function(p) { if (p===37445) return 'Intel Inc.'; if (p===37446) return 'Intel Iris OpenGL Engine'; if (p===7936||p===7937||p===35724) return ''; return _gP(p); };
+        const _gE = ctx.getExtension.bind(ctx);
+        ctx.getExtension = function(n) { if (n==='WEBGL_debug_renderer_info') return null; return _gE(n); };
+    }
+    return ctx;
+};
+const _tDU = HTMLCanvasElement.prototype.toDataURL;
+HTMLCanvasElement.prototype.toDataURL = function(...a) {
+    const img = _tDU.apply(this, a);
+    if (img.includes('image/png') && img.length > 5000) { const n = btoa(String.fromCharCode(Math.floor(Math.random()*256))); return img.slice(0,-100)+n+img.slice(-100); }
+    return img;
+};
+const _gFD = AnalyserNode.prototype.getFloatFrequencyData;
+if (_gFD) { AnalyserNode.prototype.getFloatFrequencyData = function(a) { _gFD.call(this,a); for(let i=0;i<a.length;i++) a[i]+=(Math.random()-0.5)*0.02; }; }
+Object.defineProperty(screen, 'colorDepth', { get: () => 24 });
+Object.defineProperty(screen, 'pixelDepth', { get: () => 24 });
+Object.defineProperty(navigator, 'plugins', { get: () => [{name:'Widevine Content Decryption Module',filename:'widevinecdm.dll',length:1,description:'Enables Widevine licenses'},{name:'OpenH264 Video Codec',filename:'openh264.dll',length:1,description:'H.264 video codec'}] });
+Object.defineProperty(navigator, 'mimeTypes', { get: () => [{type:'application/pdf',suffixes:'pdf',description:'Portable Document Format',enabledPlugin:navigator.plugins[0]}] });"""
+
+
+def chromium_init_script() -> str:
+    return r"""// Enhanced Chromium Anti-Detection
+Object.defineProperty(navigator, 'webdriver', { get: () => false });
+Object.defineProperty(navigator, 'languages', { get: () => ['en-US', 'en'] });
+Object.defineProperty(navigator, 'platform', { get: () => 'Win32' });
+Object.defineProperty(navigator, 'hardwareConcurrency', { get: () => 8 });
+Object.defineProperty(navigator, 'deviceMemory', { get: () => 8 });
+Object.defineProperty(navigator, 'maxTouchPoints', { get: () => 0 });
+Object.defineProperty(navigator, 'oscpu', { get: () => 'Windows NT 10.0; Win64; x64' });
+Object.defineProperty(navigator, 'vendor', { get: () => 'Google Inc.' });
+Object.defineProperty(navigator, 'vendorSub', { get: () => '' });
+Object.defineProperty(navigator, 'productSub', { get: () => '20030107' });
+Object.defineProperty(navigator, 'product', { get: () => 'Gecko' });
+Object.defineProperty(navigator, 'appCodeName', { get: () => 'Mozilla' });
+Object.defineProperty(navigator, 'appName', { get: () => 'Netscape' });
+Object.defineProperty(navigator, 'appVersion', { get: () => '5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36' });
+Object.defineProperty(navigator, 'userAgent', { get: () => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36' });
+window.chrome = { runtime: { onConnect: { addListener: () => {} }, onMessage: { addListener: () => {} }, sendMessage: () => {} }, app: { isInstalled: false, InstallState: { DISABLED: 'disabled', INSTALLED: 'installed', NOT_INSTALLED: 'not_installed' }, getDetails: () => null, getIsInstalled: () => false }, csi: () => {}, loadTimes: () => {} };
+if (navigator.permissions) {
+    const origQ = navigator.permissions.query.bind(navigator.permissions);
+    navigator.permissions.query = (p) => origQ(p).then(r => (['camera','microphone','geolocation','notifications'].includes(p.name) ? Object.assign(r, { state: 'prompt' }) : r));
+}
+const _gEC = HTMLCanvasElement.prototype.getContext;
+HTMLCanvasElement.prototype.getContext = function(t, ...a) {
+    const ctx = _gEC.call(this, t, ...a);
+    if (ctx && (t === 'webgl' || t === 'webgl2')) {
+        const _gP = ctx.getParameter.bind(ctx);
+        ctx.getParameter = function(p) { if (p===37445) return 'Intel Inc.'; if (p===37446) return 'Intel Iris OpenGL Engine'; if (p===7936||p===7937||p===35724) return ''; return _gP(p); };
+        const _gE = ctx.getExtension.bind(ctx);
+        ctx.getExtension = function(n) { if (n==='WEBGL_debug_renderer_info') return null; return _gE(n); };
+    }
+    return ctx;
+};
+const _tDU = HTMLCanvasElement.prototype.toDataURL;
+HTMLCanvasElement.prototype.toDataURL = function(...a) {
+    const img = _tDU.apply(this, a);
+    if (img.includes('image/png') && img.length > 5000) { const n = btoa(String.fromCharCode(Math.floor(Math.random()*256))); return img.slice(0,-100)+n+img.slice(-100); }
+    return img;
+};
+const _gFD = AnalyserNode.prototype.getFloatFrequencyData;
+if (_gFD) { AnalyserNode.prototype.getFloatFrequencyData = function(a) { _gFD.call(this,a); for(let i=0;i<a.length;i++) a[i]+=(Math.random()-0.5)*0.02; }; }
+Object.defineProperty(screen, 'colorDepth', { get: () => 24 });
+Object.defineProperty(screen, 'pixelDepth', { get: () => 24 });
+Object.defineProperty(navigator, 'plugins', { get: () => [1,2,3,4,5].map(i => ({name:['Chrome PDF Plugin','Chrome PDF Viewer','Native Client','Widevine Content Decryption Module','Chromium PDF Viewer'][i-1],filename:'internal-pdf-viewer',length:1,description:['Portable Document Format','','Native Client Executable','Enables Widevine licenses',''][i-1]})) });
+Object.defineProperty(navigator, 'mimeTypes', { get: () => [{type:'application/pdf',suffixes:'pdf',description:'Portable Document Format',enabledPlugin:navigator.plugins[0]},{type:'text/pdf',suffixes:'pdf',description:'Portable Document Format',enabledPlugin:navigator.plugins[0]}] });"""
+
+
 def kw_match(text: str) -> bool:
     t = text.lower()
     return any(kw in t for kw in BROAD_KEYWORDS)
@@ -306,6 +542,13 @@ async def human_scroll(page, steps: int = None, total_delay: float = None):
     for i in range(steps):
         scroll_dist = random.randint(200, 600)
         await page.evaluate(f"window.scrollBy(0, {scroll_dist})")
+        try:
+            vp = await page.evaluate('({w: window.innerWidth, h: window.innerHeight})')
+            x = random.randint(100, vp['w'] - 100)
+            y = random.randint(100, vp['h'] - 100)
+            await page.mouse.move(x, y, steps=random.randint(15, 35))
+        except Exception:
+            pass
         await page.wait_for_timeout(int(step_delay * 1000))
     if random.random() < 0.3:
         await page.evaluate(f"window.scrollBy(0, -{random.randint(100, 300)})")
@@ -421,6 +664,25 @@ async def search_facebook(page, context, query: str):
         return page, []
     return page, posts
 
+DETECTION_SIGNALS = [
+    '/checkpoint/', '/login.php?', 'action=security_check',
+    'unusual activity', 'suspicious login', 'suspicious activity',
+    'temporarily blocked', 'temporarily restricted',
+    'confirm your identity', 'enter the code we sent',
+    'we noticed unusual', 'help us confirm',
+    'security check', 'challenge', 'please verify',
+]
+
+
+def check_detection_signals(page_url: str, page_text: str = '') -> str | None:
+    """Check page for Facebook detection signals. Returns signal text or None."""
+    combined = (page_url + ' ' + page_text).lower()
+    for signal in DETECTION_SIGNALS:
+        if signal in combined:
+            return signal
+    return None
+
+
 def cleanup_chrome():
     import subprocess, signal
     try:
@@ -429,90 +691,183 @@ def cleanup_chrome():
         pass
 
 async def scrape_facebook(profile_path: str | None = None, force: bool = False) -> dict:
-    cleanup_chrome()
+    """Dispatcher — Firefox primary, browser-use Agent fallback."""
-    fb_cookies = await get_fb_cookies(profile_path)
+    effective_path = profile_path or FX_PROFILE
-    if not fb_cookies:
+    browser_type = detect_browser_from_profile(effective_path)
-        logger.warning("No Facebook cookies available")
+
-        return {"success": False, "leads": [], "flagged": False, "flag_reason": None, "error": "No cookies available"}
+    if not browser_type and CHROME_PROFILE:
+        browser_type = detect_browser_from_profile(CHROME_PROFILE)
+        effective_path = CHROME_PROFILE
+
+    logger.info("Detected browser: %s (profile: %s)", browser_type or "none", effective_path)
+
+    # Firefox primary (raw Playwright, stealth)
+    if browser_type == "firefox":
+        result = await _scrape_with_firefox(effective_path, force)
+        if result.get("success") or not result.get("flagged"):
+            return result
+        logger.warning("Firefox path failed (%s), falling back to Agent", result.get("flag_reason", "unknown"))
+        return await _scrape_with_agent(force)
+
+    # CHROME_PROFILE set or no profile → Agent
+    if browser_type == "chromium":
+        return await _scrape_with_agent(force)
+
+    # No profile at all → try Agent (fresh Chromium)
+    return await _scrape_with_agent(force)
+
+
+async def _scrape_with_firefox(profile_path: str, force: bool) -> dict:
+    """Scrape Facebook using Firefox + persistent real profile (no cookie injection)."""
+    if not profile_path:
+        return {"success": False, "leads": [], "flagged": False, "flag_reason": None, "error": "No profile path"}
+
+    profile_dir = None
     try:
+        profile_dir = copy_firefox_profile(profile_path)
         async with async_playwright() as pw:
-            browser = await pw.chromium.launch(
+            context = await pw.firefox.launch_persistent_context(
+                user_data_dir=profile_dir,
                 headless=True,
-                args=['--disable-blink-features=AutomationControlled']
+                firefox_user_prefs={
-            )
+                    "dom.webdriver.enabled": False,
-            viewport = random.choice(VIEWPORTS)
+                    "dom.webdriver.timeout": 0,
-            context = await browser.new_context(
+                    "useAutomationExtension": False,
-                user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36',
+                    "privacy.trackingprotection.enabled": False,
-                viewport=viewport,
+                    "privacy.trackingprotection.fingerprinting.enabled": False,
+                    "privacy.trackingprotection.cryptomining.enabled": False,
+                },
             )
+            pages = context.pages
+            page = pages[0] if pages else await context.new_page()
+            await context.add_init_script(firefox_init_script())
 
-            await context.add_init_script("""
-                Object.defineProperty(navigator, 'webdriver', { get: () => false });
-                Object.defineProperty(navigator, 'languages', { get: () => ['en-US', 'en'] });
-                Object.defineProperty(navigator, 'platform', { get: () => 'Win32' });
-                window.chrome = { runtime: {} };
-                Object.defineProperty(navigator, 'hardwareConcurrency', { get: () => 8 });
-                Object.defineProperty(navigator, 'deviceMemory', { get: () => 8 });
-            """)
-            for c in fb_cookies:
-                try:
-                    await context.add_cookies([c])
-                except Exception as e:
-                    logger.warning("Failed to inject cookie %s: %s", c.get('name'), e)
-
-            page = await context.new_page()
-            # Navigate through google first for a legitimate Referer header
             try:
                 await page.goto('https://www.google.com/', wait_until='domcontentloaded', timeout=15000)
                 await page.wait_for_timeout(random.randint(1000, 3000))
             except Exception:
-                logger.warning("Google navigation failed (IP may be blocked), trying Facebook directly")
+                logger.warning("Google navigation failed, trying Facebook directly")
+
             await page.goto('https://www.facebook.com/', wait_until='domcontentloaded', timeout=30000)
             await page.wait_for_timeout(random.randint(3000, 8000))
 
             url = page.url
-            if '/login' in url.lower():
+            page_text = await page.evaluate('document.body.innerText') if '/login' in url.lower() else ''
-                logger.warning("Facebook login page detected — account flagged")
+            det = check_detection_signals(url, page_text)
-                await browser.close()
+            if det or '/login' in url.lower():
-                return {"success": False, "leads": [], "flagged": True, "flag_reason": "login_page", "error": "Facebook login page detected"}
+                logger.warning("Facebook login page detected — flag: %s", det or "login_page")
+                await context.close()
+                return {"success": False, "leads": [], "flagged": True, "flag_reason": det or "login_page", "error": "Facebook login page detected"}
 
-            # Browse feed like a human before searching
             await human_scroll(page, steps=random.randint(2, 4), total_delay=random.uniform(8, 20))
             if random.random() < 0.25:
                 await page.evaluate("window.scrollTo(0, 0)")
                 await page.wait_for_timeout(random.randint(2000, 5000))
                 await human_scroll(page, steps=random.randint(1, 2))
 
-            # False start: 30% chance to idle and leave without scraping (skipped when force=true)
             if not force and random.random() < 0.3:
                 await page.wait_for_timeout(random.randint(8000, 20000))
-                await browser.close()
+                await context.close()
                 return {"success": True, "leads": [], "flagged": False, "flag_reason": None, "error": None}
 
             all_posts = []
-            searches = random.sample(FB_SEARCHES, k=random.randint(5, 8))
+            searches = random.sample(FB_SEARCHES, k=random.randint(2, 4))
             for i, query in enumerate(searches):
                 page, posts = await search_facebook(page, context, query)
                 all_posts.extend(posts)
                 if not posts:
                     continue
-                # Between searches: random break with possible small scroll
                 if random.random() < 0.4:
                     await page.evaluate(f"window.scrollBy(0, {random.randint(-300, 300)})")
                 delay = random.uniform(8, 25)
                 await page.wait_for_timeout(int(delay * 1000))
-                # Tab switch: 15% chance after 2nd-3rd search
+                if i == random.randint(0, 1) and random.random() < 0.15:
-                if i == random.randint(1, 2) and random.random() < 0.15:
                     new_page = await context.new_page()
-                    await new_page.goto('https://www.messenger.com/', wait_until='domcontentloaded', timeout=15000)
+                    try:
+                        await new_page.goto('https://www.facebook.com/groups/', wait_until='domcontentloaded', timeout=15000)
                         await new_page.wait_for_timeout(random.randint(3000, 8000))
+                    except Exception:
+                        pass
                     await new_page.close()
                     page = await _ensure_page(page, context)
 
-            # Idle before closing — human would pause
             if random.random() < 0.5:
                 await page.wait_for_timeout(random.randint(3000, 10000))
 
+            await context.close()
+
+            seen = set()
+            deduped = []
+            for p in all_posts:
+                key = p.get('content', '')[:100]
+                if key not in seen:
+                    seen.add(key)
+                    deduped.append(p)
+
+            leads = deduped[:20]
+            if leads:
+                leads = await classify_leads(leads)
+
+            return {"success": True, "leads": leads[:15], "flagged": False, "flag_reason": None, "error": None}
+
+    except Exception as e:
+        logger.error("Firefox scrape failed: %s", e)
+        return {"success": False, "leads": [], "flagged": False, "flag_reason": None, "error": str(e)}
+    finally:
+        if profile_dir and os.path.exists(profile_dir):
+            try:
+                shutil.rmtree(profile_dir, ignore_errors=True)
+            except Exception:
+                pass
+
+
+async def _scrape_with_agent(force: bool = False) -> dict:
+    """Fallback scraper — browser-use Agent + ChatOllama (free/local, Chromium)."""
+    cleanup_chrome()
+    profile_dir = None
+    try:
+        user_data_dir = None
+        if CHROME_PROFILE:
+            profile_dir = copy_chrome_profile(CHROME_PROFILE)
+            user_data_dir = profile_dir
+
+        browser = Browser(
+            headless=True,
+            args=CHROME_LAUNCH_ARGS,
+            user_data_dir=user_data_dir,
+            allowed_domains=["*.facebook.com", "*.messenger.com", "www.google.com"],
+        )
+        await browser.start()
+
+        all_posts = []
+        for query in random.sample(FB_SEARCHES, k=random.randint(2, 4)):
+            agent = Agent(
+                task=f"""You are logged into Facebook. Do the following:
+1. Navigate to facebook.com and make sure you are on the homepage
+2. Use the Facebook search to find: {query}
+3. Scroll through the results
+4. For each relevant post, extract and list:
+   - The post text content
+   - The author name  
+   - The post URL (if visible)
+5. Collect as many posts as you can (aim for 5-10 per search)
+
+When done, return the data as a JSON list with keys: content, author, url, date.""",
+                llm=make_ollama(num_ctx=32000, temperature=0.3),
+                browser=browser,
+                max_actions_per_step=3,
+                use_vision=False,
+                max_failures=6,
+                step_timeout=180,
+            )
+            history = await agent.run()
+            result = history.final_result() or ""
+            posts = extract_agent_posts(result, "")
+            all_posts.extend(posts)
+
+            if random.random() < 0.3:
+                await asyncio.sleep(random.randint(5, 15))
+
         await browser.close()
 
         seen = set()
@@ -529,8 +884,14 @@ async def scrape_facebook(profile_path: str | None = None, force: bool = False)
 
         return {"success": True, "leads": leads[:15], "flagged": False, "flag_reason": None, "error": None}
     except Exception as e:
-        logger.error("Facebook scrape failed: %s", e)
+        logger.error("Agent scrape failed: %s", e)
         return {"success": False, "leads": [], "flagged": False, "flag_reason": None, "error": str(e)}
+    finally:
+        if profile_dir and os.path.isdir(profile_dir):
+            try:
+                shutil.rmtree(profile_dir, ignore_errors=True)
+            except Exception as e:
+                logger.warning("Failed to clean up Chrome profile %s: %s", profile_dir, e)
 
 async def ask_ollama(prompt: str) -> str:
     import httpx
@@ -621,6 +982,28 @@ Return a JSON array like ["yes","no","yes"] matching the order above."""
 async def health():
     return {"status": "ok"}
 
+@app.post("/agent/run")
+async def agent_run(task: str = Body(..., embed=True)):
+    """Run a browser-use Agent with ChatOllama (free/local)."""
+    try:
+        browser = Browser(headless=True, args=CHROME_LAUNCH_ARGS)
+        await browser.start()
+        agent = Agent(
+            task=task,
+            llm=make_ollama(num_ctx=32000, temperature=0.3),
+            browser=browser,
+            use_vision=False,
+            max_actions_per_step=5,
+            max_failures=8,
+            step_timeout=180,
+        )
+        history = await agent.run()
+        await browser.close()
+        return {"success": True, "result": history.final_result()}
+    except Exception as e:
+        logger.error("Agent run failed: %s", e)
+        return {"success": False, "error": str(e)}
+
 @app.post("/scrape/facebook")
 async def scrape_facebook_endpoint(profile_path: str | None = Query(None), force: bool = Query(False)):
     result = await scrape_facebook(profile_path, force)