diff --git a/browser-use-service/main.py b/browser-use-service/main.py index 5b113b3..a6599a3 100644 --- a/browser-use-service/main.py +++ b/browser-use-service/main.py @@ -1,9 +1,21 @@ -import os, json, asyncio, re, shutil, sqlite3, urllib.parse, random, logging +import os, json, asyncio, re, shutil, sqlite3, urllib.parse, random, logging, tempfile from datetime import datetime, timedelta -from fastapi import FastAPI, Query +from fastapi import FastAPI, Query, Body from fastapi.middleware.cors import CORSMiddleware import uvicorn from playwright.async_api import async_playwright +from browser_use import Agent, Browser +from langchain_ollama import ChatOllama + + +def make_ollama(model: str | None = None, **kwargs) -> ChatOllama: + """Create ChatOllama with required attrs for browser-use Agent compatibility.""" + llm = ChatOllama(model=model or CLASSIFY_MODEL, **kwargs) + object.__setattr__(llm, 'provider', 'ollama') + object.__setattr__(llm, 'name', 'ChatOllama') + object.__setattr__(llm, 'model_name', llm.model) + return llm + logging.basicConfig(level=logging.INFO, format='%(asctime)s [%(levelname)s] %(message)s') logger = logging.getLogger(__name__) @@ -20,7 +32,137 @@ PORT = int(os.getenv("PORT", "3008")) OLLAMA_URL = os.getenv("OLLAMA_URL", "http://localhost:11434") CLASSIFY_MODEL = os.getenv("CLASSIFY_MODEL", "dolphin-llama3:8b") + FX_PROFILE = os.getenv('FX_PROFILE', '') +CHROME_PROFILE = os.getenv('CHROME_PROFILE', '') + +CHROME_LAUNCH_ARGS = [ + '--headless=new', + '--disable-blink-features=AutomationControlled', + '--disable-background-networking', + '--no-first-run', + '--disable-sync', + '--disable-field-trial-config', + '--disable-background-timer-throttling', + '--disable-backgrounding-occluded-windows', + '--disable-breakpad', + '--disable-component-update', + '--disable-default-apps', + '--disable-dev-shm-usage', + '--disable-popup-blocking', + '--disable-prompt-on-repost', + '--disable-renderer-backgrounding', + '--metrics-recording-only', + '--no-default-browser-check', + '--disable-extensions-http-throttling', + '--disable-hang-monitor', + '--disable-ipc-flooding-protection', +] + + +def detect_browser_from_profile(profile_path: str) -> str | None: + """Detect browser type from profile path. Returns 'firefox', 'chromium', or None.""" + if not profile_path: + return None + p = profile_path.lower().replace('\\', '/') + if 'firefox' in p or '.default' in p or '.dev-edition' in p: + return 'firefox' + if 'chrome' in p or 'chromium' in p or 'edge' in p: + return 'chromium' + return None + + +def copy_firefox_profile(src_path: str) -> str: + """Copy essential Firefox profile files to a temp dir for Playwright.""" + essential = ['cookies.sqlite', 'webappsstore.sqlite', 'permissions.sqlite'] + dst = tempfile.mkdtemp(prefix='fb_fx_profile_') + for f_name in essential: + s = os.path.join(src_path, f_name) + if os.path.exists(s): + shutil.copy2(s, os.path.join(dst, f_name)) + with open(os.path.join(dst, 'profiles.ini'), 'w') as fh: + fh.write("[Profile0]\nName=default\nIsRelative=0\nPath=.\nDefault=yes\n") + logger.info("Copied Firefox profile to %s", dst) + return dst + + +def copy_chrome_profile(user_data_dir: str, profile_dir: str = 'Default') -> str: + """Copy Chrome profile to temp dir for launch_persistent_context.""" + essential = ['Cookies', 'Login Data', 'Bookmarks', 'Web Data'] + dst = tempfile.mkdtemp(prefix='fb_chrome_udir_') + src_profile = os.path.join(user_data_dir, profile_dir) + os.makedirs(dst, exist_ok=True) + os.makedirs(os.path.join(dst, profile_dir), exist_ok=True) + dst_profile = os.path.join(dst, profile_dir) + for f_name in essential: + s = os.path.join(src_profile, f_name) + if os.path.exists(s): + shutil.copy2(s, os.path.join(dst_profile, f_name)) + for sub in ['Local Storage', 'Session Storage']: + s = os.path.join(src_profile, sub) + if os.path.isdir(s): + shutil.copytree(s, os.path.join(dst_profile, sub), dirs_exist_ok=True) + local_state_src = os.path.join(user_data_dir, 'Local State') + local_state_dst = os.path.join(dst, 'Local State') + if os.path.exists(local_state_src): + shutil.copy2(local_state_src, local_state_dst) + logger.info("Copied Chrome profile from %s to %s", src_profile, dst) + return dst + + +def ensure_ublock_extension() -> str | None: + """Download and extract uBlock Origin extension for ad blocking.""" + ext_dir = os.path.join(tempfile.gettempdir(), 'fb_ublock_origin') + manifest = os.path.join(ext_dir, 'manifest.json') + if os.path.exists(manifest): + return ext_dir + import urllib.request, zipfile + crx_url = ('https://clients2.google.com/service/update2/crx' + '?response=redirect&prodversion=133&acceptformat=crx3' + '&x=id%3Dddkjiahejlhfcafbddmgiahcphecmpfh%26uc') + crx_path = os.path.join(tempfile.gettempdir(), 'ublock_origin.crx') + try: + if not os.path.exists(crx_path): + logger.info("Downloading uBlock Origin extension...") + urllib.request.urlretrieve(crx_url, crx_path) + with zipfile.ZipFile(crx_path, 'r') as z: + z.extractall(ext_dir) + logger.info("uBlock Origin extension ready at %s", ext_dir) + return ext_dir + except Exception as e: + logger.warning("Failed to load uBlock Origin extension: %s", e) + return None + + +def extract_agent_posts(agent_result: str, page_text: str) -> list[dict]: + """Parse posts from agent output or page text fallback.""" + posts = [] + # Try JSON from agent output + json_match = re.search(r'\[.*?\]', agent_result, re.DOTALL) + if json_match: + try: + parsed = json.loads(json_match.group()) + if isinstance(parsed, list): + for item in parsed: + if isinstance(item, dict) and item.get('content'): + posts.append({ + 'content': item.get('content', ''), + 'author': item.get('author', ''), + 'url': item.get('url', ''), + 'date': item.get('date', ''), + }) + except json.JSONDecodeError: + pass + + if not posts: + lines = [l.strip() for l in page_text.split('\n') if l.strip()] + for line in lines: + if len(line) > 30 and not any(skip in line.lower() for skip in + ['facebook', 'messenger', 'notification', 'comment', 'like', 'share']): + posts.append({'content': line[:500], 'author': '', 'url': '', 'date': ''}) + + return posts + BROAD_KEYWORDS = [ "website", "web design", "web develop", "web dev", @@ -91,6 +233,100 @@ REQUEST_PATTERNS = [ r"\bquote\s+(please|for|to)\b", ] +def firefox_init_script() -> str: + return r"""// Firefox Anti-Detection +Object.defineProperty(navigator, 'webdriver', { get: () => undefined }); +Object.defineProperty(navigator, 'languages', { get: () => ['en-US', 'en'] }); +Object.defineProperty(navigator, 'platform', { get: () => 'Win32' }); +Object.defineProperty(navigator, 'hardwareConcurrency', { get: () => 8 }); +Object.defineProperty(navigator, 'maxTouchPoints', { get: () => 0 }); +Object.defineProperty(navigator, 'oscpu', { get: () => 'Windows NT 10.0; Win64; x64' }); +Object.defineProperty(navigator, 'vendor', { get: () => '' }); +Object.defineProperty(navigator, 'vendorSub', { get: () => '' }); +Object.defineProperty(navigator, 'productSub', { get: () => '20100101' }); +Object.defineProperty(navigator, 'product', { get: () => 'Gecko' }); +Object.defineProperty(navigator, 'appCodeName', { get: () => 'Mozilla' }); +Object.defineProperty(navigator, 'appName', { get: () => 'Netscape' }); +Object.defineProperty(navigator, 'appVersion', { get: () => '5.0 (Windows NT 10.0; Win64; x64; rv:130.0) Gecko/20100101 Firefox/130.0' }); +Object.defineProperty(navigator, 'userAgent', { get: () => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:130.0) Gecko/20100101 Firefox/130.0' }); +Object.defineProperty(navigator, 'buildID', { get: () => '20250101000000' }); +if (window.chrome !== undefined) { window.chrome = undefined; } +if (navigator.permissions) { + const origQ = navigator.permissions.query.bind(navigator.permissions); + navigator.permissions.query = (p) => origQ(p).then(r => (['camera','microphone','geolocation','notifications'].includes(p.name) ? Object.assign(r, { state: 'prompt' }) : r)); +} +const _gEC = HTMLCanvasElement.prototype.getContext; +HTMLCanvasElement.prototype.getContext = function(t, ...a) { + const ctx = _gEC.call(this, t, ...a); + if (ctx && (t === 'webgl' || t === 'webgl2')) { + const _gP = ctx.getParameter.bind(ctx); + ctx.getParameter = function(p) { if (p===37445) return 'Intel Inc.'; if (p===37446) return 'Intel Iris OpenGL Engine'; if (p===7936||p===7937||p===35724) return ''; return _gP(p); }; + const _gE = ctx.getExtension.bind(ctx); + ctx.getExtension = function(n) { if (n==='WEBGL_debug_renderer_info') return null; return _gE(n); }; + } + return ctx; +}; +const _tDU = HTMLCanvasElement.prototype.toDataURL; +HTMLCanvasElement.prototype.toDataURL = function(...a) { + const img = _tDU.apply(this, a); + if (img.includes('image/png') && img.length > 5000) { const n = btoa(String.fromCharCode(Math.floor(Math.random()*256))); return img.slice(0,-100)+n+img.slice(-100); } + return img; +}; +const _gFD = AnalyserNode.prototype.getFloatFrequencyData; +if (_gFD) { AnalyserNode.prototype.getFloatFrequencyData = function(a) { _gFD.call(this,a); for(let i=0;i 24 }); +Object.defineProperty(screen, 'pixelDepth', { get: () => 24 }); +Object.defineProperty(navigator, 'plugins', { get: () => [{name:'Widevine Content Decryption Module',filename:'widevinecdm.dll',length:1,description:'Enables Widevine licenses'},{name:'OpenH264 Video Codec',filename:'openh264.dll',length:1,description:'H.264 video codec'}] }); +Object.defineProperty(navigator, 'mimeTypes', { get: () => [{type:'application/pdf',suffixes:'pdf',description:'Portable Document Format',enabledPlugin:navigator.plugins[0]}] });""" + + +def chromium_init_script() -> str: + return r"""// Enhanced Chromium Anti-Detection +Object.defineProperty(navigator, 'webdriver', { get: () => false }); +Object.defineProperty(navigator, 'languages', { get: () => ['en-US', 'en'] }); +Object.defineProperty(navigator, 'platform', { get: () => 'Win32' }); +Object.defineProperty(navigator, 'hardwareConcurrency', { get: () => 8 }); +Object.defineProperty(navigator, 'deviceMemory', { get: () => 8 }); +Object.defineProperty(navigator, 'maxTouchPoints', { get: () => 0 }); +Object.defineProperty(navigator, 'oscpu', { get: () => 'Windows NT 10.0; Win64; x64' }); +Object.defineProperty(navigator, 'vendor', { get: () => 'Google Inc.' }); +Object.defineProperty(navigator, 'vendorSub', { get: () => '' }); +Object.defineProperty(navigator, 'productSub', { get: () => '20030107' }); +Object.defineProperty(navigator, 'product', { get: () => 'Gecko' }); +Object.defineProperty(navigator, 'appCodeName', { get: () => 'Mozilla' }); +Object.defineProperty(navigator, 'appName', { get: () => 'Netscape' }); +Object.defineProperty(navigator, 'appVersion', { get: () => '5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36' }); +Object.defineProperty(navigator, 'userAgent', { get: () => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36' }); +window.chrome = { runtime: { onConnect: { addListener: () => {} }, onMessage: { addListener: () => {} }, sendMessage: () => {} }, app: { isInstalled: false, InstallState: { DISABLED: 'disabled', INSTALLED: 'installed', NOT_INSTALLED: 'not_installed' }, getDetails: () => null, getIsInstalled: () => false }, csi: () => {}, loadTimes: () => {} }; +if (navigator.permissions) { + const origQ = navigator.permissions.query.bind(navigator.permissions); + navigator.permissions.query = (p) => origQ(p).then(r => (['camera','microphone','geolocation','notifications'].includes(p.name) ? Object.assign(r, { state: 'prompt' }) : r)); +} +const _gEC = HTMLCanvasElement.prototype.getContext; +HTMLCanvasElement.prototype.getContext = function(t, ...a) { + const ctx = _gEC.call(this, t, ...a); + if (ctx && (t === 'webgl' || t === 'webgl2')) { + const _gP = ctx.getParameter.bind(ctx); + ctx.getParameter = function(p) { if (p===37445) return 'Intel Inc.'; if (p===37446) return 'Intel Iris OpenGL Engine'; if (p===7936||p===7937||p===35724) return ''; return _gP(p); }; + const _gE = ctx.getExtension.bind(ctx); + ctx.getExtension = function(n) { if (n==='WEBGL_debug_renderer_info') return null; return _gE(n); }; + } + return ctx; +}; +const _tDU = HTMLCanvasElement.prototype.toDataURL; +HTMLCanvasElement.prototype.toDataURL = function(...a) { + const img = _tDU.apply(this, a); + if (img.includes('image/png') && img.length > 5000) { const n = btoa(String.fromCharCode(Math.floor(Math.random()*256))); return img.slice(0,-100)+n+img.slice(-100); } + return img; +}; +const _gFD = AnalyserNode.prototype.getFloatFrequencyData; +if (_gFD) { AnalyserNode.prototype.getFloatFrequencyData = function(a) { _gFD.call(this,a); for(let i=0;i 24 }); +Object.defineProperty(screen, 'pixelDepth', { get: () => 24 }); +Object.defineProperty(navigator, 'plugins', { get: () => [1,2,3,4,5].map(i => ({name:['Chrome PDF Plugin','Chrome PDF Viewer','Native Client','Widevine Content Decryption Module','Chromium PDF Viewer'][i-1],filename:'internal-pdf-viewer',length:1,description:['Portable Document Format','','Native Client Executable','Enables Widevine licenses',''][i-1]})) }); +Object.defineProperty(navigator, 'mimeTypes', { get: () => [{type:'application/pdf',suffixes:'pdf',description:'Portable Document Format',enabledPlugin:navigator.plugins[0]},{type:'text/pdf',suffixes:'pdf',description:'Portable Document Format',enabledPlugin:navigator.plugins[0]}] });""" + + def kw_match(text: str) -> bool: t = text.lower() return any(kw in t for kw in BROAD_KEYWORDS) @@ -306,6 +542,13 @@ async def human_scroll(page, steps: int = None, total_delay: float = None): for i in range(steps): scroll_dist = random.randint(200, 600) await page.evaluate(f"window.scrollBy(0, {scroll_dist})") + try: + vp = await page.evaluate('({w: window.innerWidth, h: window.innerHeight})') + x = random.randint(100, vp['w'] - 100) + y = random.randint(100, vp['h'] - 100) + await page.mouse.move(x, y, steps=random.randint(15, 35)) + except Exception: + pass await page.wait_for_timeout(int(step_delay * 1000)) if random.random() < 0.3: await page.evaluate(f"window.scrollBy(0, -{random.randint(100, 300)})") @@ -421,6 +664,25 @@ async def search_facebook(page, context, query: str): return page, [] return page, posts +DETECTION_SIGNALS = [ + '/checkpoint/', '/login.php?', 'action=security_check', + 'unusual activity', 'suspicious login', 'suspicious activity', + 'temporarily blocked', 'temporarily restricted', + 'confirm your identity', 'enter the code we sent', + 'we noticed unusual', 'help us confirm', + 'security check', 'challenge', 'please verify', +] + + +def check_detection_signals(page_url: str, page_text: str = '') -> str | None: + """Check page for Facebook detection signals. Returns signal text or None.""" + combined = (page_url + ' ' + page_text).lower() + for signal in DETECTION_SIGNALS: + if signal in combined: + return signal + return None + + def cleanup_chrome(): import subprocess, signal try: @@ -429,91 +691,110 @@ def cleanup_chrome(): pass async def scrape_facebook(profile_path: str | None = None, force: bool = False) -> dict: - cleanup_chrome() - fb_cookies = await get_fb_cookies(profile_path) - if not fb_cookies: - logger.warning("No Facebook cookies available") - return {"success": False, "leads": [], "flagged": False, "flag_reason": None, "error": "No cookies available"} + """Dispatcher — Firefox primary, browser-use Agent fallback.""" + effective_path = profile_path or FX_PROFILE + browser_type = detect_browser_from_profile(effective_path) + + if not browser_type and CHROME_PROFILE: + browser_type = detect_browser_from_profile(CHROME_PROFILE) + effective_path = CHROME_PROFILE + + logger.info("Detected browser: %s (profile: %s)", browser_type or "none", effective_path) + + # Firefox primary (raw Playwright, stealth) + if browser_type == "firefox": + result = await _scrape_with_firefox(effective_path, force) + if result.get("success") or not result.get("flagged"): + return result + logger.warning("Firefox path failed (%s), falling back to Agent", result.get("flag_reason", "unknown")) + return await _scrape_with_agent(force) + + # CHROME_PROFILE set or no profile → Agent + if browser_type == "chromium": + return await _scrape_with_agent(force) + + # No profile at all → try Agent (fresh Chromium) + return await _scrape_with_agent(force) + + +async def _scrape_with_firefox(profile_path: str, force: bool) -> dict: + """Scrape Facebook using Firefox + persistent real profile (no cookie injection).""" + if not profile_path: + return {"success": False, "leads": [], "flagged": False, "flag_reason": None, "error": "No profile path"} + + profile_dir = None try: + profile_dir = copy_firefox_profile(profile_path) async with async_playwright() as pw: - browser = await pw.chromium.launch( + context = await pw.firefox.launch_persistent_context( + user_data_dir=profile_dir, headless=True, - args=['--disable-blink-features=AutomationControlled'] - ) - viewport = random.choice(VIEWPORTS) - context = await browser.new_context( - user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36', - viewport=viewport, + firefox_user_prefs={ + "dom.webdriver.enabled": False, + "dom.webdriver.timeout": 0, + "useAutomationExtension": False, + "privacy.trackingprotection.enabled": False, + "privacy.trackingprotection.fingerprinting.enabled": False, + "privacy.trackingprotection.cryptomining.enabled": False, + }, ) + pages = context.pages + page = pages[0] if pages else await context.new_page() + await context.add_init_script(firefox_init_script()) - await context.add_init_script(""" - Object.defineProperty(navigator, 'webdriver', { get: () => false }); - Object.defineProperty(navigator, 'languages', { get: () => ['en-US', 'en'] }); - Object.defineProperty(navigator, 'platform', { get: () => 'Win32' }); - window.chrome = { runtime: {} }; - Object.defineProperty(navigator, 'hardwareConcurrency', { get: () => 8 }); - Object.defineProperty(navigator, 'deviceMemory', { get: () => 8 }); - """) - for c in fb_cookies: - try: - await context.add_cookies([c]) - except Exception as e: - logger.warning("Failed to inject cookie %s: %s", c.get('name'), e) - - page = await context.new_page() - # Navigate through google first for a legitimate Referer header try: await page.goto('https://www.google.com/', wait_until='domcontentloaded', timeout=15000) await page.wait_for_timeout(random.randint(1000, 3000)) except Exception: - logger.warning("Google navigation failed (IP may be blocked), trying Facebook directly") + logger.warning("Google navigation failed, trying Facebook directly") + await page.goto('https://www.facebook.com/', wait_until='domcontentloaded', timeout=30000) await page.wait_for_timeout(random.randint(3000, 8000)) url = page.url - if '/login' in url.lower(): - logger.warning("Facebook login page detected — account flagged") - await browser.close() - return {"success": False, "leads": [], "flagged": True, "flag_reason": "login_page", "error": "Facebook login page detected"} + page_text = await page.evaluate('document.body.innerText') if '/login' in url.lower() else '' + det = check_detection_signals(url, page_text) + if det or '/login' in url.lower(): + logger.warning("Facebook login page detected — flag: %s", det or "login_page") + await context.close() + return {"success": False, "leads": [], "flagged": True, "flag_reason": det or "login_page", "error": "Facebook login page detected"} - # Browse feed like a human before searching await human_scroll(page, steps=random.randint(2, 4), total_delay=random.uniform(8, 20)) if random.random() < 0.25: await page.evaluate("window.scrollTo(0, 0)") await page.wait_for_timeout(random.randint(2000, 5000)) await human_scroll(page, steps=random.randint(1, 2)) - # False start: 30% chance to idle and leave without scraping (skipped when force=true) if not force and random.random() < 0.3: await page.wait_for_timeout(random.randint(8000, 20000)) - await browser.close() + await context.close() return {"success": True, "leads": [], "flagged": False, "flag_reason": None, "error": None} all_posts = [] - searches = random.sample(FB_SEARCHES, k=random.randint(5, 8)) + searches = random.sample(FB_SEARCHES, k=random.randint(2, 4)) for i, query in enumerate(searches): page, posts = await search_facebook(page, context, query) all_posts.extend(posts) if not posts: continue - # Between searches: random break with possible small scroll if random.random() < 0.4: await page.evaluate(f"window.scrollBy(0, {random.randint(-300, 300)})") delay = random.uniform(8, 25) await page.wait_for_timeout(int(delay * 1000)) - # Tab switch: 15% chance after 2nd-3rd search - if i == random.randint(1, 2) and random.random() < 0.15: + if i == random.randint(0, 1) and random.random() < 0.15: new_page = await context.new_page() - await new_page.goto('https://www.messenger.com/', wait_until='domcontentloaded', timeout=15000) - await new_page.wait_for_timeout(random.randint(3000, 8000)) + try: + await new_page.goto('https://www.facebook.com/groups/', wait_until='domcontentloaded', timeout=15000) + await new_page.wait_for_timeout(random.randint(3000, 8000)) + except Exception: + pass await new_page.close() page = await _ensure_page(page, context) - # Idle before closing — human would pause if random.random() < 0.5: await page.wait_for_timeout(random.randint(3000, 10000)) - await browser.close() + await context.close() seen = set() deduped = [] @@ -528,9 +809,89 @@ async def scrape_facebook(profile_path: str | None = None, force: bool = False) leads = await classify_leads(leads) return {"success": True, "leads": leads[:15], "flagged": False, "flag_reason": None, "error": None} + except Exception as e: - logger.error("Facebook scrape failed: %s", e) + logger.error("Firefox scrape failed: %s", e) return {"success": False, "leads": [], "flagged": False, "flag_reason": None, "error": str(e)} + finally: + if profile_dir and os.path.exists(profile_dir): + try: + shutil.rmtree(profile_dir, ignore_errors=True) + except Exception: + pass + + +async def _scrape_with_agent(force: bool = False) -> dict: + """Fallback scraper — browser-use Agent + ChatOllama (free/local, Chromium).""" + cleanup_chrome() + profile_dir = None + try: + user_data_dir = None + if CHROME_PROFILE: + profile_dir = copy_chrome_profile(CHROME_PROFILE) + user_data_dir = profile_dir + + browser = Browser( + headless=True, + args=CHROME_LAUNCH_ARGS, + user_data_dir=user_data_dir, + allowed_domains=["*.facebook.com", "*.messenger.com", "www.google.com"], + ) + await browser.start() + + all_posts = [] + for query in random.sample(FB_SEARCHES, k=random.randint(2, 4)): + agent = Agent( + task=f"""You are logged into Facebook. Do the following: +1. Navigate to facebook.com and make sure you are on the homepage +2. Use the Facebook search to find: {query} +3. Scroll through the results +4. For each relevant post, extract and list: + - The post text content + - The author name + - The post URL (if visible) +5. Collect as many posts as you can (aim for 5-10 per search) + +When done, return the data as a JSON list with keys: content, author, url, date.""", + llm=make_ollama(num_ctx=32000, temperature=0.3), + browser=browser, + max_actions_per_step=3, + use_vision=False, + max_failures=6, + step_timeout=180, + ) + history = await agent.run() + result = history.final_result() or "" + posts = extract_agent_posts(result, "") + all_posts.extend(posts) + + if random.random() < 0.3: + await asyncio.sleep(random.randint(5, 15)) + + await browser.close() + + seen = set() + deduped = [] + for p in all_posts: + key = p.get('content', '')[:100] + if key not in seen: + seen.add(key) + deduped.append(p) + + leads = deduped[:20] + if leads: + leads = await classify_leads(leads) + + return {"success": True, "leads": leads[:15], "flagged": False, "flag_reason": None, "error": None} + except Exception as e: + logger.error("Agent scrape failed: %s", e) + return {"success": False, "leads": [], "flagged": False, "flag_reason": None, "error": str(e)} + finally: + if profile_dir and os.path.isdir(profile_dir): + try: + shutil.rmtree(profile_dir, ignore_errors=True) + except Exception as e: + logger.warning("Failed to clean up Chrome profile %s: %s", profile_dir, e) async def ask_ollama(prompt: str) -> str: import httpx @@ -621,6 +982,28 @@ Return a JSON array like ["yes","no","yes"] matching the order above.""" async def health(): return {"status": "ok"} +@app.post("/agent/run") +async def agent_run(task: str = Body(..., embed=True)): + """Run a browser-use Agent with ChatOllama (free/local).""" + try: + browser = Browser(headless=True, args=CHROME_LAUNCH_ARGS) + await browser.start() + agent = Agent( + task=task, + llm=make_ollama(num_ctx=32000, temperature=0.3), + browser=browser, + use_vision=False, + max_actions_per_step=5, + max_failures=8, + step_timeout=180, + ) + history = await agent.run() + await browser.close() + return {"success": True, "result": history.final_result()} + except Exception as e: + logger.error("Agent run failed: %s", e) + return {"success": False, "error": str(e)} + @app.post("/scrape/facebook") async def scrape_facebook_endpoint(profile_path: str | None = Query(None), force: bool = Query(False)): result = await scrape_facebook(profile_path, force)