Update on auto setup

browser-use-service/main.py

@@ -1,3 +1,4 @@
+# ── Imports ──────────────────────────────────────────────────────────
 import os, json, asyncio, re, shutil, sqlite3, urllib.parse, random, logging, tempfile
 from datetime import datetime, timedelta
 from fastapi import FastAPI, Query, Body
@@ -8,8 +9,12 @@ from browser_use import Agent, Browser
 from langchain_ollama import ChatOllama
 
 
+# ── Helpers ──────────────────────────────────────────────────────────
 def make_ollama(model: str | None = None, **kwargs) -> ChatOllama:
-    """Create ChatOllama with required attrs for browser-use Agent compatibility."""
+    """
+    Create ChatOllama with required attrs for browser-use Agent compatibility.
+    The browser-use Agent expects llm.name, llm.model_name, and llm.provider attrs.
+    """
     llm = ChatOllama(model=model or CLASSIFY_MODEL, **kwargs)
     object.__setattr__(llm, 'provider', 'ollama')
     object.__setattr__(llm, 'name', 'ChatOllama')
@@ -17,6 +22,7 @@ def make_ollama(model: str | None = None, **kwargs) -> ChatOllama:
     return llm
 
 
+# ── Logging & App Setup ──────────────────────────────────────────────
 logging.basicConfig(level=logging.INFO, format='%(asctime)s [%(levelname)s] %(message)s')
 logger = logging.getLogger(__name__)
 
@@ -29,13 +35,24 @@ app.add_middleware(
 )
 PORT = int(os.getenv("PORT", "3008"))
 
+# ── AI / Ollama Config ───────────────────────────────────────────────
 OLLAMA_URL = os.getenv("OLLAMA_URL", "http://localhost:11434")
 CLASSIFY_MODEL = os.getenv("CLASSIFY_MODEL", "dolphin-llama3:8b")
 
 
+# ── Env Vars (set via .env.local) ──────────────────────────────────────
+# These store detected browser profile paths from the setup wizard.
 FX_PROFILE = os.getenv('FX_PROFILE', '')
 CHROME_PROFILE = os.getenv('CHROME_PROFILE', '')
+OPERA_PROFILE = os.getenv('OPERA_PROFILE', '')
+EDGE_PROFILE = os.getenv('EDGE_PROFILE', '')
+# SELECTED_BROWSER determines which browser the scrape dispatcher tries first.
+# Values: "firefox", "chrome", "opera", "edge", or "" (auto-detect all).
+SELECTED_BROWSER = os.getenv('SELECTED_BROWSER', '')
 
+# ── Chromium Launch Args ──────────────────────────────────────────────
+# These flags reduce detectable automation signals in Chrome/Edge/Opera.
+# They disable background services, sync, updates, throttling, popups, etc.
 CHROME_LAUNCH_ARGS = [
     '--headless=new',
     '--disable-blink-features=AutomationControlled',
@@ -60,20 +77,36 @@ CHROME_LAUNCH_ARGS = [
 ]
 
 
+# ── Profile Detection ────────────────────────────────────────────────
+# All find_*_profile() functions return the parent directory (User Data or equivalent),
+# NOT the "Default" subdirectory. This is important because copy_chrome_profile()
+# appends "Default" internally, so returning the full Default path would cause
+# duplicated path segments (e.g. ".../User Data/Default/Default").
+
 def detect_browser_from_profile(profile_path: str) -> str | None:
-    """Detect browser type from profile path. Returns 'firefox', 'chromium', or None."""
+    """
+    Infer browser type from the profile path string.
+    Checks for keywords like "firefox", "opera", "edge", "chrome" in the path.
+    Returns "firefox", "chrome", "opera", "edge", or None.
+    """
     if not profile_path:
         return None
     p = profile_path.lower().replace('\\', '/')
     if 'firefox' in p or '.default' in p or '.dev-edition' in p:
         return 'firefox'
-    if 'chrome' in p or 'chromium' in p or 'edge' in p:
-        return 'chromium'
+    if 'opera' in p:
+        return 'opera'
+    if 'edge' in p:
+        return 'edge'
+    if 'chrome' in p or 'chromium' in p:
+        return 'chrome'
     return None
 
 
 def find_firefox_profile() -> str | None:
-    """Auto-detect Firefox profile directory cross-platform."""
+    """Auto-detect Firefox profile directory cross-platform.
+    Scans the Profiles dir and returns the first .default or .dev-edition folder found,
+    preferring "default-release" profiles."""
     import sys
     home = os.path.expanduser("~")
     candidates = []
@@ -102,7 +135,8 @@ def find_firefox_profile() -> str | None:
 
 
 def find_chrome_profile() -> str | None:
-    """Auto-detect Chrome/Chromium profile directory cross-platform."""
+    """Auto-detect Chrome profile (User Data dir) cross-platform.
+    Returns the User Data parent dir (NOT the Default subdirectory)."""
     import sys
     home = os.path.expanduser("~")
 
@@ -113,15 +147,60 @@ def find_chrome_profile() -> str | None:
     else:
         base = os.path.join(home, ".config", "google-chrome")
 
-    default_profile = os.path.join(base, "Default")
-    if os.path.isdir(default_profile):
-        logger.info("Auto-detected Chrome profile: %s", default_profile)
-        return default_profile
+    if os.path.isdir(os.path.join(base, "Default")):
+        logger.info("Auto-detected Chrome profile: %s", base)
+        return base
     return None
 
 
+def find_opera_profile() -> str | None:
+    """Auto-detect Opera profile directory cross-platform.
+    Opera stores its profile at the "Opera Stable" level (no "User Data/Default"
+    nesting like Chrome/Edge), so this returns the profile root directly."""
+    import sys
+    home = os.path.expanduser("~")
+
+    if sys.platform == "win32":
+        base = os.path.join(os.environ.get("APPDATA", ""), "Opera Software", "Opera Stable")
+    elif sys.platform == "darwin":
+        base = os.path.join(home, "Library", "Application Support", "com.operasoftware.Opera")
+    else:
+        base = os.path.join(home, ".config", "opera")
+
+    if os.path.isdir(base) and os.listdir(base):
+        logger.info("Auto-detected Opera profile: %s", base)
+        return base
+    return None
+
+
+def find_edge_profile() -> str | None:
+    """Auto-detect Edge profile (User Data dir) cross-platform.
+    Returns the User Data parent dir (NOT the Default subdirectory)."""
+    import sys
+    home = os.path.expanduser("~")
+
+    if sys.platform == "win32":
+        base = os.path.join(os.environ.get("LOCALAPPDATA", ""), "Microsoft", "Edge", "User Data")
+    elif sys.platform == "darwin":
+        base = os.path.join(home, "Library", "Application Support", "Microsoft Edge")
+    else:
+        base = os.path.join(home, ".config", "microsoft-edge")
+
+    if os.path.isdir(os.path.join(base, "Default")):
+        logger.info("Auto-detected Edge profile: %s", base)
+        return base
+    return None
+
+
+# ── Profile Copying ─────────────────────────────────────────────────
+# Both functions copy essential cookies/login/storage files from the user's
+# real browser profile to a temp directory. Playwright then uses these temp
+# profiles via launch_persistent_context(), preserving Facebook login state
+# without modifying the user's actual profile.
+
 def copy_firefox_profile(src_path: str) -> str:
-    """Copy essential Firefox profile files to a temp dir for Playwright."""
+    """Copy essential Firefox profile files (cookies, localStorage, permissions) to a temp dir for Playwright.
+    Also creates a minimal profiles.ini so Firefox treats it as a valid profile."""
     essential = ['cookies.sqlite', 'webappsstore.sqlite', 'permissions.sqlite']
     dst = tempfile.mkdtemp(prefix='fb_fx_profile_')
     for f_name in essential:
@@ -135,7 +214,12 @@ def copy_firefox_profile(src_path: str) -> str:
 
 
 def copy_chrome_profile(user_data_dir: str, profile_dir: str = 'Default') -> str:
-    """Copy Chrome profile to temp dir for launch_persistent_context."""
+    """Copy Chromium-based profile (Chrome/Edge/Opera) to a temp dir.
+    Copies Cookies, Login Data, Bookmarks, Web Data, Local Storage, Session Storage, and Local State.
+    Args:
+        user_data_dir: Parent directory (e.g. ".../Chrome/User Data")
+        profile_dir:  Subdirectory name, typically "Default" (default)
+    """
     essential = ['Cookies', 'Login Data', 'Bookmarks', 'Web Data']
     dst = tempfile.mkdtemp(prefix='fb_chrome_udir_')
     src_profile = os.path.join(user_data_dir, profile_dir)
@@ -159,7 +243,8 @@ def copy_chrome_profile(user_data_dir: str, profile_dir: str = 'Default') -> str
 
 
 def ensure_ublock_extension() -> str | None:
-    """Download and extract uBlock Origin extension for ad blocking."""
+    """Download and extract uBlock Origin extension for ad blocking via Chromium.
+    Cached after first download to avoid re-downloading on every scrape."""
     ext_dir = os.path.join(tempfile.gettempdir(), 'fb_ublock_origin')
     manifest = os.path.join(ext_dir, 'manifest.json')
     if os.path.exists(manifest):
@@ -182,10 +267,15 @@ def ensure_ublock_extension() -> str | None:
         return None
 
 
+# ── Agent Output Parsing ──────────────────────────────────────────────
+# When the browser-use Agent fallback is used, it returns natural language
+# output containing post listings. This function tries to extract structured
+# data from that output, first attempting JSON parsing, then falling back to
+# a line-by-line heuristic (numbered or bulleted items).
+
 def extract_agent_posts(agent_result: str, page_text: str) -> list[dict]:
     """Parse posts from agent output or page text fallback."""
     posts = []
-    # Try JSON from agent output
     json_match = re.search(r'\[.*?\]', agent_result, re.DOTALL)
     if json_match:
         try:
@@ -221,6 +311,12 @@ def extract_agent_posts(agent_result: str, page_text: str) -> list[dict]:
     return posts
 
 
+# ── Keyword Filters for Lead Detection ──────────────────────────────
+# BROAD_KEYWORDS — catch any post that might relate to web development needs.
+# OFFER_PATTERNS — regex patterns matching service offers (which we ignore).
+# REQUEST_PATTERNS — regex patterns matching people asking for help/services.
+# Together these form a two-pass filter + AI classification pipeline.
+
 BROAD_KEYWORDS = [
     "website", "web design", "web develop", "web dev",
     "build my website", "build a website", "create a website",
@@ -290,6 +386,13 @@ REQUEST_PATTERNS = [
     r"\bquote\s+(please|for|to)\b",
 ]
 
+# ── Anti-Detection Scripts ────────────────────────────────────────────
+# These JS scripts are injected into every page via add_init_script().
+# They override navigator properties (webdriver, userAgent, plugins, etc.)
+# and WebGL/canvas/audio fingerprints to make Playwright harder to detect.
+# Firefox and Chromium variants differ slightly (e.g. Chrome needs the
+# "window.chrome" object; Firefox must NOT have it).
+
 def firefox_init_script() -> str:
     return r"""// Firefox Anti-Detection
 Object.defineProperty(navigator, 'webdriver', { get: () => undefined });
@@ -384,18 +487,25 @@ Object.defineProperty(navigator, 'plugins', { get: () => [1,2,3,4,5].map(i => ({
 Object.defineProperty(navigator, 'mimeTypes', { get: () => [{type:'application/pdf',suffixes:'pdf',description:'Portable Document Format',enabledPlugin:navigator.plugins[0]},{type:'text/pdf',suffixes:'pdf',description:'Portable Document Format',enabledPlugin:navigator.plugins[0]}] });"""
 
 
+# ── Text Classification Utilities ────────────────────────────────────
 def kw_match(text: str) -> bool:
     t = text.lower()
     return any(kw in t for kw in BROAD_KEYWORDS)
 
 def is_request(text: str) -> bool:
+    """Detect if text contains a request for services (e.g. "I need", "looking for")."""
     t = text.lower()
     return any(re.search(p, t) for p in REQUEST_PATTERNS)
 
 def is_offer(text: str) -> bool:
+    """Detect if text is an offer/advertisement (we skip these, not leads)."""
     t = text.lower()
     return any(re.search(p, t) for p in OFFER_PATTERNS)
 
+# ── Facebook Search Queries ───────────────────────────────────────────
+# These are the search terms the scraper will use on Facebook.
+# They target people looking for web development / website services.
+# Each run picks 2-4 random queries to vary behavior and reduce detection.
 FB_SEARCHES = [
     "looking for web developer",
     "need a website designed",
@@ -428,6 +538,12 @@ VIEWPORTS = [
     {'width': 1920, 'height': 1080},
 ]
 
+# ── Cookie Extraction ────────────────────────────────────────────────
+# Reads Facebook cookies from the browser profile's SQLite database.
+# Firefox stores cookies in cookies.sqlite; Chromium stores them in Cookies
+# (a SQLite DB as well). This function handles Firefox's format.
+# The cookies are copied to a temp file first to avoid locking issues.
+
 async def get_fb_cookies(profile_path: str | None = None):
     cookie_db_path = profile_path or FX_PROFILE
     if not cookie_db_path:
@@ -472,6 +588,9 @@ async def get_fb_cookies(profile_path: str | None = None):
             pass
         return []
 
+# ── Date Parsing ─────────────────────────────────────────────────────
+# Facebook displays dates in relative format ("2h ago", "Yesterday", "Monday")
+# or various absolute formats. These functions normalize them to YYYY-MM-DD.
 WEEKDAY_ORDER = ['monday', 'tuesday', 'wednesday', 'thursday', 'friday', 'saturday', 'sunday']
 
 def _parse_fb_date(block: list[str]) -> str:
@@ -525,6 +644,12 @@ def _clean_fb_text(text: str) -> str:
         cleaned_lines.append(stripped)
     return '\n'.join(cleaned_lines)
 
+# ── Post Extraction ──────────────────────────────────────────────────
+# Two strategies for extracting posts from page content:
+#   1. _extract_posts_from_elements — uses structured DOM data from _get_article_elements()
+#   2. _extract_posts_from_text     — fallback that parses raw page text line by line
+# Both apply dedup (seen_texts), offer filtering, and request scoring.
+
 def _extract_posts_from_elements(elements: list[dict], base_url: str) -> list[dict]:
     posts = []
     seen_texts = set()
@@ -617,6 +742,11 @@ def _extract_posts_from_text(raw: str, url: str) -> list[dict]:
             cur.pop(0)
     return posts
 
+# ── Human-like Behavior Simulation ──────────────────────────────────
+# These functions add random delays, mouse movements, and scroll patterns
+# to make automated browsing look more like a real human user.
+# This is critical for avoiding Facebook's bot detection.
+
 async def human_scroll(page, steps: int = None, total_delay: float = None):
     steps = steps or random.randint(2, 5)
     total_delay = total_delay or random.uniform(6, 18)
@@ -652,6 +782,12 @@ async def random_idle(page):
         except Exception:
             pass
 
+# ── Facebook DOM Parsing ─────────────────────────────────────────────
+# This JS function runs inside the page context to extract structured post
+# data (text, author, URL, date) from Facebook's complex DOM structure.
+# It tries multiple selectors (article, feed, etc.) and uses the first
+# one that returns results, since Facebook's DOM changes frequently.
+
 async def _get_article_elements(page) -> list[dict]:
     return await page.evaluate('''() => {
         const results = [];
@@ -713,6 +849,8 @@ async def _get_article_elements(page) -> list[dict]:
     }''')
 
 async def _ensure_page(page, context):
+    """Check if the current page is still alive. If closed (e.g. by a popup),
+    create a fresh page and navigate to Facebook."""
     try:
         await page.evaluate('1')
         return page
@@ -728,6 +866,12 @@ async def _ensure_page(page, context):
         await page.wait_for_timeout(random.randint(3000, 8000))
         return page
 
+
+# ── Facebook Search & Scrape ─────────────────────────────────────────
+# Performs a Facebook search for a given query, scrolls through results,
+# extracts posts, and returns them. Includes randomization of scroll behavior,
+# mouse movements, and idle actions to mimic human browsing patterns.
+
 async def search_facebook(page, context, query: str):
     page = await _ensure_page(page, context)
     url = f'https://www.facebook.com/search/posts/?q={urllib.parse.quote(query)}'
@@ -764,6 +908,10 @@ async def search_facebook(page, context, query: str):
         return page, []
     return page, posts
 
+# ── Detection Signals ────────────────────────────────────────────────
+# Keywords/phrases in page URL or body text that indicate Facebook's
+# security/bot detection systems have flagged the session.
+
 DETECTION_SIGNALS = [
     '/checkpoint/', '/login.php?', 'action=security_check',
     'unusual activity', 'suspicious login', 'suspicious activity',
@@ -784,54 +932,90 @@ def check_detection_signals(page_url: str, page_text: str = '') -> str | None:
 
 
 def cleanup_chrome():
+    """Kill orphaned Chrome headless shell processes. These sometimes linger
+    after failed scrapes and interfere with subsequent runs."""
     import subprocess, signal
     try:
         subprocess.run(["taskkill", "/F", "/IM", "chrome-headless-shell.exe"], capture_output=True, timeout=5)
     except Exception:
         pass
 
+
+# ── Main Scrape Dispatcher ────────────────────────────────────────────
+# scrape_facebook() is the main entry point. It:
+#   1. Resolves the browser profile path (from SELECTED_BROWSER env var or auto-detect)
+#   2. Dispatches to the correct browser-specific scraper
+#   3. If the browser scrape is flagged by Facebook, falls through to the Agent fallback
+#
+# Priority order for browser selection:
+#   - SELECTED_BROWSER env var (set by setup wizard) → that browser's scraper
+#   - Auto-detect: Firefox → Opera → Chrome → Edge (first found wins)
+#   - If no profile found → Agent (browser-use + ChatOllama) fallback
+
 async def scrape_facebook(profile_path: str | None = None, force: bool = False) -> dict:
-    """Dispatcher — Firefox primary, browser-use Agent fallback."""
-    effective_path = profile_path or FX_PROFILE
+    """Dispatcher — respect SELECTED_BROWSER, fall through on flag, then Agent."""
+    effective_path = profile_path or ""
+    browser_type = ""
 
-    # Auto-detect Firefox profile if none provided
-    if not effective_path:
-        detected = find_firefox_profile()
-        if detected:
-            effective_path = detected
-            os.environ["FX_PROFILE"] = detected
+    # Resolve profile and browser type
+    if effective_path:
+        browser_type = detect_browser_from_profile(effective_path) or ""
+    elif SELECTED_BROWSER == "firefox":
+        effective_path = FX_PROFILE or find_firefox_profile() or ""
+        browser_type = "firefox"
+    elif SELECTED_BROWSER == "opera":
+        effective_path = OPERA_PROFILE or find_opera_profile() or ""
+        browser_type = "opera"
+    elif SELECTED_BROWSER == "edge":
+        effective_path = EDGE_PROFILE or find_edge_profile() or ""
+        browser_type = "edge"
+    elif SELECTED_BROWSER == "chrome":
+        effective_path = CHROME_PROFILE or find_chrome_profile() or ""
+        browser_type = "chrome"
+    else:
+        # Auto-detect — try all in priority order
+        for name, env_var, find_fn in [
+            ("firefox", "FX_PROFILE", find_firefox_profile),
+            ("opera", "OPERA_PROFILE", find_opera_profile),
+            ("chrome", "CHROME_PROFILE", find_chrome_profile),
+            ("edge", "EDGE_PROFILE", find_edge_profile),
+        ]:
+            p = os.getenv(env_var, "") or find_fn() or ""
+            if p:
+                effective_path = p
+                browser_type = name
+                break
 
-    browser_type = detect_browser_from_profile(effective_path)
+    if not browser_type or not effective_path:
+        logger.info("No profile found — falling back to Agent")
+        return await _scrape_with_agent(force)
 
-    if not browser_type and CHROME_PROFILE:
-        browser_type = detect_browser_from_profile(CHROME_PROFILE)
-        effective_path = CHROME_PROFILE
+    logger.info("Selected browser: %s (profile: %s)", browser_type, effective_path)
 
-    # Auto-detect Chrome profile if still nothing
-    if not browser_type:
-        detected_chrome = find_chrome_profile()
-        if detected_chrome:
-            effective_path = detected_chrome
-            browser_type = 'chromium'
-
-    logger.info("Detected browser: %s (profile: %s)", browser_type or "none", effective_path)
-
-    # Firefox primary (raw Playwright, stealth)
+    # Firefox path
     if browser_type == "firefox":
         result = await _scrape_with_firefox(effective_path, force)
         if result.get("success") or not result.get("flagged"):
             return result
-        logger.warning("Firefox path failed (%s), falling back to Agent", result.get("flag_reason", "unknown"))
+        logger.warning("Firefox flagged (%s), trying Agent", result.get("flag_reason", "unknown"))
         return await _scrape_with_agent(force)
 
-    # CHROME_PROFILE set or no profile → Agent
-    if browser_type == "chromium":
-        return await _scrape_with_agent(force)
-
-    # No profile at all → try Agent (fresh Chromium)
+    # Chromium-based (chrome / opera / edge)
+    result = await _scrape_with_chromium(effective_path, browser_type, force)
+    if result.get("success") or not result.get("flagged"):
+        return result
+    logger.warning("%s flagged (%s), trying Agent", browser_type, result.get("flag_reason", "unknown"))
     return await _scrape_with_agent(force)
 
 
+# ── Firefox Scraper ──────────────────────────────────────────────────
+# Launches a headless Firefox with the user's real profile cookies (copied
+# to a temp dir). Uses firefox_user_prefs to disable automation flags and
+# tracking protection. Injects anti-detection script via add_init_script().
+# If force=False, randomly skips 30% of runs as a decoy (looks like random
+# human browsing). If flagged by Facebook, returns flagged=True for the
+# dispatcher to fall through to the Agent.
+
 async def _scrape_with_firefox(profile_path: str, force: bool) -> dict:
     """Scrape Facebook using Firefox + persistent real profile (no cookie injection)."""
     if not profile_path:
@@ -939,6 +1123,136 @@ async def _scrape_with_firefox(profile_path: str, force: bool) -> dict:
                 pass
 
 
+# ── Chromium Scraper ─────────────────────────────────────────────────
+# Generic scraper for Chrome/Edge/Opera. Uses the same structure as the
+# Firefox scraper but with Chromium-specific launch config:
+#   - Chrome: channel="chrome"
+#   - Edge:   channel="msedge"
+#   - Opera:  executable_path from shutil.which("opera")
+# Cookies come from copy_chrome_profile (temp copy of user's profile data).
+# Anti-detection script differs slightly from Firefox variant.
+# Same decoy-skip and flagged-fallback behavior as Firefox.
+
+async def _scrape_with_chromium(profile_path: str, browser: str, force: bool = False) -> dict:
+    """Scrape Facebook using a Chromium-based browser profile (Chrome/Edge/Opera)."""
+    if not profile_path:
+        return {"success": False, "leads": [], "flagged": False, "flag_reason": None, "error": "No profile path"}
+
+    channel = None
+    executable_path = None
+    if browser == "chrome":
+        channel = "chrome"
+    elif browser == "edge":
+        channel = "msedge"
+    elif browser == "opera":
+        executable_path = shutil.which("opera") or shutil.which("Opera")
+
+    profile_dir = None
+    try:
+        profile_dir = copy_chrome_profile(profile_path)
+        async with async_playwright() as pw:
+            launch_kwargs = dict(
+                user_data_dir=profile_dir,
+                headless=True,
+                args=CHROME_LAUNCH_ARGS,
+            )
+            if channel:
+                launch_kwargs["channel"] = channel
+            if executable_path:
+                launch_kwargs["executable_path"] = executable_path
+
+            context = await pw.chromium.launch_persistent_context(**launch_kwargs)
+            pages = context.pages
+            page = pages[0] if pages else await context.new_page()
+            await context.add_init_script(chromium_init_script())
+
+            try:
+                await page.goto('https://www.google.com/', wait_until='domcontentloaded', timeout=15000)
+                await page.wait_for_timeout(random.randint(1000, 3000))
+            except Exception:
+                logger.warning("Google navigation failed, trying Facebook directly")
+
+            await page.goto('https://www.facebook.com/', wait_until='domcontentloaded', timeout=30000)
+            await page.wait_for_timeout(random.randint(3000, 8000))
+
+            url = page.url
+            page_text = await page.evaluate('document.body.innerText') if '/login' in url.lower() else ''
+            det = check_detection_signals(url, page_text)
+            if det or '/login' in url.lower():
+                logger.warning("Facebook login page detected — flag: %s", det or "login_page")
+                await context.close()
+                return {"success": False, "leads": [], "flagged": True, "flag_reason": det or "login_page", "error": "Facebook login page detected"}
+
+            await human_scroll(page, steps=random.randint(2, 4), total_delay=random.uniform(8, 20))
+            if random.random() < 0.25:
+                await page.evaluate("window.scrollTo(0, 0)")
+                await page.wait_for_timeout(random.randint(2000, 5000))
+                await human_scroll(page, steps=random.randint(1, 2))
+
+            if not force and random.random() < 0.3:
+                await page.wait_for_timeout(random.randint(8000, 20000))
+                await context.close()
+                return {"success": True, "leads": [], "flagged": False, "flag_reason": None, "error": None}
+
+            all_posts = []
+            searches = random.sample(FB_SEARCHES, k=random.randint(2, 4))
+            for i, query in enumerate(searches):
+                page, posts = await search_facebook(page, context, query)
+                all_posts.extend(posts)
+                if not posts:
+                    continue
+                if random.random() < 0.4:
+                    await page.evaluate(f"window.scrollBy(0, {random.randint(-300, 300)})")
+                delay = random.uniform(8, 25)
+                await page.wait_for_timeout(int(delay * 1000))
+                if i == random.randint(0, 1) and random.random() < 0.15:
+                    new_page = await context.new_page()
+                    try:
+                        await new_page.goto('https://www.facebook.com/groups/', wait_until='domcontentloaded', timeout=15000)
+                        await new_page.wait_for_timeout(random.randint(3000, 8000))
+                    except Exception:
+                        pass
+                    await new_page.close()
+                    page = await _ensure_page(page, context)
+
+            if random.random() < 0.5:
+                await page.wait_for_timeout(random.randint(3000, 10000))
+
+            await context.close()
+
+            seen = set()
+            deduped = []
+            for p in all_posts:
+                key = p.get('content', '')[:100]
+                if key not in seen:
+                    seen.add(key)
+                    deduped.append(p)
+
+            deduped = [p for p in deduped if _is_within_days(p.get('date', ''), 3)]
+            leads = deduped[:20]
+            if leads:
+                leads = await classify_leads(leads)
+
+            return {"success": True, "leads": leads[:15], "flagged": False, "flag_reason": None, "error": None}
+
+    except Exception as e:
+        logger.error("%s scrape failed: %s", browser, e)
+        return {"success": False, "leads": [], "flagged": False, "flag_reason": None, "error": str(e)}
+    finally:
+        if profile_dir and os.path.exists(profile_dir):
+            try:
+                shutil.rmtree(profile_dir, ignore_errors=True)
+            except Exception:
+                pass
+
+
+# ── Agent Fallback ──────────────────────────────────────────────────
+# When all browser-based scrapers fail/are flagged, this fallback uses
+# browser-use Agent with ChatOllama (local, free) to navigate Facebook
+# autonomously via GPT-style prompting. No API keys needed.
+# Uses Chromium headless with the same launch args as _scrape_with_chromium.
+# The Agent is prompted to extract structured post data and return JSON.
+
 async def _scrape_with_agent(force: bool = False) -> dict:
     """Fallback scraper — browser-use Agent + ChatOllama (free/local, Chromium)."""
     cleanup_chrome()
@@ -1016,6 +1330,12 @@ When done, return the data as a JSON list with keys: content, author, url, date.
             except Exception as e:
                 logger.warning("Failed to clean up Chrome profile %s: %s", profile_dir, e)
 
+# ── AI Classification ────────────────────────────────────────────────
+# Uses the local Ollama model to classify scraped posts as LEAD or NOT LEAD.
+# Falls back to keyword-based filtering if the AI is unavailable.
+# ask_ollama() sends a raw prompt to Ollama's /api/chat endpoint.
+# classify_leads() builds a classification prompt and parses the JSON response.
+
 async def ask_ollama(prompt: str) -> str:
     import httpx
     async with httpx.AsyncClient(timeout=120) as c:
@@ -1101,39 +1421,97 @@ Return a JSON array like ["yes","no","yes"] matching the order above."""
         return filtered[:10]
     return []
 
+# ══════════════════════════════════════════════════════════════════════
+# FastAPI Endpoints
+# ══════════════════════════════════════════════════════════════════════
+
 @app.get("/health")
 async def health():
+    """Simple health check for the splash page status polling."""
     return {"status": "ok"}
 
+def _detect_all_profiles() -> dict:
+    """Return all detected browser profiles."""
+    return {
+        "firefox": FX_PROFILE or find_firefox_profile() or "",
+        "opera": OPERA_PROFILE or find_opera_profile() or "",
+        "chrome": CHROME_PROFILE or find_chrome_profile() or "",
+        "edge": EDGE_PROFILE or find_edge_profile() or "",
+    }
+
 @app.get("/setup/profile")
 async def setup_profile():
-    """Auto-detect Firefox profile path."""
-    path = FX_PROFILE or find_firefox_profile()
-    return {"path": path, "detected": bool(path)}
+    """Auto-detect all browser profiles."""
+    return _detect_all_profiles()
+
+async def _check_browser_login(profile_path: str, browser: str) -> dict:
+    """Check Facebook login for a specific browser type."""
+    profile_dir = None
+    try:
+        if browser == "firefox":
+            profile_dir = copy_firefox_profile(profile_path)
+            async with async_playwright() as pw:
+                context = await pw.firefox.launch_persistent_context(
+                    user_data_dir=profile_dir, headless=True,
+                    firefox_user_prefs={"dom.webdriver.enabled": False},
+                )
+                page = context.pages[0] if context.pages else await context.new_page()
+                await page.goto("https://www.facebook.com/", wait_until="domcontentloaded", timeout=30000)
+                await page.wait_for_timeout(3000)
+                logged_in = "/login" not in page.url.lower()
+                await context.close()
+            shutil.rmtree(profile_dir, ignore_errors=True)
+            return {"logged_in": logged_in, "browser": browser, "path": profile_path}
+        else:
+            channel = None
+            exe = None
+            if browser == "chrome":
+                channel = "chrome"
+            elif browser == "edge":
+                channel = "msedge"
+            elif browser == "opera":
+                exe = shutil.which("opera") or shutil.which("Opera")
+            profile_dir = copy_chrome_profile(profile_path)
+            async with async_playwright() as pw:
+                kw = dict(user_data_dir=profile_dir, headless=True, args=CHROME_LAUNCH_ARGS)
+                if channel:
+                    kw["channel"] = channel
+                if exe:
+                    kw["executable_path"] = exe
+                context = await pw.chromium.launch_persistent_context(**kw)
+                page = context.pages[0] if context.pages else await context.new_page()
+                await page.goto("https://www.facebook.com/", wait_until="domcontentloaded", timeout=30000)
+                await page.wait_for_timeout(3000)
+                logged_in = "/login" not in page.url.lower()
+                await context.close()
+            shutil.rmtree(profile_dir, ignore_errors=True)
+            return {"logged_in": logged_in, "browser": browser, "path": profile_path}
+    except Exception as e:
+        if profile_dir:
+            try: shutil.rmtree(profile_dir, ignore_errors=True)
+            except: pass
+        return {"logged_in": False, "browser": browser, "path": profile_path, "error": str(e)}
 
 @app.post("/setup/check-login")
 async def setup_check_login(body: dict):
-    """Check if Facebook is logged in using the given profile."""
-    profile_path = (body.get("profile_path") or "").strip() or FX_PROFILE or find_firefox_profile()
-    if not profile_path:
-        return {"logged_in": False, "reason": "no_profile", "error": "No Firefox profile found"}
-    try:
-        profile_dir = copy_firefox_profile(profile_path)
-        async with async_playwright() as pw:
-            context = await pw.firefox.launch_persistent_context(
-                user_data_dir=profile_dir, headless=True,
-                firefox_user_prefs={"dom.webdriver.enabled": False},
-            )
-            page = context.pages[0] if context.pages else await context.new_page()
-            await page.goto("https://www.facebook.com/", wait_until="domcontentloaded", timeout=30000)
-            await page.wait_for_timeout(3000)
-            logged_in = "/login" not in page.url.lower()
-            await context.close()
-        shutil.rmtree(profile_dir, ignore_errors=True)
-        return {"logged_in": logged_in, "reason": None if logged_in else "login_page"}
-    except Exception as e:
-        logger.error("Setup check-login failed: %s", e)
-        return {"logged_in": False, "reason": "error", "error": str(e)}
+    """Check if Facebook is logged in. Accepts optional 'browser' param, tries all if not given."""
+    browser = (body.get("browser") or "").strip().lower()
+    profile_path = (body.get("profile_path") or "").strip()
+
+    if browser and profile_path:
+        return await _check_browser_login(profile_path, browser)
+
+    # No specific browser — try all detected
+    profiles = _detect_all_profiles()
+    priority = ["firefox", "opera", "chrome", "edge"]
+    for b in priority:
+        p = profiles.get(b, "")
+        if p:
+            result = await _check_browser_login(p, b)
+            if result.get("logged_in"):
+                return result
+            logger.info("%s not logged in, trying next", b)
+    return {"logged_in": False, "reason": "none_logged_in", "message": "No browser with Facebook login found"}
 
 @app.post("/agent/run")
 async def agent_run(task: str = Body(..., embed=True)):

browser-use-service/requirements.txt

@@ -1,5 +1,18 @@
+# ── Python Dependencies for the Facebook Scraper (FastAPI) ───────
+# Install via: pip install -r requirements.txt
+
+# Web framework for the REST API (health, setup, scrape endpoints)
 fastapi>=0.115.0
+
+# ASGI server for running the FastAPI app
 uvicorn>=0.34.0
+
+# Browser automation (launches Firefox, Chrome, Edge, Opera via Playwright)
 playwright>=1.49.0
+
+# AI-powered browser agent (fallback when direct browser scraping is flagged)
+# Uses ChatOllama locally — no API keys needed
 browser-use>=0.1.0
+
+# LangChain integration for ChatOllama (provides the LLM for browser-use Agent)
 langchain-ollama>=0.2.0