import os, json, asyncio, re, shutil, sqlite3, urllib.parse, random, logging, tempfile
from datetime import datetime, timedelta
from fastapi import FastAPI, Query, Body
from fastapi.middleware.cors import CORSMiddleware
import uvicorn
from playwright.async_api import async_playwright
from browser_use import Agent, Browser
from langchain_ollama import ChatOllama


def make_ollama(model: str | None = None, **kwargs) -> ChatOllama:
    """Create ChatOllama with required attrs for browser-use Agent compatibility."""
    llm = ChatOllama(model=model or CLASSIFY_MODEL, **kwargs)
    object.__setattr__(llm, 'provider', 'ollama')
    object.__setattr__(llm, 'name', 'ChatOllama')
    object.__setattr__(llm, 'model_name', llm.model)
    return llm


logging.basicConfig(level=logging.INFO, format='%(asctime)s [%(levelname)s] %(message)s')
logger = logging.getLogger(__name__)

app = FastAPI()
app.add_middleware(
    CORSMiddleware,
    allow_origins=["http://localhost:3006", "http://127.0.0.1:3006"],
    allow_methods=["POST"],
    allow_headers=["*"],
)
PORT = int(os.getenv("PORT", "3008"))

OLLAMA_URL = os.getenv("OLLAMA_URL", "http://localhost:11434")
CLASSIFY_MODEL = os.getenv("CLASSIFY_MODEL", "dolphin-llama3:8b")


FX_PROFILE = os.getenv('FX_PROFILE', '')
CHROME_PROFILE = os.getenv('CHROME_PROFILE', '')

CHROME_LAUNCH_ARGS = [
    '--headless=new',
    '--disable-blink-features=AutomationControlled',
    '--disable-background-networking',
    '--no-first-run',
    '--disable-sync',
    '--disable-field-trial-config',
    '--disable-background-timer-throttling',
    '--disable-backgrounding-occluded-windows',
    '--disable-breakpad',
    '--disable-component-update',
    '--disable-default-apps',
    '--disable-dev-shm-usage',
    '--disable-popup-blocking',
    '--disable-prompt-on-repost',
    '--disable-renderer-backgrounding',
    '--metrics-recording-only',
    '--no-default-browser-check',
    '--disable-extensions-http-throttling',
    '--disable-hang-monitor',
    '--disable-ipc-flooding-protection',
]


def detect_browser_from_profile(profile_path: str) -> str | None:
    """Detect browser type from profile path. Returns 'firefox', 'chromium', or None."""
    if not profile_path:
        return None
    p = profile_path.lower().replace('\\', '/')
    if 'firefox' in p or '.default' in p or '.dev-edition' in p:
        return 'firefox'
    if 'chrome' in p or 'chromium' in p or 'edge' in p:
        return 'chromium'
    return None


def find_firefox_profile() -> str | None:
    """Auto-detect Firefox profile directory cross-platform."""
    import sys
    home = os.path.expanduser("~")
    candidates = []

    if sys.platform == "win32":
        appdata = os.environ.get("APPDATA", "")
        if appdata:
            profiles_dir = os.path.join(appdata, "Mozilla", "Firefox", "Profiles")
        else:
            profiles_dir = os.path.join(home, "AppData", "Roaming", "Mozilla", "Firefox", "Profiles")
    elif sys.platform == "darwin":
        profiles_dir = os.path.join(home, "Library", "Application Support", "Firefox", "Profiles")
    else:
        profiles_dir = os.path.join(home, ".mozilla", "firefox")

    if os.path.isdir(profiles_dir):
        for entry in os.listdir(profiles_dir):
            full = os.path.join(profiles_dir, entry)
            if os.path.isdir(full) and (".default" in entry.lower() or ".dev-edition" in entry.lower()):
                candidates.append(full)
        candidates.sort(key=lambda p: "default-release" in p.lower(), reverse=True)
        if candidates:
            logger.info("Auto-detected Firefox profile: %s", candidates[0])
            return candidates[0]
    return None


def find_chrome_profile() -> str | None:
    """Auto-detect Chrome/Chromium profile directory cross-platform."""
    import sys
    home = os.path.expanduser("~")

    if sys.platform == "win32":
        base = os.path.join(os.environ.get("LOCALAPPDATA", ""), "Google", "Chrome", "User Data")
    elif sys.platform == "darwin":
        base = os.path.join(home, "Library", "Application Support", "Google", "Chrome")
    else:
        base = os.path.join(home, ".config", "google-chrome")

    default_profile = os.path.join(base, "Default")
    if os.path.isdir(default_profile):
        logger.info("Auto-detected Chrome profile: %s", default_profile)
        return default_profile
    return None


def copy_firefox_profile(src_path: str) -> str:
    """Copy essential Firefox profile files to a temp dir for Playwright."""
    essential = ['cookies.sqlite', 'webappsstore.sqlite', 'permissions.sqlite']
    dst = tempfile.mkdtemp(prefix='fb_fx_profile_')
    for f_name in essential:
        s = os.path.join(src_path, f_name)
        if os.path.exists(s):
            shutil.copy2(s, os.path.join(dst, f_name))
    with open(os.path.join(dst, 'profiles.ini'), 'w') as fh:
        fh.write("[Profile0]\nName=default\nIsRelative=0\nPath=.\nDefault=yes\n")
    logger.info("Copied Firefox profile to %s", dst)
    return dst


def copy_chrome_profile(user_data_dir: str, profile_dir: str = 'Default') -> str:
    """Copy Chrome profile to temp dir for launch_persistent_context."""
    essential = ['Cookies', 'Login Data', 'Bookmarks', 'Web Data']
    dst = tempfile.mkdtemp(prefix='fb_chrome_udir_')
    src_profile = os.path.join(user_data_dir, profile_dir)
    os.makedirs(dst, exist_ok=True)
    os.makedirs(os.path.join(dst, profile_dir), exist_ok=True)
    dst_profile = os.path.join(dst, profile_dir)
    for f_name in essential:
        s = os.path.join(src_profile, f_name)
        if os.path.exists(s):
            shutil.copy2(s, os.path.join(dst_profile, f_name))
    for sub in ['Local Storage', 'Session Storage']:
        s = os.path.join(src_profile, sub)
        if os.path.isdir(s):
            shutil.copytree(s, os.path.join(dst_profile, sub), dirs_exist_ok=True)
    local_state_src = os.path.join(user_data_dir, 'Local State')
    local_state_dst = os.path.join(dst, 'Local State')
    if os.path.exists(local_state_src):
        shutil.copy2(local_state_src, local_state_dst)
    logger.info("Copied Chrome profile from %s to %s", src_profile, dst)
    return dst


def ensure_ublock_extension() -> str | None:
    """Download and extract uBlock Origin extension for ad blocking."""
    ext_dir = os.path.join(tempfile.gettempdir(), 'fb_ublock_origin')
    manifest = os.path.join(ext_dir, 'manifest.json')
    if os.path.exists(manifest):
        return ext_dir
    import urllib.request, zipfile
    crx_url = ('https://clients2.google.com/service/update2/crx'
               '?response=redirect&prodversion=133&acceptformat=crx3'
               '&x=id%3Dddkjiahejlhfcafbddmgiahcphecmpfh%26uc')
    crx_path = os.path.join(tempfile.gettempdir(), 'ublock_origin.crx')
    try:
        if not os.path.exists(crx_path):
            logger.info("Downloading uBlock Origin extension...")
            urllib.request.urlretrieve(crx_url, crx_path)
        with zipfile.ZipFile(crx_path, 'r') as z:
            z.extractall(ext_dir)
        logger.info("uBlock Origin extension ready at %s", ext_dir)
        return ext_dir
    except Exception as e:
        logger.warning("Failed to load uBlock Origin extension: %s", e)
        return None


def extract_agent_posts(agent_result: str, page_text: str) -> list[dict]:
    """Parse posts from agent output or page text fallback."""
    posts = []
    # Try JSON from agent output
    json_match = re.search(r'\[.*?\]', agent_result, re.DOTALL)
    if json_match:
        try:
            parsed = json.loads(json_match.group())
            if isinstance(parsed, list):
                for item in parsed:
                    if isinstance(item, dict) and item.get('content'):
                        posts.append({
                            'content': item.get('content', ''),
                            'author': item.get('author', '') or item.get('publisher', '') or '',
                            'url': item.get('url', ''),
                            'date': item.get('date', ''),
                        })
        except json.JSONDecodeError:
            pass

    if not posts:
        lines = [l.strip() for l in agent_result.split('\n') if l.strip()]
        cur = []
        for l in lines:
            if l.startswith(('1.', '2.', '3.', '4.', '5.', '6.', '7.', '8.', '9.', '- ', '* ')):
                if cur:
                    combined = ' '.join(cur)
                    if len(combined) > 30:
                        posts.append({'content': combined[:500], 'author': '', 'url': '', 'date': ''})
                    cur = []
            cur.append(l)
        if cur:
            combined = ' '.join(cur)
            if len(combined) > 30:
                posts.append({'content': combined[:500], 'author': '', 'url': '', 'date': ''})

    return posts


BROAD_KEYWORDS = [
    "website", "web design", "web develop", "web dev",
    "build my website", "build a website", "create a website",
    "need web", "looking for web", "new website",
    "landing page", "wordpress",
    "need a website", "my website", "business website",
    "need a designer", "help with my website",
    "redesign", "update my website",
    "looking for", "need a", "need an", "looking to",
    "help me build", "create my", "build me",
    "design my", "make me a", "would like",
    "need someone", "hire a", "looking to hire",
    "want someone", "need help with",
]

OFFER_PATTERNS = [
    r"\bfree\s+domain\b",
    r"\bweb\s+hosting\b",
    r"\bfree\s+website\b",
    r"\bprofessional\s+website",
    r"\baffordable\s+web\b",
    r"\bget\s+a\s+free\b",
    r"\bsee\s+more\b",
    r"\bbook\s+your\b",
    r"\bhire\s+a\s+freelancer\b",
    r"\bcontent\s+strategy\b",
    r"\bdigital\s+marketing\b",
    r"\bseo\s+service",
    r"\bsocial\s+media\b",
    r"\blimited\s+time\s+offer\b",
    r"\bspecial\s+offer\b",
    r"\bsign\s+up\s+now\b",
    r"\br\d[\d,.]",
    r"\bstarting\s+at\b",
    r"\bper\s+month\b",
    r"\bmonthly\b",
    r"\bfreelancer\s+marketplace\b",
    r"\bfirst\s+order\b",
    r"\bdomain\s+name\b",
    r"\bregister\s+your\s+domain\b",
    r"\bsponsored\b",
    r"\bpromoted\b",
    r"\badvertisement\b",
]

REQUEST_PATTERNS = [
    r"\bi\s+need\b",
    r"\bi\s+need\s+someone\b",
    r"\bi\s+need\s+a\b",
    r"\bi\s+need\s+an\b",
    r"\bi\s+want\b",
    r"\bi\s+would\s+like\b",
    r"\bi'm\s+looking\b",
    r"\bi\s+am\s+looking\b",
    r"\blooking\s+for\b",
    r"\blooking\s+to\b",
    r"\bwho\s+can\b",
    r"\bcan\s+someone\b",
    r"\bhelp\s+me\b",
    r"\bneed\s+help\b",
    r"\bwould\s+like\s+to\b",
    r"\bwant\s+to\b",
    r"\banyone\s+know\b",
    r"\bdoes\s+anyone\b",
    r"\brecommend\b",
    r"\bsuggestion\b",
    r"\bquote\s+(please|for|to)\b",
]

def firefox_init_script() -> str:
    return r"""// Firefox Anti-Detection
Object.defineProperty(navigator, 'webdriver', { get: () => undefined });
Object.defineProperty(navigator, 'languages', { get: () => ['en-US', 'en'] });
Object.defineProperty(navigator, 'platform', { get: () => 'Win32' });
Object.defineProperty(navigator, 'hardwareConcurrency', { get: () => 8 });
Object.defineProperty(navigator, 'maxTouchPoints', { get: () => 0 });
Object.defineProperty(navigator, 'oscpu', { get: () => 'Windows NT 10.0; Win64; x64' });
Object.defineProperty(navigator, 'vendor', { get: () => '' });
Object.defineProperty(navigator, 'vendorSub', { get: () => '' });
Object.defineProperty(navigator, 'productSub', { get: () => '20100101' });
Object.defineProperty(navigator, 'product', { get: () => 'Gecko' });
Object.defineProperty(navigator, 'appCodeName', { get: () => 'Mozilla' });
Object.defineProperty(navigator, 'appName', { get: () => 'Netscape' });
Object.defineProperty(navigator, 'appVersion', { get: () => '5.0 (Windows NT 10.0; Win64; x64; rv:130.0) Gecko/20100101 Firefox/130.0' });
Object.defineProperty(navigator, 'userAgent', { get: () => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:130.0) Gecko/20100101 Firefox/130.0' });
Object.defineProperty(navigator, 'buildID', { get: () => '20250101000000' });
if (window.chrome !== undefined) { window.chrome = undefined; }
if (navigator.permissions) {
    const origQ = navigator.permissions.query.bind(navigator.permissions);
    navigator.permissions.query = (p) => origQ(p).then(r => (['camera','microphone','geolocation','notifications'].includes(p.name) ? Object.assign(r, { state: 'prompt' }) : r));
}
const _gEC = HTMLCanvasElement.prototype.getContext;
HTMLCanvasElement.prototype.getContext = function(t, ...a) {
    const ctx = _gEC.call(this, t, ...a);
    if (ctx && (t === 'webgl' || t === 'webgl2')) {
        const _gP = ctx.getParameter.bind(ctx);
        ctx.getParameter = function(p) { if (p===37445) return 'Intel Inc.'; if (p===37446) return 'Intel Iris OpenGL Engine'; if (p===7936||p===7937||p===35724) return ''; return _gP(p); };
        const _gE = ctx.getExtension.bind(ctx);
        ctx.getExtension = function(n) { if (n==='WEBGL_debug_renderer_info') return null; return _gE(n); };
    }
    return ctx;
};
const _tDU = HTMLCanvasElement.prototype.toDataURL;
HTMLCanvasElement.prototype.toDataURL = function(...a) {
    const img = _tDU.apply(this, a);
    if (img.includes('image/png') && img.length > 5000) { const n = btoa(String.fromCharCode(Math.floor(Math.random()*256))); return img.slice(0,-100)+n+img.slice(-100); }
    return img;
};
const _gFD = AnalyserNode.prototype.getFloatFrequencyData;
if (_gFD) { AnalyserNode.prototype.getFloatFrequencyData = function(a) { _gFD.call(this,a); for(let i=0;i<a.length;i++) a[i]+=(Math.random()-0.5)*0.02; }; }
Object.defineProperty(screen, 'colorDepth', { get: () => 24 });
Object.defineProperty(screen, 'pixelDepth', { get: () => 24 });
Object.defineProperty(navigator, 'plugins', { get: () => [{name:'Widevine Content Decryption Module',filename:'widevinecdm.dll',length:1,description:'Enables Widevine licenses'},{name:'OpenH264 Video Codec',filename:'openh264.dll',length:1,description:'H.264 video codec'}] });
Object.defineProperty(navigator, 'mimeTypes', { get: () => [{type:'application/pdf',suffixes:'pdf',description:'Portable Document Format',enabledPlugin:navigator.plugins[0]}] });"""


def chromium_init_script() -> str:
    return r"""// Enhanced Chromium Anti-Detection
Object.defineProperty(navigator, 'webdriver', { get: () => false });
Object.defineProperty(navigator, 'languages', { get: () => ['en-US', 'en'] });
Object.defineProperty(navigator, 'platform', { get: () => 'Win32' });
Object.defineProperty(navigator, 'hardwareConcurrency', { get: () => 8 });
Object.defineProperty(navigator, 'deviceMemory', { get: () => 8 });
Object.defineProperty(navigator, 'maxTouchPoints', { get: () => 0 });
Object.defineProperty(navigator, 'oscpu', { get: () => 'Windows NT 10.0; Win64; x64' });
Object.defineProperty(navigator, 'vendor', { get: () => 'Google Inc.' });
Object.defineProperty(navigator, 'vendorSub', { get: () => '' });
Object.defineProperty(navigator, 'productSub', { get: () => '20030107' });
Object.defineProperty(navigator, 'product', { get: () => 'Gecko' });
Object.defineProperty(navigator, 'appCodeName', { get: () => 'Mozilla' });
Object.defineProperty(navigator, 'appName', { get: () => 'Netscape' });
Object.defineProperty(navigator, 'appVersion', { get: () => '5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36' });
Object.defineProperty(navigator, 'userAgent', { get: () => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36' });
window.chrome = { runtime: { onConnect: { addListener: () => {} }, onMessage: { addListener: () => {} }, sendMessage: () => {} }, app: { isInstalled: false, InstallState: { DISABLED: 'disabled', INSTALLED: 'installed', NOT_INSTALLED: 'not_installed' }, getDetails: () => null, getIsInstalled: () => false }, csi: () => {}, loadTimes: () => {} };
if (navigator.permissions) {
    const origQ = navigator.permissions.query.bind(navigator.permissions);
    navigator.permissions.query = (p) => origQ(p).then(r => (['camera','microphone','geolocation','notifications'].includes(p.name) ? Object.assign(r, { state: 'prompt' }) : r));
}
const _gEC = HTMLCanvasElement.prototype.getContext;
HTMLCanvasElement.prototype.getContext = function(t, ...a) {
    const ctx = _gEC.call(this, t, ...a);
    if (ctx && (t === 'webgl' || t === 'webgl2')) {
        const _gP = ctx.getParameter.bind(ctx);
        ctx.getParameter = function(p) { if (p===37445) return 'Intel Inc.'; if (p===37446) return 'Intel Iris OpenGL Engine'; if (p===7936||p===7937||p===35724) return ''; return _gP(p); };
        const _gE = ctx.getExtension.bind(ctx);
        ctx.getExtension = function(n) { if (n==='WEBGL_debug_renderer_info') return null; return _gE(n); };
    }
    return ctx;
};
const _tDU = HTMLCanvasElement.prototype.toDataURL;
HTMLCanvasElement.prototype.toDataURL = function(...a) {
    const img = _tDU.apply(this, a);
    if (img.includes('image/png') && img.length > 5000) { const n = btoa(String.fromCharCode(Math.floor(Math.random()*256))); return img.slice(0,-100)+n+img.slice(-100); }
    return img;
};
const _gFD = AnalyserNode.prototype.getFloatFrequencyData;
if (_gFD) { AnalyserNode.prototype.getFloatFrequencyData = function(a) { _gFD.call(this,a); for(let i=0;i<a.length;i++) a[i]+=(Math.random()-0.5)*0.02; }; }
Object.defineProperty(screen, 'colorDepth', { get: () => 24 });
Object.defineProperty(screen, 'pixelDepth', { get: () => 24 });
Object.defineProperty(navigator, 'plugins', { get: () => [1,2,3,4,5].map(i => ({name:['Chrome PDF Plugin','Chrome PDF Viewer','Native Client','Widevine Content Decryption Module','Chromium PDF Viewer'][i-1],filename:'internal-pdf-viewer',length:1,description:['Portable Document Format','','Native Client Executable','Enables Widevine licenses',''][i-1]})) });
Object.defineProperty(navigator, 'mimeTypes', { get: () => [{type:'application/pdf',suffixes:'pdf',description:'Portable Document Format',enabledPlugin:navigator.plugins[0]},{type:'text/pdf',suffixes:'pdf',description:'Portable Document Format',enabledPlugin:navigator.plugins[0]}] });"""


def kw_match(text: str) -> bool:
    t = text.lower()
    return any(kw in t for kw in BROAD_KEYWORDS)

def is_request(text: str) -> bool:
    t = text.lower()
    return any(re.search(p, t) for p in REQUEST_PATTERNS)

def is_offer(text: str) -> bool:
    t = text.lower()
    return any(re.search(p, t) for p in OFFER_PATTERNS)

FB_SEARCHES = [
    "looking for web developer",
    "need a website designed",
    "need website built South Africa",
    "need someone to build my website",
    "need web designer",
    "looking for someone to create website",
    "need ecommerce website built",
    "want to hire web developer",
    "need wordpress website",
    "I need a website for my business",
    "need a site for my business",
    "looking for website designer South Africa",
    "need website for my small business",
    "who can build me a website",
    "want to create a website for my business",
    "looking for affordable web design",
    "need a web developer South Africa",
    "help me build a website",
    "need someone to design my website",
    "looking for web designer near me",
    "need a website for my startup",
]

VIEWPORTS = [
    {'width': 1280, 'height': 800},
    {'width': 1366, 'height': 768},
    {'width': 1440, 'height': 900},
    {'width': 1536, 'height': 864},
    {'width': 1920, 'height': 1080},
]

async def get_fb_cookies(profile_path: str | None = None):
    cookie_db_path = profile_path or FX_PROFILE
    if not cookie_db_path:
        logger.warning("No profile path provided")
        return []
    cookie_db = os.path.join(cookie_db_path, 'cookies.sqlite')
    if not os.path.exists(cookie_db):
        logger.warning("Cookie DB not found at %s", cookie_db)
        return []
    tmp = os.path.join(os.path.dirname(cookie_db), f'cookies_tmp_{os.getpid()}.sqlite')
    for attempt in range(3):
        try:
            shutil.copy2(cookie_db, tmp)
            break
        except PermissionError:
            if attempt < 2:
                await asyncio.sleep(0.5)
            else:
                logger.error("Failed to copy cookie DB after 3 attempts")
                return []
    try:
        conn = sqlite3.connect(tmp)
        c = conn.cursor()
        c.execute("SELECT name, value, host, path FROM moz_cookies WHERE host LIKE '%facebook.com'")
        rows = c.fetchall()
        conn.close()
        try:
            os.remove(tmp)
        except Exception:
            pass
        return [{
            "name": name, "value": value,
            "domain": host if host.startswith('.') else f'.{host}',
            "path": path,
            "httpOnly": True, "secure": True, "sameSite": "Lax",
        } for name, value, host, path in rows]
    except Exception as e:
        logger.error("Cookie DB read error: %s", e)
        try:
            os.remove(tmp)
        except Exception:
            pass
        return []

WEEKDAY_ORDER = ['monday', 'tuesday', 'wednesday', 'thursday', 'friday', 'saturday', 'sunday']

def _parse_fb_date(block: list[str]) -> str:
    for l in block:
        l = l.strip()
        m = re.match(r'(\d+)\s*(h|hr|hrs|hour|hours)\s*ago', l)
        if m:
            hours = int(m.group(1))
            d = datetime.now() - timedelta(hours=hours)
            return d.strftime('%Y-%m-%d')
        if l.lower() == 'yesterday':
            d = datetime.now() - timedelta(days=1)
            return d.strftime('%Y-%m-%d')
        low = l.lower()
        if low in WEEKDAY_ORDER:
            day_idx = WEEKDAY_ORDER.index(low)
            today_idx = datetime.now().weekday()
            days_ago = (today_idx - day_idx) % 7
            d = datetime.now() - timedelta(days=days_ago)
            return d.strftime('%Y-%m-%d')
        for fmt in ['%Y-%m-%d', '%d %B %Y', '%B %d %Y', '%d %b %Y', '%b %d %Y', '%d %b', '%b %d']:
            try:
                dt = datetime.strptime(l, fmt)
                return dt.strftime('%Y-%m-%d')
            except ValueError:
                pass
    return datetime.now().strftime('%Y-%m-%d')


def _is_within_days(date_str: str, max_days: int = 3) -> bool:
    """Check if date is within max_days from now. Empty/unparseable = keep."""
    if not date_str:
        return True
    try:
        dt = datetime.strptime(date_str.strip(), '%Y-%m-%d')
        return (datetime.now() - dt).days <= max_days
    except ValueError:
        return True


def _clean_fb_text(text: str) -> str:
    cleaned_lines = []
    for l in text.split('\n'):
        stripped = l.strip()
        if not stripped:
            continue
        if len(stripped) < 3:
            continue
        if all(not c.isalpha() for c in stripped):
            continue
        cleaned_lines.append(stripped)
    return '\n'.join(cleaned_lines)

def _extract_posts_from_elements(elements: list[dict], base_url: str) -> list[dict]:
    posts = []
    seen_texts = set()
    for el in elements:
        raw_text = el.get('text', '')
        if len(raw_text) < 40:
            continue
        text = _clean_fb_text(raw_text)
        if len(text) < 40:
            continue
        if is_offer(text):
            continue
        lines = text.split('\n')
        dekey = text[:80]
        if dekey in seen_texts:
            continue
        seen_texts.add(dekey)
        request_score = 2 if is_request(text) else 0
        post_url = el.get('url') or base_url

        # Prefer JS-extracted date, fall back to text parsing
        js_date = (el.get('date') or '').strip()
        if js_date:
            # Try ISO datetime from <time datetime>
            try:
                dt = datetime.fromisoformat(js_date.replace('Z', '+00:00'))
                date_str = dt.strftime('%Y-%m-%d')
            except (ValueError, TypeError):
                date_str = _parse_fb_date([js_date])
        else:
            date_str = _parse_fb_date(lines)

        posts.append({
            "title": text[:300],
            "content": text[:1000],
            "author": el.get('author', ''),
            "url": post_url,
            "source": "facebook",
            "date": date_str,
            "_score": request_score,
        })
    posts.sort(key=lambda p: p.get('_score', 0), reverse=True)
    for p in posts:
        p.pop('_score', None)
    return posts

def _extract_posts_from_text(raw: str, url: str) -> list[dict]:
    lines = [l.strip() for l in raw.split('\n') if l.strip() and len(l.strip()) >= 15]
    posts = []
    seen_texts = set()
    cur = []
    for l in lines:
        words = re.findall(r'[A-Za-z]{2,}', l)
        if len(words) < 2:
            continue
        lower = l.lower()
        if any(kw in lower for kw in BROAD_KEYWORDS) and not is_offer(l):
            if cur:
                snippet = ' '.join(cur[-3:] + [l])
                if len(snippet) >= 40 and not is_offer(snippet):
                    dekey = snippet[:80]
                    if dekey not in seen_texts:
                        seen_texts.add(dekey)
                        posts.append({
                            "title": snippet[:300],
                            "content": snippet[:1000],
                            "author": '',
                            "url": url,
                            "source": "facebook",
                            "date": _parse_fb_date(cur + [l]),
                        })
                cur = []
            else:
                if not is_offer(l):
                    snippet = l
                    if len(snippet) >= 40:
                        dekey = snippet[:80]
                        if dekey not in seen_texts:
                            seen_texts.add(dekey)
                            posts.append({
                                "title": snippet[:300],
                                "content": snippet[:1000],
                                "author": '',
                                "url": url,
                                "source": "facebook",
                                "date": _parse_fb_date([l]),
                            })
        cur.append(l)
        if len(cur) > 10:
            cur.pop(0)
    return posts

async def human_scroll(page, steps: int = None, total_delay: float = None):
    steps = steps or random.randint(2, 5)
    total_delay = total_delay or random.uniform(6, 18)
    step_delay = total_delay / steps
    for i in range(steps):
        scroll_dist = random.randint(200, 600)
        await page.evaluate(f"window.scrollBy(0, {scroll_dist})")
        try:
            vp = await page.evaluate('({w: window.innerWidth, h: window.innerHeight})')
            x = random.randint(100, vp['w'] - 100)
            y = random.randint(100, vp['h'] - 100)
            await page.mouse.move(x, y, steps=random.randint(15, 35))
        except Exception:
            pass
        await page.wait_for_timeout(int(step_delay * 1000))
    if random.random() < 0.3:
        await page.evaluate(f"window.scrollBy(0, -{random.randint(100, 300)})")
        await page.wait_for_timeout(random.randint(1000, 3000))

async def random_idle(page):
    action = random.random()
    if action < 0.15:
        try:
            elems = await page.query_selector_all('a, span, div[role="button"]')
            if elems:
                el = random.choice(elems[:20])
                box = await el.bounding_box()
                if box:
                    x = box['x'] + box['width'] / 2 + random.randint(-20, 20)
                    y = box['y'] + box['height'] / 2 + random.randint(-10, 10)
                    await page.mouse.move(x, y, steps=random.randint(20, 40))
                    await page.wait_for_timeout(random.randint(500, 2000))
        except Exception:
            pass

async def _get_article_elements(page) -> list[dict]:
    return await page.evaluate('''() => {
        const results = [];
        const seenTexts = new Set();
        const selectors = [
            'div[role="article"]',
            'div[role="feed"] > div',
            'div.x1yztbdb',
            'div[data-pagelet]',
        ];
        for (const sel of selectors) {
            const els = document.querySelectorAll(sel);
            for (const el of els) {
                const text = (el.innerText || '').trim();
                if (text.length < 40) continue;
                const key = text.substring(0, 80);
                if (seenTexts.has(key)) continue;
                seenTexts.add(key);

                // --- Publisher name ---
                let author = '';
                const nameEl = el.querySelector('h4, strong, a[role="link"]');
                if (nameEl) {
                    author = (nameEl.innerText || '').trim();
                }
                if (!author) {
                    const links = el.querySelectorAll('a');
                    for (const a of links) {
                        const t = (a.innerText || '').trim();
                        if (t && t.length > 1 && t.length < 60 && /^[A-Za-zÀ-ÿ]/.test(t) && !t.includes('facebook') && !t.includes('/')) {
                            author = t;
                            break;
                        }
                    }
                }

                // --- Post URL ---
                let postUrl = '';
                for (const a of el.querySelectorAll('a')) {
                    const h = (a.getAttribute('href') || '');
                    if (h.includes('/posts/') || h.includes('/photo/') || h.includes('/video/') || h.includes('/groups/') || h.includes('/permalink/')) {
                        postUrl = h.startsWith('http') ? h : 'https://www.facebook.com' + h;
                        break;
                    }
                }

                // --- Date from <time datetime> ---
                let date = '';
                const timeEl = el.querySelector('time');
                if (timeEl) {
                    date = timeEl.getAttribute('datetime') || timeEl.getAttribute('aria-label') || '';
                }

                results.push({ text, author, url: postUrl, date });
            }
            if (results.length > 0) break;
        }
        return results;
    }''')

async def _ensure_page(page, context):
    try:
        await page.evaluate('1')
        return page
    except Exception:
        logger.warning("Page was closed mid-scrape, creating a fresh page")
        page = await context.new_page()
        try:
            await page.goto('https://www.google.com/', wait_until='domcontentloaded', timeout=15000)
            await page.wait_for_timeout(random.randint(1000, 3000))
        except Exception:
            logger.warning("Google navigation failed during page recreation")
        await page.goto('https://www.facebook.com/', wait_until='domcontentloaded', timeout=30000)
        await page.wait_for_timeout(random.randint(3000, 8000))
        return page

async def search_facebook(page, context, query: str):
    page = await _ensure_page(page, context)
    url = f'https://www.facebook.com/search/posts/?q={urllib.parse.quote(query)}'
    try:
        await page.goto(url, wait_until='domcontentloaded', timeout=30000)
        try:
            await page.wait_for_selector('div[role="article"], div[role="feed"]', timeout=15000)
        except Exception:
            pass
        await page.wait_for_timeout(random.randint(3000, 8000))

        await human_scroll(page, steps=random.randint(2, 4), total_delay=random.uniform(6, 15))

        if random.random() < 0.1:
            await page.evaluate("window.scrollTo(0, document.body.scrollHeight)")
            await page.wait_for_timeout(random.randint(1000, 3000))
            await page.evaluate("window.scrollBy(0, -300)")
            await page.wait_for_timeout(random.randint(1000, 3000))

        if random.random() < 0.2:
            await page.evaluate("window.scrollTo(0, 0)")
            await page.wait_for_timeout(random.randint(2000, 5000))

        if random.random() < 0.3:
            await random_idle(page)

        raw_articles = await _get_article_elements(page)
        posts = _extract_posts_from_elements(raw_articles, url) if raw_articles else []
        if not posts:
            raw = await page.evaluate('document.body.innerText')
            posts = _extract_posts_from_text(raw, url)
    except Exception as e:
        logger.warning("Facebook search '%s' failed: %s", query, e)
        return page, []
    return page, posts

DETECTION_SIGNALS = [
    '/checkpoint/', '/login.php?', 'action=security_check',
    'unusual activity', 'suspicious login', 'suspicious activity',
    'temporarily blocked', 'temporarily restricted',
    'confirm your identity', 'enter the code we sent',
    'we noticed unusual', 'help us confirm',
    'security check', 'challenge', 'please verify',
]


def check_detection_signals(page_url: str, page_text: str = '') -> str | None:
    """Check page for Facebook detection signals. Returns signal text or None."""
    combined = (page_url + ' ' + page_text).lower()
    for signal in DETECTION_SIGNALS:
        if signal in combined:
            return signal
    return None


def cleanup_chrome():
    import subprocess, signal
    try:
        subprocess.run(["taskkill", "/F", "/IM", "chrome-headless-shell.exe"], capture_output=True, timeout=5)
    except Exception:
        pass

async def scrape_facebook(profile_path: str | None = None, force: bool = False) -> dict:
    """Dispatcher — Firefox primary, browser-use Agent fallback."""
    effective_path = profile_path or FX_PROFILE

    # Auto-detect Firefox profile if none provided
    if not effective_path:
        detected = find_firefox_profile()
        if detected:
            effective_path = detected
            os.environ["FX_PROFILE"] = detected

    browser_type = detect_browser_from_profile(effective_path)

    if not browser_type and CHROME_PROFILE:
        browser_type = detect_browser_from_profile(CHROME_PROFILE)
        effective_path = CHROME_PROFILE

    # Auto-detect Chrome profile if still nothing
    if not browser_type:
        detected_chrome = find_chrome_profile()
        if detected_chrome:
            effective_path = detected_chrome
            browser_type = 'chromium'

    logger.info("Detected browser: %s (profile: %s)", browser_type or "none", effective_path)

    # Firefox primary (raw Playwright, stealth)
    if browser_type == "firefox":
        result = await _scrape_with_firefox(effective_path, force)
        if result.get("success") or not result.get("flagged"):
            return result
        logger.warning("Firefox path failed (%s), falling back to Agent", result.get("flag_reason", "unknown"))
        return await _scrape_with_agent(force)

    # CHROME_PROFILE set or no profile → Agent
    if browser_type == "chromium":
        return await _scrape_with_agent(force)

    # No profile at all → try Agent (fresh Chromium)
    return await _scrape_with_agent(force)


async def _scrape_with_firefox(profile_path: str, force: bool) -> dict:
    """Scrape Facebook using Firefox + persistent real profile (no cookie injection)."""
    if not profile_path:
        return {"success": False, "leads": [], "flagged": False, "flag_reason": None, "error": "No profile path"}

    profile_dir = None
    try:
        profile_dir = copy_firefox_profile(profile_path)
        async with async_playwright() as pw:
            context = await pw.firefox.launch_persistent_context(
                user_data_dir=profile_dir,
                headless=True,
                firefox_user_prefs={
                    "dom.webdriver.enabled": False,
                    "dom.webdriver.timeout": 0,
                    "useAutomationExtension": False,
                    "privacy.trackingprotection.enabled": False,
                    "privacy.trackingprotection.fingerprinting.enabled": False,
                    "privacy.trackingprotection.cryptomining.enabled": False,
                },
            )
            pages = context.pages
            page = pages[0] if pages else await context.new_page()
            await context.add_init_script(firefox_init_script())

            try:
                await page.goto('https://www.google.com/', wait_until='domcontentloaded', timeout=15000)
                await page.wait_for_timeout(random.randint(1000, 3000))
            except Exception:
                logger.warning("Google navigation failed, trying Facebook directly")

            await page.goto('https://www.facebook.com/', wait_until='domcontentloaded', timeout=30000)
            await page.wait_for_timeout(random.randint(3000, 8000))

            url = page.url
            page_text = await page.evaluate('document.body.innerText') if '/login' in url.lower() else ''
            det = check_detection_signals(url, page_text)
            if det or '/login' in url.lower():
                logger.warning("Facebook login page detected — flag: %s", det or "login_page")
                await context.close()
                return {"success": False, "leads": [], "flagged": True, "flag_reason": det or "login_page", "error": "Facebook login page detected"}

            await human_scroll(page, steps=random.randint(2, 4), total_delay=random.uniform(8, 20))
            if random.random() < 0.25:
                await page.evaluate("window.scrollTo(0, 0)")
                await page.wait_for_timeout(random.randint(2000, 5000))
                await human_scroll(page, steps=random.randint(1, 2))

            if not force and random.random() < 0.3:
                await page.wait_for_timeout(random.randint(8000, 20000))
                await context.close()
                return {"success": True, "leads": [], "flagged": False, "flag_reason": None, "error": None}

            all_posts = []
            searches = random.sample(FB_SEARCHES, k=random.randint(2, 4))
            for i, query in enumerate(searches):
                page, posts = await search_facebook(page, context, query)
                all_posts.extend(posts)
                if not posts:
                    continue
                if random.random() < 0.4:
                    await page.evaluate(f"window.scrollBy(0, {random.randint(-300, 300)})")
                delay = random.uniform(8, 25)
                await page.wait_for_timeout(int(delay * 1000))
                if i == random.randint(0, 1) and random.random() < 0.15:
                    new_page = await context.new_page()
                    try:
                        await new_page.goto('https://www.facebook.com/groups/', wait_until='domcontentloaded', timeout=15000)
                        await new_page.wait_for_timeout(random.randint(3000, 8000))
                    except Exception:
                        pass
                    await new_page.close()
                    page = await _ensure_page(page, context)

            if random.random() < 0.5:
                await page.wait_for_timeout(random.randint(3000, 10000))

            await context.close()

            seen = set()
            deduped = []
            for p in all_posts:
                key = p.get('content', '')[:100]
                if key not in seen:
                    seen.add(key)
                    deduped.append(p)

            # Filter to last 3 days only
            deduped = [p for p in deduped if _is_within_days(p.get('date', ''), 3)]

            leads = deduped[:20]
            if leads:
                leads = await classify_leads(leads)

            return {"success": True, "leads": leads[:15], "flagged": False, "flag_reason": None, "error": None}

    except Exception as e:
        logger.error("Firefox scrape failed: %s", e)
        return {"success": False, "leads": [], "flagged": False, "flag_reason": None, "error": str(e)}
    finally:
        if profile_dir and os.path.exists(profile_dir):
            try:
                shutil.rmtree(profile_dir, ignore_errors=True)
            except Exception:
                pass


async def _scrape_with_agent(force: bool = False) -> dict:
    """Fallback scraper — browser-use Agent + ChatOllama (free/local, Chromium)."""
    cleanup_chrome()
    profile_dir = None
    try:
        user_data_dir = None
        if CHROME_PROFILE:
            profile_dir = copy_chrome_profile(CHROME_PROFILE)
            user_data_dir = profile_dir

        browser = Browser(
            headless=True,
            args=CHROME_LAUNCH_ARGS,
            user_data_dir=user_data_dir,
            allowed_domains=["*.facebook.com", "*.messenger.com", "www.google.com"],
        )
        await browser.start()

        all_posts = []
        for query in random.sample(FB_SEARCHES, k=random.randint(2, 4)):
            agent = Agent(
                task=f"""You are logged into Facebook. Do the following:
1. Navigate to facebook.com and make sure you are on the homepage
2. Use the Facebook search to find: {query}
3. Scroll through the results
4. For each relevant post, extract and list:
   - The publisher/author name (who posted it)
   - The post text content
   - The post URL (if visible)
   - The post date
5. ONLY include posts from the last 3 days
6. Collect as many posts as you can (aim for 5-10 per search)

When done, return the data as a JSON list with keys: content, author, url, date.""",
                llm=make_ollama(num_ctx=32000, temperature=0.3),
                browser=browser,
                max_actions_per_step=3,
                use_vision=False,
                max_failures=6,
                step_timeout=180,
            )
            history = await agent.run()
            result = history.final_result() or ""
            posts = extract_agent_posts(result, "")
            all_posts.extend(posts)

            if random.random() < 0.3:
                await asyncio.sleep(random.randint(5, 15))

        await browser.close()

        seen = set()
        deduped = []
        for p in all_posts:
            key = p.get('content', '')[:100]
            if key not in seen:
                seen.add(key)
                deduped.append(p)

        # Filter to last 3 days only
        deduped = [p for p in deduped if _is_within_days(p.get('date', ''), 3)]

        leads = deduped[:20]
        if leads:
            leads = await classify_leads(leads)

        return {"success": True, "leads": leads[:15], "flagged": False, "flag_reason": None, "error": None}
    except Exception as e:
        logger.error("Agent scrape failed: %s", e)
        return {"success": False, "leads": [], "flagged": False, "flag_reason": None, "error": str(e)}
    finally:
        if profile_dir and os.path.isdir(profile_dir):
            try:
                shutil.rmtree(profile_dir, ignore_errors=True)
            except Exception as e:
                logger.warning("Failed to clean up Chrome profile %s: %s", profile_dir, e)

async def ask_ollama(prompt: str) -> str:
    import httpx
    async with httpx.AsyncClient(timeout=120) as c:
        r = await c.post(f"{OLLAMA_URL}/api/chat", json={
            "model": CLASSIFY_MODEL,
            "messages": [
                {"role": "system", "content": "You classify forum posts. Return only valid JSON."},
                {"role": "user", "content": prompt}
            ],
            "stream": False,
            "options": {"temperature": 0.05, "num_predict": 1024},
        })
        r.raise_for_status()
        data = r.json()
        return data["message"]["content"]

async def classify_leads(results: list[dict]) -> list[dict]:
    if not results:
        return []
    briefs = [r["title"][:200] for r in results]
    prompt = f"""Classify each post as LEAD or NOT.
LEAD = someone REQUESTING/POSTING/WANTING a website built, designed, or created for them.
LEAD examples: "Need a website for my business", "Looking for web developer to build my site", "I need someone to create my website", "Want a new website for my company", "Looking for someone to design my WordPress site"

NOT LEAD:
- Offering web design services: "I build websites", "I offer web design", "Affordable web design packages"
- Already have a website and need marketing, SEO, content, video, link building, email marketing, affiliates
- Recruiting employees, hiring staff, looking for business partners
- Selling products, promoting services, affiliate offers
- "Need web hosting", "Looking for a partner", "Looking for content writer", "Video spokesperson"

For each numbered post, answer ONLY "yes" (LEAD) or "no" (NOT LEAD):
{chr(10).join(f'{i+1}. {t}' for i, t in enumerate(briefs))}
Return a JSON array like ["yes","no","yes"] matching the order above."""
    ai_succeeded = False
    try:
        raw = await ask_ollama(prompt)
        raw = raw.strip()
        if raw.startswith("```"):
            first_nl = raw.find('\n')
            if first_nl != -1:
                raw = raw[first_nl + 1:]
            if raw.endswith("```"):
                raw = raw[:-3]
            raw = raw.strip()
        answers = json.loads(raw)
        if isinstance(answers, list) and len(answers) == len(results):
            ai_succeeded = True
            filtered = []
            for i, ans in enumerate(answers):
                if isinstance(ans, str) and ans.lower() == 'yes':
                    filtered.append(results[i])
            if filtered:
                return filtered[:10]
            logger.info("AI classified all %d items as NOT LEAD — returning empty", len(results))
            return []
    except Exception as e:
        logger.warning("AI classification failed, falling back to keyword filter: %s", e)

    if not ai_succeeded:
        strict_keywords = [
            "website", "web design", "web develop", "web dev",
            "build my website", "build a website", "create a website",
            "need web", "looking for web", "new website",
            "landing page", "wordpress",
            "need a website", "my website", "business website",
            "need a designer", "help with my website",
            "redesign", "update my website",
            "looking for", "need a", "need an", "looking to",
            "need someone", "hire a", "want someone",
            "need help with", "would like", "build me",
            "design my", "make me a", "create my",
        ]
        filtered = []
        for r in results:
            t = r['title'].lower()
            if not any(kw in t for kw in strict_keywords):
                continue
            if any(kw in t for kw in ['i build website', 'i offer web', 'i am a web developer',
                                       'affordable web design package', 'web hosting']):
                continue
            filtered.append(r)
        return filtered[:10]
    return []

@app.get("/health")
async def health():
    return {"status": "ok"}

@app.post("/agent/run")
async def agent_run(task: str = Body(..., embed=True)):
    """Run a browser-use Agent with ChatOllama (free/local)."""
    try:
        browser = Browser(headless=True, args=CHROME_LAUNCH_ARGS)
        await browser.start()
        agent = Agent(
            task=task,
            llm=make_ollama(num_ctx=32000, temperature=0.3),
            browser=browser,
            use_vision=False,
            max_actions_per_step=5,
            max_failures=8,
            step_timeout=180,
        )
        history = await agent.run()
        await browser.close()
        return {"success": True, "result": history.final_result()}
    except Exception as e:
        logger.error("Agent run failed: %s", e)
        return {"success": False, "error": str(e)}

@app.post("/scrape/facebook")
async def scrape_facebook_endpoint(profile_path: str | None = Query(None), force: bool = Query(False)):
    result = await scrape_facebook(profile_path, force)
    return result

if __name__ == "__main__":
    uvicorn.run(app, host="0.0.0.0", port=PORT)