new changes

Refactor code structure for improved readability and maintainability
feat: add Facebook account tracking and management
2026-06-23 15:29:34 +02:00 · 2026-06-23 14:25:39 +02:00 · 2026-06-23 14:18:18 +02:00
134 changed files with 799 additions and 219 deletions
@@ -1,9 +1,7 @@
-import os, json, asyncio, re, shutil, sqlite3, traceback, urllib.parse, random, time, logging
+import os, json, asyncio, re, shutil, sqlite3, urllib.parse, random, logging
 from datetime import datetime, timedelta
-from bs4 import BeautifulSoup
-import httpx
-from fastapi import FastAPI
-from pydantic import BaseModel
+from fastapi import FastAPI, Query
+from fastapi.middleware.cors import CORSMiddleware
 import uvicorn
 from playwright.async_api import async_playwright

@@ -11,22 +9,20 @@ logging.basicConfig(level=logging.INFO, format='%(asctime)s [%(levelname)s] %(me
 logger = logging.getLogger(__name__)

 app = FastAPI()
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["http://localhost:3006", "http://127.0.0.1:3006"],
+    allow_methods=["POST"],
+    allow_headers=["*"],
+)
 PORT = int(os.getenv("PORT", "3008"))

 OLLAMA_URL = os.getenv("OLLAMA_URL", "http://localhost:11434")
 CLASSIFY_MODEL = os.getenv("CLASSIFY_MODEL", "dolphin-llama3:8b")

 FX_PROFILE = os.getenv('FX_PROFILE', '')
-FX_COOKIE_DB = os.path.join(FX_PROFILE, 'cookies.sqlite') if FX_PROFILE else ''

-HEADERS = {
-    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
-    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
-}
-
-last_scrape_time: datetime | None = None
-
-STRICT_KEYWORDS = [
+BROAD_KEYWORDS = [
    "website", "web design", "web develop", "web dev",
    "build my website", "build a website", "create a website",
    "need web", "looking for web", "new website",
@@ -34,9 +30,6 @@ STRICT_KEYWORDS = [
    "need a website", "my website", "business website",
    "need a designer", "help with my website",
    "redesign", "update my website",
-]
-
-BROAD_KEYWORDS = STRICT_KEYWORDS + [
    "looking for", "need a", "need an", "looking to",
    "help me build", "create my", "build me",
    "design my", "make me a", "would like",
@@ -44,10 +37,6 @@ BROAD_KEYWORDS = STRICT_KEYWORDS + [
    "want someone", "need help with",
 ]

-def kw_match_strict(text: str) -> bool:
-    t = text.lower()
-    return any(kw in t for kw in STRICT_KEYWORDS)
-
 def kw_match(text: str) -> bool:
    t = text.lower()
    return any(kw in t for kw in BROAD_KEYWORDS)
@@ -67,14 +56,27 @@ FB_SEARCHES = [
    "need a site for my business",
 ]

-async def get_fb_cookies():
-    if not FX_COOKIE_DB or not os.path.exists(FX_COOKIE_DB):
-        logger.warning("FX_COOKIE_DB not found or FX_PROFILE not set")
+VIEWPORTS = [
+    {'width': 1280, 'height': 800},
+    {'width': 1366, 'height': 768},
+    {'width': 1440, 'height': 900},
+    {'width': 1536, 'height': 864},
+    {'width': 1920, 'height': 1080},
+]
+
+async def get_fb_cookies(profile_path: str | None = None):
+    cookie_db_path = profile_path or FX_PROFILE
+    if not cookie_db_path:
+        logger.warning("No profile path provided")
        return []
-    tmp = os.path.join(os.path.dirname(FX_COOKIE_DB), f'cookies_tmp_{os.getpid()}.sqlite')
+    cookie_db = os.path.join(cookie_db_path, 'cookies.sqlite')
+    if not os.path.exists(cookie_db):
+        logger.warning("Cookie DB not found at %s", cookie_db)
+        return []
+    tmp = os.path.join(os.path.dirname(cookie_db), f'cookies_tmp_{os.getpid()}.sqlite')
    for attempt in range(3):
        try:
-            shutil.copy2(FX_COOKIE_DB, tmp)
+            shutil.copy2(cookie_db, tmp)
            break
        except PermissionError:
            if attempt < 2:
@@ -182,35 +184,95 @@ def _extract_posts_from_text(raw: str, url: str) -> list[dict]:
        })
    return posts

+async def human_scroll(page, steps: int = None, total_delay: float = None):
+    steps = steps or random.randint(2, 5)
+    total_delay = total_delay or random.uniform(6, 18)
+    step_delay = total_delay / steps
+    for i in range(steps):
+        scroll_dist = random.randint(200, 600)
+        await page.evaluate(f"window.scrollBy(0, {scroll_dist})")
+        await page.wait_for_timeout(int(step_delay * 1000))
+    if random.random() < 0.3:
+        await page.evaluate(f"window.scrollBy(0, -{random.randint(100, 300)})")
+        await page.wait_for_timeout(random.randint(1000, 3000))
+
+async def random_idle(page):
+    action = random.random()
+    if action < 0.15:
+        try:
+            elems = await page.query_selector_all('a, span, div[role="button"]')
+            if elems:
+                el = random.choice(elems[:20])
+                box = await el.bounding_box()
+                if box:
+                    x = box['x'] + box['width'] / 2 + random.randint(-20, 20)
+                    y = box['y'] + box['height'] / 2 + random.randint(-10, 10)
+                    await page.mouse.move(x, y, steps=random.randint(20, 40))
+                    await page.wait_for_timeout(random.randint(500, 2000))
+        except Exception:
+            pass
+
 async def search_facebook(page, query: str) -> list[dict]:
    url = f'https://www.facebook.com/search/posts/?q={urllib.parse.quote(query)}'
    try:
        await page.goto(url, wait_until='domcontentloaded', timeout=30000)
-        await page.wait_for_timeout(6000)
-    except Exception as e:
-        logger.warning("Facebook search navigation failed for '%s': %s", query, e)
-        return []
+        await page.wait_for_timeout(random.randint(3000, 8000))
+
+        await human_scroll(page, steps=random.randint(2, 4), total_delay=random.uniform(6, 15))
+
+        # Accidental overscroll: 10% chance to jump to bottom and back
+        if random.random() < 0.1:
+            await page.evaluate("window.scrollTo(0, document.body.scrollHeight)")
+            await page.wait_for_timeout(random.randint(1000, 3000))
+            await page.evaluate("window.scrollBy(0, -300)")
+            await page.wait_for_timeout(random.randint(1000, 3000))
+
+        if random.random() < 0.2:
+            await page.evaluate("window.scrollTo(0, 0)")
+            await page.wait_for_timeout(random.randint(2000, 5000))
+
+        if random.random() < 0.3:
+            await random_idle(page)

-    try:
        raw = await page.evaluate('document.body.innerText')
    except Exception as e:
-        logger.warning("Failed to evaluate page text: %s", e)
+        logger.warning("Facebook search failed: %s", e)
        return []
    return _extract_posts_from_text(raw, url)

-async def scrape_facebook() -> list[dict]:
-    all_posts = []
-    fb_cookies = await get_fb_cookies()
+def cleanup_chrome():
+    import subprocess, signal
+    try:
+        subprocess.run(["taskkill", "/F", "/IM", "chrome-headless-shell.exe"], capture_output=True, timeout=5)
+    except Exception:
+        pass
+
+async def scrape_facebook(profile_path: str | None = None, force: bool = False) -> dict:
+    cleanup_chrome()
+    fb_cookies = await get_fb_cookies(profile_path)
    if not fb_cookies:
        logger.warning("No Facebook cookies available")
-        return []
+        return {"success": False, "leads": [], "flagged": False, "flag_reason": None, "error": "No cookies available"}
    try:
        async with async_playwright() as pw:
-            browser = await pw.chromium.launch(headless=True)
+            browser = await pw.chromium.launch(
+                headless=True,
+                args=['--disable-blink-features=AutomationControlled']
+            )
+            viewport = random.choice(VIEWPORTS)
            context = await browser.new_context(
                user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:150.0) Gecko/20100101 Firefox/150.0',
-                viewport={'width': 1280, 'height': 800},
+                viewport=viewport,
            )
+
+            await context.add_init_script("""
+                Object.defineProperty(navigator, 'webdriver', { get: () => false });
+                Object.defineProperty(navigator, 'languages', { get: () => ['en-US', 'en'] });
+                Object.defineProperty(navigator, 'platform', { get: () => 'Win32' });
+                window.chrome = { runtime: {} };
+                Object.defineProperty(navigator, 'hardwareConcurrency', { get: () => 8 });
+                Object.defineProperty(navigator, 'deviceMemory', { get: () => 8 });
+            """)
            for c in fb_cookies:
                try:
                    await context.add_cookies([c])
@@ -218,36 +280,77 @@ async def scrape_facebook() -> list[dict]:
                    logger.warning("Failed to inject cookie %s: %s", c.get('name'), e)

            page = await context.new_page()
+            # Navigate through google first for a legitimate Referer header
+            await page.goto('https://www.google.com/', wait_until='domcontentloaded', timeout=15000)
+            await page.wait_for_timeout(random.randint(1000, 3000))
            await page.goto('https://www.facebook.com/', wait_until='domcontentloaded', timeout=30000)
-            await page.wait_for_timeout(5000)
+            await page.wait_for_timeout(random.randint(3000, 8000))

            url = page.url
            if '/login' in url.lower():
-                logger.warning("Facebook login page detected — cookies may be expired")
-                return []
+                logger.warning("Facebook login page detected — account flagged")
+                await browser.close()
+                return {"success": False, "leads": [], "flagged": True, "flag_reason": "login_page", "error": "Facebook login page detected"}

-            for query in FB_SEARCHES[:6]:
+            # Browse feed like a human before searching
+            await human_scroll(page, steps=random.randint(2, 4), total_delay=random.uniform(8, 20))
+            if random.random() < 0.25:
+                await page.evaluate("window.scrollTo(0, 0)")
+                await page.wait_for_timeout(random.randint(2000, 5000))
+                await human_scroll(page, steps=random.randint(1, 2))
+
+            # False start: 30% chance to idle and leave without scraping (skipped when force=true)
+            if not force and random.random() < 0.3:
+                await page.wait_for_timeout(random.randint(8000, 20000))
+                await browser.close()
+                return {"success": True, "leads": [], "flagged": False, "flag_reason": None, "error": None}
+
+            all_posts = []
+            searches = random.sample(FB_SEARCHES, k=random.randint(3, 6))
+            for i, query in enumerate(searches):
                try:
                    posts = await search_facebook(page, query)
                    all_posts.extend(posts)
-                    delay = random.uniform(3, 7)
+                    # Between searches: random break with possible small scroll
+                    if random.random() < 0.4:
+                        await page.evaluate(f"window.scrollBy(0, {random.randint(-300, 300)})")
+                    delay = random.uniform(8, 25)
                    await page.wait_for_timeout(int(delay * 1000))
+                    # Tab switch: 15% chance after 2nd-3rd search
+                    if i == random.randint(1, 2) and random.random() < 0.15:
+                        new_page = await context.new_page()
+                        await new_page.goto('https://www.messenger.com/', wait_until='domcontentloaded', timeout=15000)
+                        await new_page.wait_for_timeout(random.randint(3000, 8000))
+                        await new_page.close()
+                        await page.wait_for_timeout(random.randint(1000, 3000))
                except Exception as e:
                    logger.warning("Facebook search '%s' failed: %s", query, e)
+
+            # Idle before closing — human would pause
+            if random.random() < 0.5:
+                await page.wait_for_timeout(random.randint(3000, 10000))
+
+            await browser.close()
+
+            seen = set()
+            deduped = []
+            for p in all_posts:
+                key = p.get('content', '')[:100]
+                if key not in seen:
+                    seen.add(key)
+                    deduped.append(p)
+
+            leads = deduped[:20]
+            if leads:
+                leads = await classify_leads(leads)
+
+            return {"success": True, "leads": leads[:15], "flagged": False, "flag_reason": None, "error": None}
    except Exception as e:
        logger.error("Facebook scrape failed: %s", e)
-        return []
-
-    seen = set()
-    deduped = []
-    for p in all_posts:
-        key = p.get('content', '')[:100]
-        if key not in seen:
-            seen.add(key)
-            deduped.append(p)
-    return deduped[:20]
+        return {"success": False, "leads": [], "flagged": False, "flag_reason": None, "error": str(e)}

 async def ask_ollama(prompt: str) -> str:
+    import httpx
    async with httpx.AsyncClient(timeout=120) as c:
        r = await c.post(f"{OLLAMA_URL}/api/chat", json={
            "model": CLASSIFY_MODEL,
@@ -262,42 +365,6 @@ async def ask_ollama(prompt: str) -> str:
        data = r.json()
        return data["message"]["content"]

-async def scrape_warriorforum() -> list[dict]:
-    results = []
-    try:
-        async with httpx.AsyncClient(headers=HEADERS, timeout=15, follow_redirects=True) as c:
-            r = await c.get('https://www.warriorforum.com/wanted-members-looking-hire-you/')
-            soup = BeautifulSoup(r.text, 'lxml')
-            for row in soup.find_all('td', class_='FlexTable-item--title'):
-                a = row.find('a', href=True)
-                if not a:
-                    continue
-                href = a['href']
-                title = a.text.strip()
-                if not title or len(title) < 15 or not kw_match_strict(title):
-                    continue
-                author_div = row.find('div', class_='FlexTable-item-author')
-                author = author_div.get_text(strip=True).lstrip('by') if author_div else ''
-                date_str = ''
-                date_div = row.find('div', class_=lambda c: c and 'media--available' in c)
-                if date_div:
-                    txt = date_div.get_text(strip=True)
-                    m = re.search(r'(\d{10})', txt)
-                    if m:
-                        date_str = datetime.utcfromtimestamp(int(m.group(1))).strftime('%Y-%m-%d')
-                if not href.startswith('http'):
-                    href = 'https://www.warriorforum.com' + href
-                results.append({
-                    "title": title, "url": href,
-                    "author": author,
-                    "content": title[:300],
-                    "source": "warriorforum",
-                    "date": date_str
-                })
-    except Exception:
-        logger.error("WarriorForum scrape failed", exc_info=True)
-    return results
-
 async def classify_leads(results: list[dict]) -> list[dict]:
    if not results:
        return []
@@ -320,13 +387,10 @@ Return a JSON array like ["yes","no","yes"] matching the order above."""
    try:
        raw = await ask_ollama(prompt)
        raw = raw.strip()
-        # Strip markdown code fences
        if raw.startswith("```"):
-            # Remove opening fence
            first_nl = raw.find('\n')
            if first_nl != -1:
                raw = raw[first_nl + 1:]
-            # Remove closing fence
            if raw.endswith("```"):
                raw = raw[:-3]
            raw = raw.strip()
@@ -339,18 +403,25 @@ Return a JSON array like ["yes","no","yes"] matching the order above."""
                    filtered.append(results[i])
            if filtered:
                return filtered[:10]
-            # AI successfully classified but returned all "no" — respect that decision
            logger.info("AI classified all %d items as NOT LEAD — returning empty", len(results))
            return []
    except Exception as e:
        logger.warning("AI classification failed, falling back to keyword filter: %s", e)

-    # Fallback: only use keyword filter when AI failed
    if not ai_succeeded:
+        strict_keywords = [
+            "website", "web design", "web develop", "web dev",
+            "build my website", "build a website", "create a website",
+            "need web", "looking for web", "new website",
+            "landing page", "wordpress",
+            "need a website", "my website", "business website",
+            "need a designer", "help with my website",
+            "redesign", "update my website",
+        ]
        filtered = []
        for r in results:
            t = r['title'].lower()
-            if not kw_match_strict(t):
+            if not any(kw in t for kw in strict_keywords):
                continue
            if any(kw in t for kw in ['i build', 'i offer', 'i create', 'i am a', 'web developer available',
                                       'affordable web', 'web hosting', 'free website',
@@ -365,46 +436,10 @@ Return a JSON array like ["yes","no","yes"] matching the order above."""
 async def health():
    return {"status": "ok"}

-class ScrapeRequest(BaseModel):
-    query: str = ""
-
-@app.post("/scrape/requests")
-async def scrape_requests(req: ScrapeRequest):
-    global last_scrape_time
-    now = datetime.now()
-    if last_scrape_time and (now - last_scrape_time) < timedelta(seconds=15):
-        return {"error": "rate_limited", "retry_after": 15}
-    last_scrape_time = now
-    results = await scrape_warriorforum()
-    if results:
-        results = await classify_leads(results)
-    return results[:10]
-
@app.post("/scrape/facebook")
-async def scrape_facebook_endpoint():
-    results = await scrape_facebook()
-    if results:
-        results = await classify_leads(results)
-    return results[:15]
-
-@app.post("/scrape/all")
-async def scrape_all():
-    results = []
-    try:
-        wf = await scrape_warriorforum()
-        if wf:
-            wf = await classify_leads(wf)
-            results.extend(wf[:5])
-    except Exception as e:
-        logger.error("WarriorForum scrape in /scrape/all failed: %s", e)
-    try:
-        fb = await scrape_facebook()
-        if fb:
-            fb = await classify_leads(fb)
-            results.extend(fb[:8])
-    except Exception as e:
-        logger.error("Facebook scrape in /scrape/all failed: %s", e)
-    return results[:12]
+async def scrape_facebook_endpoint(profile_path: str | None = Query(None), force: bool = Query(False)):
+    result = await scrape_facebook(profile_path, force)
+    return result

 if __name__ == "__main__":
    uvicorn.run(app, host="0.0.0.0", port=PORT)
@@ -1,7 +0,0 @@
-python : INFO:     Started server process [20044]
-    + CategoryInfo          : NotSpecified: (INFO:     Started server process [20044]:String) [], RemoteException
-    + FullyQualifiedErrorId : NativeCommandError
- 
-INFO:     Waiting for application startup.
-INFO:     Application startup complete.
-INFO:     Uvicorn running on http://0.0.0.0:3008 (Press CTRL+C to quit)
@@ -1,6 +0,0 @@
-[2026-06-22T19:24:48.030482] Browser Use service starting on port 3008
-[2026-06-22T19:27:19.231701] Browser Use service starting on port 3008
-[2026-06-22T19:28:28.335765] Browser Use service starting on port 3008
-[2026-06-22T19:29:05.796265] Browser Use service starting on port 3008
-[2026-06-22T19:29:17.042807] Scraping WarriorForum...
-[2026-06-22T19:29:19.075166] WarriorForum: 55 results
@@ -1,4 +0,0 @@
-INFO:     Started server process [24268]
-INFO:     Waiting for application startup.
-INFO:     Application startup complete.
-INFO:     Uvicorn running on http://0.0.0.0:3008 (Press CTRL+C to quit)
@@ -1,5 +0,0 @@
-Browser Use service starting on port 3008
-INFO:     127.0.0.1:65003 - "GET /health HTTP/1.1" 200 OK
-Scraping WarriorForum...
-WarriorForum: 55 results
-INFO:     127.0.0.1:54587 - "POST /scrape/all HTTP/1.1" 200 OK
@@ -126,10 +126,10 @@ separate 1:N child tables.

 | Username | Password | Role |
 |----------|----------|------|
-| `superadmin_demo` | `SuperAdmin@2026` | SUPER_ADMIN |
-| `admin_demo` | `AdminAccess@2026` | ADMIN |
-| `sales_demo` | `SalesAccess@2026` | SALES_USER |
-| `dev_demo` | `DevTesting@2026` | DEVELOPER |
+| `superadmin_demo` | `[REDACTED]` | SUPER_ADMIN |
+| `admin_demo` | `[REDACTED]` | ADMIN |
+| `sales_demo` | `[REDACTED]` | SALES_USER |
+| `dev_demo` | `[REDACTED]` | DEVELOPER |

 ---

@@ -0,0 +1,34 @@
+CREATE TABLE IF NOT EXISTS facebook_accounts (
+  id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
+  label VARCHAR(100) NOT NULL,
+  profile_path TEXT NOT NULL,
+  cookie_file TEXT NOT NULL,
+  is_active BOOLEAN NOT NULL DEFAULT TRUE,
+  last_scrape_at TIMESTAMPTZ,
+  last_success_at TIMESTAMPTZ,
+  last_error_at TIMESTAMPTZ,
+  last_error_message TEXT,
+  consecutive_failures INT NOT NULL DEFAULT 0,
+  flagged BOOLEAN NOT NULL DEFAULT FALSE,
+  flagged_at TIMESTAMPTZ,
+  flagged_reason TEXT,
+  created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+  updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
+);
+
+CREATE INDEX IF NOT EXISTS idx_fb_accounts_active ON facebook_accounts(is_active);
+CREATE INDEX IF NOT EXISTS idx_fb_accounts_flagged ON facebook_accounts(flagged);
+
+CREATE TABLE IF NOT EXISTS facebook_scrape_logs (
+  id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
+  account_id UUID NOT NULL REFERENCES facebook_accounts(id) ON DELETE CASCADE,
+  started_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+  completed_at TIMESTAMPTZ,
+  success BOOLEAN NOT NULL DEFAULT FALSE,
+  leads_found INT NOT NULL DEFAULT 0,
+  error_message TEXT,
+  detected_flag VARCHAR(50),
+  created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
+);
+
+CREATE INDEX IF NOT EXISTS idx_fb_scrape_logs_account ON facebook_scrape_logs(account_id, created_at DESC);
@@ -34,8 +34,5 @@ BEGIN;
 \echo '=== Running 009_settings.sql (Company Settings + User Preferences) ==='
 \i 009_settings.sql

-\echo '=== Running 010_chat_notifications.sql (Chat Message Notifications) ==='
-\i 010_chat_notifications.sql
-
 \echo '=== Migration Complete ==='
 COMMIT;
@@ -15,6 +15,7 @@ use tokio::sync::Mutex;
 use tracing::{error, info, warn};
 use uuid::Uuid;
 use rand::Rng;
+use chrono::Timelike;
 use std::time::Duration;
 use std::time::{SystemTime, UNIX_EPOCH};

@@ -93,6 +94,25 @@ struct LeadStore {
    max_size: usize,
 }

+#[derive(Debug, Deserialize)]
+struct ScrapeResponse {
+    success: bool,
+    leads: Vec<ScrapeLead>,
+    flagged: bool,
+    flag_reason: Option<String>,
+    error: Option<String>,
+}
+
+#[derive(Debug, Deserialize, Clone)]
+struct ScrapeLead {
+    title: String,
+    url: String,
+    author: String,
+    date: String,
+    content: String,
+    source: Option<String>,
+}
+
 impl LeadStore {
    fn new(max_size: usize) -> Self {
        Self { leads: Vec::new(), max_size }
@@ -188,7 +208,7 @@ fn extract_claims(headers: &HeaderMap, state: &AppState) -> Result<Claims, (Stat
    let claims = verify_jwt(token, &state.jwt_secret).ok_or_else(|| {
        (StatusCode::UNAUTHORIZED, "Unauthorized".to_string())
    })?;
-    match claims.role.as_str() {
+    match claims.role.to_lowercase().as_str() {
        "sales" | "admin" | "super_admin" => Ok(claims),
        _ => Err((StatusCode::FORBIDDEN, "Forbidden".to_string())),
    }
@@ -266,30 +286,64 @@ async fn handle_chat(
    let has_job = msg_words.contains(&"jobs") || msg_words.contains(&"job");
    if has_listing || (has_show && has_job) || (has_show && msg_lower.contains("links")) || msg_lower.contains("recent leads") {
        let now = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs();
-        let service_url = "http://localhost:3008/scrape/all".to_string();
-        if let Ok(resp) = state.http_client
-            .post(&service_url)
-            .timeout(Duration::from_secs(120))
-            .send()
-            .await
+        let base_url = "http://localhost:3008/scrape/facebook";
+        use std::fmt::Write;
+        let mut service_url = base_url.to_string();
+        if let Ok(Some((_, path))) = sqlx::query_as::<_, (uuid::Uuid, String)>(
+            "SELECT id, profile_path FROM facebook_accounts \
+             WHERE is_active = TRUE AND flagged = FALSE \
+             ORDER BY last_scrape_at ASC NULLS FIRST LIMIT 1"
+        )
+        .fetch_optional(&state.db)
+        .await
        {
-            if let Ok(leads_data) = resp.json::<Vec<serde_json::Value>>().await {
-                info!("Scraped {} leads from {}", leads_data.len(), service_url);
-                let mut store = state.leads.lock().await;
-                for item in &leads_data {
-                    store.push(Lead {
-                        title: truncate(item["title"].as_str().unwrap_or(""), 120),
-                        url: item["url"].as_str().unwrap_or("").to_string(),
-                        source: item["source"].as_str().unwrap_or("unknown").to_string(),
-                        found_at: now,
-                        author: truncate(item["author"].as_str().unwrap_or(""), 60),
-                        date: truncate(item["date"].as_str().unwrap_or(""), 30),
-                        content: truncate(item["content"].as_str().unwrap_or(""), 300),
-                    });
+            let encoded: String = path.chars().map(|c| match c {
+                'A'..='Z' | 'a'..='z' | '0'..='9' | '.' | '-' | '_' | '~' => c.to_string(),
+                _ => format!("%{:02X}", c as u8),
+            }).collect();
+            write!(service_url, "?profile_path={}&force=true", encoded).unwrap();
+            info!("Calling Python scrape at: {}?profile_path=...&force=true", base_url);
+        } else {
+            warn!("No active Facebook account found for on-demand scrape");
+        }
+        let req_builder = state.http_client.post(&service_url);
+
+        match req_builder.send().await {
+            Ok(resp) => {
+                let status = resp.status();
+                let body = resp.text().await.unwrap_or_default();
+                info!("Python scrape response ({}): {} bytes", status, body.len());
+                if body.starts_with('{') {
+                    match serde_json::from_str::<ScrapeResponse>(&body) {
+                        Ok(scrape_resp) => {
+                            info!("Scraped {} leads from Facebook", scrape_resp.leads.len());
+                            if scrape_resp.leads.is_empty() && scrape_resp.error.is_some() {
+                                warn!("Python returned error: {:?}", scrape_resp.error);
+                            }
+                            let mut store = state.leads.lock().await;
+                            for item in &scrape_resp.leads {
+                                store.push(Lead {
+                                    title: truncate(&item.title, 120),
+                                    url: item.url.clone(),
+                                    source: item.source.clone().unwrap_or_else(|| "facebook".to_string()),
+                                    found_at: now,
+                                    author: truncate(&item.author, 60),
+                                    date: truncate(&item.date, 30),
+                                    content: truncate(&item.content, 300),
+                                });
+                            }
+                        }
+                        Err(e) => {
+                            warn!("Failed to parse Python scrape response: {} body: {}", e, &body[..body.len().min(200)]);
+                        }
+                    }
+                } else {
+                    warn!("Python returned non-JSON response: {}", &body[..body.len().min(200)]);
                }
            }
-        } else {
-            error!("Scraper service unreachable: {}", service_url);
+            Err(e) => {
+                error!("Scraper request error: {} - URL: {}", e, base_url);
+            }
        }

        let recent_leads = state.leads.lock().await.recent(604800, 20);
@@ -411,7 +465,7 @@ async fn main() {
    info!("Connected to PostgreSQL");

    let http_client = reqwest::Client::builder()
-        .timeout(Duration::from_secs(120))
+        .timeout(Duration::from_secs(300))
        .build()
        .expect("Failed to build HTTP client");

@@ -441,7 +495,7 @@ async fn main() {
        .route("/ai/jobs", get(handle_jobs))
        .route("/health", get(handle_health))
        .layer(cors)
-        .with_state(state);
+        .with_state(state.clone());

    let addr = format!("{}:{}", host, port);
    info!("CRM AI server listening on {}", addr);
@@ -451,10 +505,11 @@ async fn main() {
        .expect("Failed to bind address");

    let bg_leads = lead_store.clone();
-    let bg_url = "http://localhost:3008/scrape/all".to_string();
+    let bg_db = state.db.clone();
+    let bg_url = "http://localhost:3008/scrape/facebook".to_string();
    tokio::spawn(async move {
        let client = match reqwest::Client::builder()
-            .timeout(Duration::from_secs(120))
+            .timeout(Duration::from_secs(300))
            .build()
        {
            Ok(c) => c,
@@ -463,40 +518,149 @@ async fn main() {
                return;
            }
        };
+        // Initial delay to let user on-demand requests take priority
+        tokio::time::sleep(Duration::from_secs(300)).await;
+
        loop {
+            // 10% random cycle skip — makes pattern non-periodic
+            if rand::thread_rng().gen_range(0..100) < 10 {
+                info!("Skipping this scrape cycle (random 10% skip)");
+                let jitter = rand::thread_rng().gen_range(16200..19800);
+                tokio::time::sleep(Duration::from_secs(jitter)).await;
+                continue;
+            }
+
+            // Skip night hours (23:00 – 06:00)
+            let hour = chrono::Local::now().hour();
+            if hour < 6 || hour >= 23 {
+                info!("Night hours ({}) — skipping scrape", hour);
+                let jitter = rand::thread_rng().gen_range(16200..19800);
+                tokio::time::sleep(Duration::from_secs(jitter)).await;
+                continue;
+            }
+
            let now = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs();
-            match client.post(&bg_url).send().await {
-                Ok(resp) => {
-                    if resp.status().is_success() {
-                        match resp.json::<Vec<serde_json::Value>>().await {
-                            Ok(data) => {
-                                let mut store = bg_leads.lock().await;
-                                for item in &data {
-                                    store.push(Lead {
-                                        title: truncate(item["title"].as_str().unwrap_or(""), 120),
-                                        url: item["url"].as_str().unwrap_or("").to_string(),
-                                        source: item["source"].as_str().unwrap_or("unknown").to_string(),
-                                        found_at: now,
-                                        author: truncate(item["author"].as_str().unwrap_or(""), 60),
-                                        date: truncate(item["date"].as_str().unwrap_or(""), 30),
-                                        content: truncate(item["content"].as_str().unwrap_or(""), 300),
-                                    });
+
+            // Pick next active un-flagged account
+            let account = sqlx::query_as::<_, (uuid::Uuid, String)>(
+                "SELECT id, profile_path FROM facebook_accounts \
+                 WHERE is_active = TRUE AND flagged = FALSE \
+                 ORDER BY last_scrape_at ASC NULLS FIRST LIMIT 1"
+            )
+            .fetch_optional(&bg_db)
+            .await;
+
+            match account {
+                Ok(Some((account_id, profile_path))) => {
+                    match client.post(&bg_url).query(&[("profile_path", &profile_path)]).send().await {
+                        Ok(resp) => {
+                            if resp.status().is_success() {
+                                match resp.json::<ScrapeResponse>().await {
+                                    Ok(data) => {
+                                        let leads_count = data.leads.len() as i32;
+                                        if data.flagged {
+                                            let _ = sqlx::query(
+                                                "UPDATE facebook_accounts SET flagged = TRUE, flagged_at = NOW(), \
+                                                 flagged_reason = $2, last_error_at = NOW(), \
+                                                 last_error_message = $3, consecutive_failures = consecutive_failures + 1 \
+                                                 WHERE id = $1"
+                                            )
+                                            .bind(account_id)
+                                            .bind(&data.flag_reason)
+                                            .bind(&data.error)
+                                            .execute(&bg_db)
+                                            .await;
+                                            warn!("Facebook account {} flagged: {:?}", account_id, data.flag_reason);
+                                            let reason = data.flag_reason.as_deref().unwrap_or("unknown");
+                                            let _ = sqlx::query(
+                                                "INSERT INTO notifications (user_id, type, title, description, link) \
+                                                 SELECT id, 'warning', 'Facebook Account Flagged', \
+                                                 $1 || ' - ' || COALESCE($2, 'unknown reason'), \
+                                                 NULL \
+                                                 FROM users u JOIN user_roles ur ON ur.user_id = u.id \
+                                                 JOIN roles r ON r.id = ur.role_id \
+                                                 WHERE r.name IN ('ADMIN', 'SUPER_ADMIN')"
+                                            )
+                                            .bind(&account_id.to_string())
+                                            .bind(reason)
+                                            .execute(&bg_db)
+                                            .await;
+                                        } else if data.success {
+                                            let _ = sqlx::query(
+                                                "UPDATE facebook_accounts SET last_scrape_at = NOW(), \
+                                                 last_success_at = NOW(), consecutive_failures = 0, \
+                                                 updated_at = NOW() WHERE id = $1"
+                                            )
+                                            .bind(account_id)
+                                            .execute(&bg_db)
+                                            .await;
+
+                                            let mut store = bg_leads.lock().await;
+                                            for item in &data.leads {
+                                                store.push(Lead {
+                                                    title: truncate(&item.title, 120),
+                                                    url: item.url.clone(),
+                                                    source: item.source.clone().unwrap_or_else(|| "facebook".to_string()),
+                                                    found_at: now,
+                                                    author: truncate(&item.author, 60),
+                                                    date: truncate(&item.date, 30),
+                                                    content: truncate(&item.content, 300),
+                                                });
+                                            }
+                                            info!("Scraped {} leads from Facebook account {}", leads_count, account_id);
+                                        } else {
+                                            // Increment failures; auto-flag if >= 3 consecutive
+                                            let _ = sqlx::query(
+                                                "UPDATE facebook_accounts SET last_error_at = NOW(), \
+                                                 last_error_message = $2, consecutive_failures = consecutive_failures + 1, \
+                                                 flagged = CASE WHEN consecutive_failures + 1 >= 3 THEN TRUE ELSE flagged END, \
+                                                 flagged_reason = CASE WHEN consecutive_failures + 1 >= 3 THEN 'too_many_failures' ELSE flagged_reason END, \
+                                                 flagged_at = CASE WHEN consecutive_failures + 1 >= 3 THEN NOW() ELSE flagged_at END, \
+                                                 updated_at = NOW() WHERE id = $1"
+                                            )
+                                            .bind(account_id)
+                                            .bind(&data.error)
+                                            .execute(&bg_db)
+                                            .await;
+                                            warn!("Facebook scrape failed for account {}: {:?}", account_id, data.error);
+                                        }
+
+                                        let _ = sqlx::query(
+                                            "INSERT INTO facebook_scrape_logs \
+                                             (account_id, started_at, completed_at, success, leads_found, error_message, detected_flag) \
+                                             VALUES ($1, NOW() - interval '5 hours', NOW(), $2, $3, $4, $5)"
+                                        )
+                                        .bind(account_id)
+                                        .bind(data.success && !data.flagged)
+                                        .bind(leads_count)
+                                        .bind(&data.error)
+                                        .bind(&data.flag_reason)
+                                        .execute(&bg_db)
+                                        .await;
+                                    }
+                                    Err(e) => {
+                                        warn!("Failed to parse scraper JSON: {}", e);
+                                    }
                                }
-                            }
-                            Err(e) => {
-                                warn!("Failed to parse scraper JSON: {}", e);
+                            } else {
+                                warn!("Scraper returned status: {}", resp.status());
                            }
                        }
-                    } else {
-                        warn!("Scraper returned status: {}", resp.status());
+                        Err(e) => {
+                            warn!("Scraper request failed: {}", e);
+                        }
                    }
                }
+                Ok(None) => {
+                    info!("No active Facebook accounts available — skipping scrape cycle");
+                }
                Err(e) => {
-                    warn!("Scraper request failed: {}", e);
+                    warn!("Failed to query Facebook accounts: {}", e);
                }
            }
-            let delay = rand::thread_rng().gen_range(120..300);
-            tokio::time::sleep(Duration::from_secs(delay)).await;
+
+            let jitter = rand::thread_rng().gen_range(16200..19800); // 4.5h – 5.5h
+            tokio::time::sleep(Duration::from_secs(jitter)).await;
        }
    });

--- a/Show More
+++ b/Show More