Merge branch 'main' of https://git.coastit.co.za/caitlin/CRM_ENVR

2026-07-10 03:05:43 +02:00 · 2026-07-01 13:14:35 +02:00
parent bc422edcf7 da5f8360bc
commit 38fb3a47a8
10 changed files with 637 additions and 112 deletions
@@ -341,6 +341,7 @@ OFFER_PATTERNS = [
    r"\bfree\s+domain\b",
    r"\bweb\s+hosting\b",
    r"\bfree\s+website\b",
+    r"\bfree\s+\w+\s+website\b",
    r"\bprofessional\s+website",
    r"\baffordable\s+web\b",
    r"\bget\s+a\s+free\b",
@@ -365,6 +366,47 @@ OFFER_PATTERNS = [
    r"\bsponsored\b",
    r"\bpromoted\b",
    r"\badvertisement\b",
+    r"\bdo\s+you\s+(need|want)\b",
+    r"\bwe\s+(can|will)\s+(build|design|create|develop|do)\b",
+    r"\bi\s+(can|will)\s+(build|design|create|develop|do)\b",
+    r"\bmy\s+portfolio\b",
+    r"\breasonable\s+(price|pricing|rate)\b",
+    r"\bwhatsapp\s+(me|us|at)\b",
+    r"\blooking\s+for\s+(a\s+)?(business|client|customer)\b",
+    r"\bhelp\s+your\s+business\b",
+    r"\bi\s+am\s+a\s+(web|freelance|designer|developer)\b",
+    r"\bcontact\s+me\b",
+    r"\bwe\s+(are|offer|provide)\s+(web|website|design|service)\b",
+    r"\btake\s+(the\s+|this\s+)?quiz\b",
+    r"\bhomeschool\b",
+    r"\byour\s+(home\s+)?tutor\b",
+    r"\blink\s+(in\s+)?(bio|comment|below)\b",
+    r"\bapply\s+now\b",
+    r"\bget\s+started\b",
+    r"\bfor\s+only\b",
+    r"\blow\s+(price|cost|rate)\b",
+    r"\bhit\s+me\s+up\b",
+    r"\bsend\s+me\s+(a\s+)?(message|dm|pm)\b",
+    r"\bi\s+do\s+(web|website|design|development)\b",
+    r"\bwe\s+do\s+(web|website|design|development)\b",
+    r"\bi'm\s+offering\b",
+    r"\bwe're\s+offering\b",
+    r"\bstarting\s+a\s+\w+\s+(service|business)\b",
+    r"\blooking\s+for\s+a\s+few\s+businesses?\b",
+    r"\blet('?)s\s+connect\b",
+    r"\bi\s+have\s+projects?\b",
+    r"\byour\s+business\b",
+    r"\bwordpress\s+development\b",
+    r"\be.?commerce\s+website\b",
+    r"\bwebsite\s+builder\b",
+    r"\bai\s+studio\b",
+    r"\bpage\s+speed\b",
+    r"\bguest\s+post",
+    r"\bguest\s+blog",
+    r"\bfor\s+sale\b",
+    r"\bselling\s+(my|a|the|this)\b",
+    r"\bpremium\b",
+    r"\bi'm\s+selling\b",
 ]

 REQUEST_PATTERNS = [
@@ -549,6 +591,48 @@ FB_SEARCHES = [
    "want to build a website for my",
 ]

+TUTORING_SEARCHES = [
+    "need a tutor",
+    "looking for a tutor",
+    "tutor for my child",
+    "need help with homework",
+    "private tutor needed",
+    "looking for tutoring",
+    "need a math tutor",
+    "english tutor needed",
+    "online tutor for",
+    "lessons for my child",
+    "help my child with",
+    "need someone to tutor",
+    "looking for someone to teach",
+    "tutoring for my",
+    "need help learning",
+    "recommend a tutor",
+    "anyone know a tutor",
+    "need a private tutor",
+    "tutoring services for my child",
+    "need academic help",
+    "need a reading tutor",
+    "looking for piano teacher",
+    "need help with maths",
+    "need a science tutor",
+    "looking for language tutor",
+    "need programming tutor",
+    "coding tutor for my",
+    "sat tutor needed",
+    "exam preparation tutor",
+    "need a homeschool tutor",
+    "tutor near me for",
+]
+
+def _search_list_for_query(query: str) -> list[str]:
+    """Pick the appropriate search query pool based on the search term."""
+    tl = query.lower()
+    tutoring_terms = ["tutor", "tutoring", "lessons", "homework", "teach", "learning", "child", "math", "english", "science", "exam", "homeschool", "coding", "programming", "piano", "reading"]
+    if any(t in tl for t in tutoring_terms):
+        return TUTORING_SEARCHES
+    return FB_SEARCHES
+
 VIEWPORTS = [
    {'width': 1280, 'height': 800},
    {'width': 1366, 'height': 768},
@@ -663,6 +747,9 @@ def _clean_fb_text(text: str) -> str:
        cleaned_lines.append(stripped)
    return '\n'.join(cleaned_lines)

+# Words that should never be treated as person names
+_NON_NAMES = {"online","tutor","tutoring","school","academy","learning","urgently","looking","south","africa","philippines","australia","creative","professional","digital","services","solutions","statistics","actuarial","homeschooling","homeschool","reading","math","english","science","grade","group","page","website","design","development","agency","company","studio","graphic","technical","support","customer","guest","post","fashion","wordpress","shopify","ecommerce","business","marketing","social","media","content","strategy","seo","free","domain","hosting","sponsored","promoted","advertisement","need","want","help","please","contact","send","message","whatsapp","hire","freelance","writer","blogger","expert","specialist","consultant","freelancer","software","creators","village","town","city","university","college","institute","evolve","aesthetics","sale","selling","premium","builder","pro","people","ecommerce"}
+
 # ── Post Extraction ──────────────────────────────────────────────────
 # Two strategies for extracting posts from page content:
 #   1. _extract_posts_from_elements — uses structured DOM data from _get_article_elements()
@@ -720,6 +807,25 @@ def _extract_posts_from_text(raw: str, url: str) -> list[dict]:
    posts = []
    seen_texts = set()
    cur = []
+
+    def _find_author(candidate_lines: list[str]) -> str:
+        for line in reversed(candidate_lines):
+            ln = line.strip()
+            if not ln or len(ln) > 60 or len(ln) < 3:
+                continue
+            words = ln.split()
+            if not (1 < len(words) < 8):
+                continue
+            if not all(w[0].isalpha() and w[0].isupper() for w in words if w):
+                continue
+            lower_ln = ln.lower()
+            if any(kw in lower_ln for kw in BROAD_KEYWORDS) or is_offer(ln):
+                continue
+            if 'http' in ln or '/' in ln or '@' in ln:
+                continue
+            return ln
+        return ''
+
    for l in lines:
        words = re.findall(r'[A-Za-z]{2,}', l)
        if len(words) < 2:
@@ -735,7 +841,7 @@ def _extract_posts_from_text(raw: str, url: str) -> list[dict]:
                        posts.append({
                            "title": snippet[:300],
                            "content": snippet[:1000],
-                            "author": '',
+                            "author": _find_author(cur[-3:]),
                            "url": url,
                            "source": "facebook",
                            "date": _parse_fb_date(cur + [l]),
@@ -759,7 +865,40 @@ def _extract_posts_from_text(raw: str, url: str) -> list[dict]:
        cur.append(l)
        if len(cur) > 10:
            cur.pop(0)
-    return posts
+    # Post-process: scan each post's text for inline names
+    def _scan_inline_name(text: str) -> str:
+        for m in re.finditer(r'([A-Z][a-z]{2,}(?:\s+[A-Z][a-z]{2,}){1,3})[\s,;:]', text):
+            cand = m.group(1).strip()
+            if cand and 3 <= len(cand) <= 60:
+                lower = cand.lower()
+                words = lower.split()
+                if not any(kw in lower for kw in BROAD_KEYWORDS) and not is_offer(cand):
+                    if not any(w in _NON_NAMES for w in words):
+                        return cand
+        return ''
+    for p in posts:
+        if not p.get('author'):
+            txt = p.get('content') or p.get('title') or ''
+            name = _scan_inline_name(txt)
+            if name:
+                p['author'] = name
+    # Remove group posts: any post whose text looks like it came from a group
+    _group_suffixes = [" - South Africa", " - Australia", " PH", "Group", "help group", "info group"]
+    filtered = []
+    for p in posts:
+        t = (p.get('title') or p.get('content') or '')
+        if ' / ' in t:
+            continue
+        lower = t.lower()
+        skip = False
+        for suff in _group_suffixes:
+            if suff.lower() in lower:
+                skip = True
+                break
+        if skip:
+            continue
+        filtered.append(p)
+    return filtered

 # ── Human-like Behavior Simulation ──────────────────────────────────
 # These functions add random delays, mouse movements, and scroll patterns
@@ -811,58 +950,31 @@ async def _get_article_elements(page) -> list[dict]:
    return await page.evaluate('''() => {
        const results = [];
        const seenTexts = new Set();
-        const selectors = [
-            'div[role="article"]',
-            'div[role="feed"] > div',
-            'div.x1yztbdb',
-            'div[data-pagelet]',
-        ];
-        for (const sel of selectors) {
-            const els = document.querySelectorAll(sel);
-            for (const el of els) {
-                const text = (el.innerText || '').trim();
-                if (text.length < 40) continue;
-                const key = text.substring(0, 80);
+        const terms = ["website","web design","web develop","need a","looking for","build my","create a","wordpress","landing page","ecommerce"];
+        const anchors = document.querySelectorAll('a[href*="/posts/"], a[href*="/story.php"], a[href*="/photo.php"], a[href*="/watch"], a[href*="/reel/"], a[href*="/permalink/"]');
+        for (const a of anchors) {
+            const h = a.getAttribute('href') || '';
+            if (!h) continue;
+            const cell = a.closest('div[style]') || a.parentElement;
+            if (!cell) continue;
+            const txt = (cell.innerText || '').trim();
+            if (txt.length < 40) continue;
+            const lower = txt.toLowerCase();
+            let matched = false;
+            for (const t of terms) { if (lower.includes(t)) { matched = true; break; } }
+            if (!matched) continue;
+            const key = txt.substring(0, 80);
            if (seenTexts.has(key)) continue;
            seenTexts.add(key);
-
-                // --- Publisher name ---
            let author = '';
-                const nameEl = el.querySelector('h4, strong, a[role="link"]');
-                if (nameEl) {
-                    author = (nameEl.innerText || '').trim();
-                }
-                if (!author) {
-                    const links = el.querySelectorAll('a');
-                    for (const a of links) {
-                        const t = (a.innerText || '').trim();
-                        if (t && t.length > 1 && t.length < 60 && /^[A-Za-zÀ-ÿ]/.test(t) && !t.includes('facebook') && !t.includes('/')) {
-                            author = t;
-                            break;
-                        }
-                    }
-                }
-
-                // --- Post URL ---
-                let postUrl = '';
-                for (const a of el.querySelectorAll('a')) {
-                    const h = (a.getAttribute('href') || '');
-                    if (h.includes('/posts/') || h.includes('/photo/') || h.includes('/video/') || h.includes('/groups/') || h.includes('/permalink/')) {
-                        postUrl = h.startsWith('http') ? h : 'https://www.facebook.com' + h;
-                        break;
-                    }
-                }
-
-                // --- Date from <time datetime> ---
+            const nameEl = cell.querySelector('h4, strong, a[role="link"], span[dir="auto"]');
+            if (nameEl) author = (nameEl.innerText || '').trim();
+            if (!author || author.length > 60 || author.length < 2) author = '';
+            let postUrl = h.startsWith('http') ? h : 'https://www.facebook.com' + h;
            let date = '';
-                const timeEl = el.querySelector('time');
-                if (timeEl) {
-                    date = timeEl.getAttribute('datetime') || timeEl.getAttribute('aria-label') || '';
-                }
-
-                results.push({ text, author, url: postUrl, date });
-            }
-            if (results.length > 0) break;
+            const timeEl = cell.querySelector('time');
+            if (timeEl) date = timeEl.getAttribute('datetime') || timeEl.getAttribute('aria-label') || '';
+            results.push({ text: txt, author, url: postUrl, date });
        }
        return results;
    }''')
@@ -896,15 +1008,11 @@ async def search_facebook(page, context, query: str):
    url = f'https://www.facebook.com/search/posts/?q={urllib.parse.quote(query)}'
    try:
        await page.goto(url, wait_until='domcontentloaded', timeout=30000)
-        try:
-            await page.wait_for_selector('div[role="article"], div[role="feed"]', timeout=15000)
-        except Exception:
-            pass
-        await page.wait_for_timeout(random.randint(3000, 8000))
+        await page.wait_for_timeout(random.randint(5000, 10000))

-        await human_scroll(page, steps=random.randint(2, 4), total_delay=random.uniform(6, 15))
+        await human_scroll(page, steps=random.randint(4, 7), total_delay=random.uniform(12, 25))

-        if random.random() < 0.1:
+        if random.random() < 0.3:
            await page.evaluate("window.scrollTo(0, document.body.scrollHeight)")
            await page.wait_for_timeout(random.randint(1000, 3000))
            await page.evaluate("window.scrollBy(0, -300)")
@@ -919,9 +1027,100 @@ async def search_facebook(page, context, query: str):

        raw_articles = await _get_article_elements(page)
        posts = _extract_posts_from_elements(raw_articles, url) if raw_articles else []
-        if not posts:
        raw = await page.evaluate('document.body.innerText')
-            posts = _extract_posts_from_text(raw, url)
+        text_posts = _extract_posts_from_text(raw, url)
+        existing = {(p.get('title') or p.get('content',''))[:80] for p in posts}
+        for tp in text_posts:
+            key = (tp.get('title') or tp.get('content',''))[:80]
+            if key not in existing:
+                posts.append(tp)
+        if posts:
+            # Extract author names from profile links in the DOM
+            try:
+                profiles = await page.evaluate(r'''() => {
+                    const out = [];
+                    const seenTxt = new Set();
+                    for (const a of document.querySelectorAll('a[href*="/profile.php"], a[href*="/user/"], a[href*="/people/"], a[href*="/me/"]')) {
+                        const name = (a.innerText || '').trim();
+                        if (!name || name.length < 3 || name.length > 60) continue;
+                        const words = name.split(' ');
+                        if (words.length < 2 || words.length > 6) continue;
+                        if (!/^[A-Z]/.test(name)) continue;
+                        if (name.includes('facebook') || name.includes('/')) continue;
+                        const cell = a.closest('div[style]') || a.parentElement;
+                        const txt = cell ? (cell.innerText || '').substring(0, 200) : '';
+                        if (!txt) continue;
+                        const key = txt.substring(0, 80);
+                        if (seenTxt.has(key)) continue;
+                        seenTxt.add(key);
+                        out.push({ name, textKey: key });
+                    }
+                    return out;
+                }''')
+                if profiles:
+                    for p in posts:
+                        pk = (p.get('content') or p.get('title') or '')[:80].strip()
+                        if not pk: continue
+                        for pr in profiles:
+                            if pk[:30] in pr['textKey'] or pr['textKey'][:30] in pk:
+                                if not p.get('author'):
+                                    p['author'] = pr['name']
+                                break
+            except Exception:
+                pass
+            try:
+                meta = await page.evaluate(r'''() => {
+                    const out = [];
+                    const seenHref = new Set();
+                    const all = document.querySelectorAll('a[href]');
+                    for (const a of all) {
+                        const h = a.href;
+                        if (!h || seenHref.has(h)) continue;
+                        const u = new URL(h);
+                        if (!u.hostname.includes('facebook.com') && !u.hostname.includes('fb.com')) continue;
+                        const path = u.pathname;
+                        if (path === '/' || path === '/search/' || path === '/marketplace/' || path.startsWith('/messages/') || path.startsWith('/notifications/') || path.startsWith('/settings/')) continue;
+                        const isPost = path.includes('/posts/') || path.includes('/videos/') || path.includes('/photos/') || path.startsWith('/story.php') || path.startsWith('/photo.php') || path.startsWith('/watch') || path.includes('/reel/') || path.startsWith('/permalink/');
+                        if (!isPost) continue;
+                        const cell = a.closest('[style*="position"], [role="article"], div[data-pagelet], div.x1yztbdb, div.x78zum5, div.x1n2onr6') || a.parentElement;
+                        if (!cell) continue;
+                        const txt = (cell.innerText || '').trim();
+                        if (txt.length < 40) continue;
+                        let author = '';
+                        const nameEl = cell.querySelector('h4, strong, a[role="link"], span[dir="auto"]');
+                        if (nameEl) author = (nameEl.innerText || '').trim();
+                        if (!author || author.length > 60) author = '';
+                        const key = txt.substring(0, 100);
+                        seenHref.add(h);
+                        out.push({ key, href: h, author, textMap: txt.substring(0,200) });
+                    }
+                    return out;
+                }''')
+                if meta:
+                    for p in posts:
+                        pk = (p.get('content') or p.get('title') or '')[:100].strip()
+                        if not pk: continue
+                        for m in meta:
+                            mk = m.get('key') or ''
+                            if pk[:40] in mk or mk[:40] in pk:
+                                if m.get('href') and 'search/' not in m['href']:
+                                    p['url'] = m['href']
+                                if m.get('author') and not p.get('author'):
+                                    p['author'] = m['author']
+                                break
+            except Exception:
+                pass
+        # Clean up: remove group names passed off as authors
+        for p in posts:
+            if p.get('author'):
+                a = p['author']
+                al = a.lower()
+                if any(kw in al for kw in BROAD_KEYWORDS) or is_offer(a) or len(a.split()) < 2 or any(w in _NON_NAMES for w in al.split()):
+                    p['author'] = ''
+        # Final filter: strip any remaining group posts
+        posts = [p for p in posts if not (
+            '/groups/' in p.get('url', '') or ' / ' in (p.get('title') or p.get('content') or '')
+        )]
    except Exception as e:
        logger.warning("Facebook search '%s' failed: %s", query, e)
        return page, []
@@ -971,7 +1170,7 @@ def cleanup_chrome():
 #   - Auto-detect: Firefox → Opera → Chrome → Edge (first found wins)
 #   - If no profile found → Agent (browser-use + ChatOllama) fallback

-async def scrape_facebook(profile_path: str | None = None, force: bool = False) -> dict:
+async def scrape_facebook(profile_path: str | None = None, force: bool = False, query: str | None = None) -> dict:
    """Dispatcher — respect SELECTED_BROWSER, fall through on flag, then Agent."""
    effective_path = profile_path or ""
    browser_type = ""
@@ -1013,14 +1212,14 @@ async def scrape_facebook(profile_path: str | None = None, force: bool = False)

    # Firefox path
    if browser_type == "firefox":
-        result = await _scrape_with_firefox(effective_path, force)
+        result = await _scrape_with_firefox(effective_path, force, query)
        if result.get("success") or not result.get("flagged"):
            return result
        logger.warning("Firefox flagged (%s), trying Agent", result.get("flag_reason", "unknown"))
        return await _scrape_with_agent(force)

    # Chromium-based (chrome / opera / edge)
-    result = await _scrape_with_chromium(effective_path, browser_type, force)
+    result = await _scrape_with_chromium(effective_path, browser_type, force, query)
    if result.get("success") or not result.get("flagged"):
        return result
    logger.warning("%s flagged (%s), trying Agent", browser_type, result.get("flag_reason", "unknown"))
@@ -1035,7 +1234,7 @@ async def scrape_facebook(profile_path: str | None = None, force: bool = False)
 # human browsing). If flagged by Facebook, returns flagged=True for the
 # dispatcher to fall through to the Agent.

-async def _scrape_with_firefox(profile_path: str, force: bool) -> dict:
+async def _scrape_with_firefox(profile_path: str, force: bool, query: str | None = None) -> dict:
    """Scrape Facebook using Firefox + persistent real profile (no cookie injection)."""
    if not profile_path:
        return {"success": False, "leads": [], "flagged": False, "flag_reason": None, "error": "No profile path"}
@@ -1089,9 +1288,13 @@ async def _scrape_with_firefox(profile_path: str, force: bool) -> dict:
                return {"success": True, "leads": [], "flagged": False, "flag_reason": None, "error": None}

            all_posts = []
+            if query:
+                query_pool = _search_list_for_query(query)
+                searches = [query] + random.sample(query_pool, k=random.randint(1, 2))
+            else:
                searches = random.sample(FB_SEARCHES, k=random.randint(2, 4))
-            for i, query in enumerate(searches):
-                page, posts = await search_facebook(page, context, query)
+            for i, sq in enumerate(searches):
+                page, posts = await search_facebook(page, context, sq)
                all_posts.extend(posts)
                if not posts:
                    continue
@@ -1142,6 +1345,12 @@ async def _scrape_with_firefox(profile_path: str, force: bool) -> dict:
                pass


+# ── Chromium Scraper ─────────────────────────────────────────────────
+
+
+# ── Chromium Scraper ─────────────────────────────────────────────────
+
+
 # ── Chromium Scraper ─────────────────────────────────────────────────
 # Generic scraper for Chrome/Edge/Opera. Uses the same structure as the
 # Firefox scraper but with Chromium-specific launch config:
@@ -1152,7 +1361,7 @@ async def _scrape_with_firefox(profile_path: str, force: bool) -> dict:
 # Anti-detection script differs slightly from Firefox variant.
 # Same decoy-skip and flagged-fallback behavior as Firefox.

-async def _scrape_with_chromium(profile_path: str, browser: str, force: bool = False) -> dict:
+async def _scrape_with_chromium(profile_path: str, browser: str, force: bool = False, query: str | None = None) -> dict:
    """Scrape Facebook using a Chromium-based browser profile (Chrome/Edge/Opera)."""
    if not profile_path:
        return {"success": False, "leads": [], "flagged": False, "flag_reason": None, "error": "No profile path"}
@@ -1214,9 +1423,13 @@ async def _scrape_with_chromium(profile_path: str, browser: str, force: bool = F
                return {"success": True, "leads": [], "flagged": False, "flag_reason": None, "error": None}

            all_posts = []
+            if query:
+                query_pool = _search_list_for_query(query)
+                searches = [query] + random.sample(query_pool, k=random.randint(1, 2))
+            else:
                searches = random.sample(FB_SEARCHES, k=random.randint(2, 4))
-            for i, query in enumerate(searches):
-                page, posts = await search_facebook(page, context, query)
+            for i, sq in enumerate(searches):
+                page, posts = await search_facebook(page, context, sq)
                all_posts.extend(posts)
                if not posts:
                    continue
@@ -1449,15 +1662,35 @@ Return a JSON array like ["yes","no","yes"] matching the order above."""
        "price for",
    ]
    keyword_leads: list[dict] = []
+    offer_reject = [
+        'i build website', 'i offer web', 'i am a web developer',
+        'affordable web design package', 'web hosting',
+        'i do web design', 'i develop websites',
+        'do you need a', 'do you want a', 'we can build',
+        'i can build', 'my portfolio', 'reasonable price',
+        'whatsapp me', 'looking for a business', 'looking for client',
+        'help your business', 'i am a web', 'contact me',
+        'we offer web', 'we provide web',
+        'take the quiz', 'homeschool', 'your home tutor',
+        'link in bio', 'apply now', 'get started',
+        'for only', 'low price', 'hit me up',
+        'send me a message', 'i do website', 'we do website',
+        'we do web', 'i do web',
+        'website designer / web developer', 'website & software creators',
+        'website builders for small businesses', 'australia web designers',
+        'south africa', 'wix website design',
+        'for sale', 'selling my', 'premium',
+        'i\'m selling', 'i\'m offering', 'we\'re offering',
+        'free ecommerce', 'free website design',
+        'starting a', 'looking for a few businesses',
+    ]
    for r in results:
        t = r['title'].lower()
        has_web = any(kw in t for kw in web_terms)
        has_request = any(kw in t for kw in request_terms)
        if not has_web or not has_request:
            continue
-        if any(kw in t for kw in ['i build website', 'i offer web', 'i am a web developer',
-                                   'affordable web design package', 'web hosting',
-                                   'i do web design', 'i develop websites']):
+        if any(kw in t for kw in offer_reject):
            continue
        keyword_leads.append(r)

@@ -1469,6 +1702,8 @@ Return a JSON array like ["yes","no","yes"] matching the order above."""
        if key not in seen_titles:
            seen_titles.add(key)
            merged.append(r)
+    # Final sweep: strip any remaining offers or group posts from merged
+    merged = [r for r in merged if not any(kw in (r.get('title','') or '').lower() for kw in offer_reject)]

    # Fill to 5 with loose keyword matches (at least web OR request term)
    if len(merged) < 5:
@@ -1477,7 +1712,10 @@ Return a JSON array like ["yes","no","yes"] matching the order above."""
            if key in seen_titles:
                continue
            t = r['title'].lower()
-            if any(kw in t for kw in web_terms) or any(kw in t for kw in request_terms):
+            if not (any(kw in t for kw in web_terms) or any(kw in t for kw in request_terms)):
+                continue
+            if any(kw in t for kw in offer_reject):
+                continue
            seen_titles.add(key)
            merged.append(r)
            if len(merged) >= 5:
@@ -1601,8 +1839,8 @@ async def agent_run(task: str = Body(..., embed=True)):
        return {"success": False, "error": str(e)}

@app.post("/scrape/facebook")
-async def scrape_facebook_endpoint(profile_path: str | None = Query(None), force: bool = Query(False)):
-    result = await scrape_facebook(profile_path, force)
+async def scrape_facebook_endpoint(profile_path: str | None = Query(None), force: bool = Query(False), query: str | None = Query(None)):
+    result = await scrape_facebook(profile_path, force, query)
    return result

 if __name__ == "__main__":
@@ -1,10 +1,2 @@
-{"job_title":"Software Developer","keywords":["developer","programmer","software engineer","coder","full stack","backend","frontend"],"industry":"Technology","description":"Builds and maintains software applications and systems"}
-{"job_title":"Marketing Specialist","keywords":["marketing","digital marketing","brand manager","content marketer","social media"],"industry":"Marketing","description":"Plans and executes marketing campaigns across channels"}
-{"job_title":"Sales Representative","keywords":["sales rep","account executive","business development","sales consultant"],"industry":"Sales","description":"Drives revenue through client acquisition and relationship management"}
-{"job_title":"Project Manager","keywords":["project manager","program manager","scrum master","agile coach"],"industry":"Business","description":"Oversees project timelines, resources, and deliverables"}
-{"job_title":"Graphic Designer","keywords":["designer","graphic designer","ui designer","ux designer","visual designer"],"industry":"Creative","description":"Creates visual concepts and designs for digital and print media"}
-{"job_title":"Data Analyst","keywords":["data analyst","business analyst","data scientist","analytics"],"industry":"Technology","description":"Analyzes data to provide actionable business insights"}
-{"job_title":"Customer Support Specialist","keywords":["customer support","customer service","support agent","help desk"],"industry":"Customer Service","description":"Assists customers with inquiries, issues, and product support"}
-{"job_title":"Human Resources Manager","keywords":["HR manager","HR","recruiter","talent acquisition","people operations"],"industry":"Human Resources","description":"Manages recruitment, employee relations, and HR operations"}
-{"job_title":"Financial Advisor","keywords":["financial advisor","financial planner","wealth manager","investment advisor"],"industry":"Finance","description":"Provides financial guidance and investment planning to clients"}
-{"job_title":"Operations Manager","keywords":["operations manager","operations","logistics","supply chain"],"industry":"Business","description":"Oversees daily operations and process optimization"}
+{"job_title":"Website Creation","keywords":["need a website","build my website","create a website for me","website for my business","need someone to build","who can build me","looking for a web developer","need a web designer","i need a website","need help with my website","looking for someone to create","need a site for","want a website for","need ecommerce website","need landing page","website for my small business","need my website redesigned","can someone build me","anyone know a web developer","recommend a web developer","need a new website"],"industry":"Technology","description":"Find people asking for websites, landing pages, or online stores to be built"}
+{"job_title":"Tutoring","keywords":["need a tutor","looking for a tutor","tutor for my child","need help with homework","private tutor needed","looking for tutoring","need a math tutor","english tutor needed","online tutor for","lessons for my child","help my child with","need someone to tutor","looking for someone to teach","tutoring for my","need help learning","recommend a tutor","anyone know a tutor","need a private tutor","tutoring services for my child","need academic help"],"industry":"Education","description":"Find parents and students asking for tutoring services"}
@@ -0,0 +1,31 @@
+-- ============================================================================
+-- Fixes: password_change_required + audit trigger UUID cast
+-- ============================================================================
+
+-- 1. All users use the passwords already set in the seed — no forced change
+UPDATE users SET password_change_required = FALSE WHERE password_change_required = TRUE;
+
+-- 2. Fix audit_password_change trigger: current_setting returns TEXT but
+--    audit_logs.changed_by is UUID — cast to UUID to prevent type error
+CREATE OR REPLACE FUNCTION audit_password_change()
+RETURNS TRIGGER AS $$
+BEGIN
+    IF OLD.password_hash IS DISTINCT FROM NEW.password_hash THEN
+        INSERT INTO audit_logs (
+            table_name, record_id, action, old_data, new_data, changed_by, ip_address
+        ) VALUES (
+            'users',
+            NEW.id,
+            'UPDATE',
+            jsonb_build_object('password_changed', true, 'password_change_required', OLD.password_change_required),
+            jsonb_build_object('password_changed', true, 'password_change_required', NEW.password_change_required),
+            NULLIF(current_setting('app.current_user_id', true), '')::UUID,
+            NULLIF(current_setting('app.current_ip', true), '')
+        );
+    END IF;
+    RETURN NEW;
+END;
+$$ LANGUAGE plpgsql SECURITY DEFINER;
+
+-- 3. Re-enable the trigger (was disabled as workaround for the UUID bug)
+ALTER TABLE users ENABLE TRIGGER trg_audit_password_change;
@@ -75,5 +75,9 @@ BEGIN;

 \echo '=== Running 019_allow_null_start_time.sql (Allow Null Start Time) ==='
 \i 019_allow_null_start_time.sql
+
+\echo '=== Running 020_fixes.sql (password_change_required + audit trigger fix) ==='
+\i 020_fixes.sql
+
 \echo '=== Migration Complete ==='
 COMMIT;
@@ -9,7 +9,7 @@
    "dev:repair": "node scripts/code-repair-agent.mjs --watch",
    "dev:start": "concurrently -n REPAIR,AI,BROWSE,SIGNAL,NEXT,OPEN -c red,cyan,magenta,yellow,green,white \"npm run dev:repair\" \"npm run dev:rust\" \"npm run dev:browser-use\" \"npm run dev:signaling\" \"npm run dev:next\" \"npm run dev:open\"",
    "dev:next": "next dev -p 3006",
-    "dev:precheck": "node scripts/precheck.mjs",
+    "dev:precheck": "node scripts/precheck.mjs && node scripts/run-migrations.mjs",
    "dev:ollama": "node scripts/ensure-ollama.mjs",
    "dev:rust": "node ai-server/index.mjs",
    "dev:browser-use": "cd browser-use-service && node ../scripts/run-python.mjs main.py",
@@ -0,0 +1,184 @@
+// ── Automated Database Migration Runner ──────────────────────────────
+// Reads .env.local for DATABASE_URL, tracks applied migrations in
+// a _migrations table, and runs unapplied .sql files via psql.
+// Runs automatically on npm run dev and npm run setup.
+// ============================================================================
+
+import { readFileSync, existsSync } from "node:fs"
+import { execSync } from "node:child_process"
+import { resolve, dirname } from "node:path"
+import { fileURLToPath } from "node:url"
+import { createRequire } from "node:module"
+
+const __dirname = dirname(fileURLToPath(import.meta.url))
+const ROOT = resolve(__dirname, "..")
+
+// ── Load .env.local into process.env ─────────────────────────────────
+
+const envPath = resolve(ROOT, ".env.local")
+if (existsSync(envPath)) {
+  for (const line of readFileSync(envPath, "utf-8").split("\n")) {
+    const trimmed = line.trim()
+    if (!trimmed || trimmed.startsWith("#")) continue
+    const eq = trimmed.indexOf("=")
+    if (eq === -1) continue
+    const key = trimmed.slice(0, eq).trim()
+    let value = trimmed.slice(eq + 1).trim()
+    if ((value.startsWith('"') && value.endsWith('"')) || (value.startsWith("'") && value.endsWith("'"))) {
+      value = value.slice(1, -1)
+    }
+    if (!process.env[key]) process.env[key] = value
+  }
+}
+
+const DATABASE_URL = process.env.DATABASE_URL
+if (!DATABASE_URL) {
+  console.log("  ~ DATABASE_URL not set — skipping migrations")
+  process.exit(0)
+}
+
+// ── Find psql ────────────────────────────────────────────────────────
+
+function findPsql() {
+  try {
+    execSync("psql --version", { stdio: "pipe", timeout: 5000 })
+    return "psql"
+  } catch {}
+
+  const candidates = [
+    "C:\\Program Files\\PostgreSQL\\16\\bin\\psql.exe",
+    "C:\\Program Files\\PostgreSQL\\17\\bin\\psql.exe",
+    "C:\\Program Files\\PostgreSQL\\15\\bin\\psql.exe",
+    "C:\\Program Files\\PostgreSQL\\14\\bin\\psql.exe",
+  ]
+  for (const p of candidates) {
+    if (existsSync(p)) return `"${p}"`
+  }
+  return null
+}
+
+const PSQL = findPsql()
+if (!PSQL) {
+  console.log("  ~ psql not found — skipping migrations (install PostgreSQL client tools)")
+  process.exit(0)
+}
+
+// ── Database connection (for tracking table) ─────────────────────────
+
+const require = createRequire(import.meta.url)
+const { Pool } = require("pg")
+
+let pool
+try {
+  pool = new Pool({
+    connectionString: DATABASE_URL,
+    max: 1,
+    connectionTimeoutMillis: 5000,
+    idleTimeoutMillis: 10000,
+  })
+  await pool.query("SELECT 1")
+} catch {
+  console.log("  ~ Database not available — skipping migrations")
+  process.exit(0)
+}
+
+// ── Migration logic ──────────────────────────────────────────────────
+
+async function runMigrations() {
+  // 1. Create tracking table if not exists
+  await pool.query(`
+    CREATE TABLE IF NOT EXISTS _migrations (
+      id SERIAL PRIMARY KEY,
+      filename VARCHAR(255) NOT NULL UNIQUE,
+      applied_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
+    )
+  `)
+
+  // 2. Determine file order from run_all.sql (authoritative)
+  const runAllPath = resolve(ROOT, "database", "migrations", "run_all.sql")
+  const runAllContent = readFileSync(runAllPath, "utf-8")
+  const fileOrder = []
+  for (const line of runAllContent.split("\n")) {
+    const match = line.match(/\\i\s+(\S+\.sql)/)
+    if (match) fileOrder.push(match[1])
+  }
+
+  if (fileOrder.length === 0) {
+    console.log("  ~ No migration files found in run_all.sql")
+    return
+  }
+
+  // 3. Get already-applied files
+  let appliedSet
+  try {
+    const { rows: applied } = await pool.query("SELECT filename FROM _migrations ORDER BY filename")
+    appliedSet = new Set(applied.map((r) => r.filename))
+  } catch {
+    appliedSet = new Set()
+  }
+
+  // 4. Build psql command base
+  const psqlCmd = `${PSQL} -d "${DATABASE_URL}" -v ON_ERROR_STOP=1`
+
+  // 5. Run unapplied files in order.
+  //    psql -v ON_ERROR_STOP=1 aborts on the first error with exit code 3.
+  //    If the error is "already exists", the file was already applied before
+  //    tracking existed — mark it as applied and continue.
+  //    Other errors are genuine failures — abort.
+  const ALREADY_APPLIED_PATTERNS = [
+    "already exists",
+    "duplicate key",
+    "cannot insert multiple commands into a prepared statement",
+  ]
+
+  let count = 0
+  for (const file of fileOrder) {
+    if (appliedSet.has(file)) continue
+
+    const filePath = resolve(ROOT, "database", "migrations", file)
+    if (!existsSync(filePath)) {
+      console.error(`  ✗ Migration file not found: ${file}`)
+      process.exit(1)
+    }
+
+    let stderr = ""
+    try {
+      const result = execSync(`${psqlCmd} -f "${filePath}"`, {
+        timeout: 30000,
+        encoding: "utf-8",
+        maxBuffer: 10 * 1024 * 1024,
+        stdio: ["ignore", "pipe", "pipe"],
+      })
+      // psql outputs notices/warnings to stderr even on success
+      stderr = result.stderr || ""
+    } catch (err) {
+      const msg = err.stderr || err.stdout || err.message
+      const isAlreadyApplied = ALREADY_APPLIED_PATTERNS.some((p) => msg.includes(p))
+      if (isAlreadyApplied) {
+        await pool.query("INSERT INTO _migrations (filename) VALUES ($1) ON CONFLICT DO NOTHING", [file])
+        console.log(`  ~ Already applied: ${file}`)
+        count++
+        continue
+      }
+      console.error(`  ✗ Migration failed: ${file}`)
+      console.error(`    ${msg.split("\n").slice(0, 5).join("\n    ")}`)
+      process.exit(1)
+    }
+
+    await pool.query("INSERT INTO _migrations (filename) VALUES ($1)", [file])
+    console.log(`  ✓ Applied: ${file}`)
+    count++
+  }
+
+  if (count === 0) {
+    console.log("  ~ All migrations up to date")
+  }
+}
+
+// ── Run ──────────────────────────────────────────────────────────────
+
+try {
+  await runMigrations()
+} finally {
+  await pool.end()
+}
@@ -202,7 +202,16 @@ if (!existsSync(resolve(ROOT, ".env.local"))) {
  console.log("\n── .env.local already exists, skipping ──")
 }

-// 5. Self-healing build check (--self-heal flag)
+// 5. Database migrations (auto-applies on fresh install)
+console.log("\n── Running Database Migrations ──")
+try {
+  execSync("node scripts/run-migrations.mjs", { stdio: "inherit", timeout: 30000, cwd: ROOT })
+  console.log("  ✓ Migrations complete")
+} catch {
+  console.log("  ~ Database not available — run migrations later with: npm run dev")
+}
+
+// 6. Self-healing build check (--self-heal flag)
 if (doSelfHeal) {
  console.log("\n── Self-Healing Build Check ──")
  const { generateModule } = loadRegistry()
@@ -252,7 +261,7 @@ if (doSelfHeal) {
  }
 }

-// 6. Final summary
+// 7. Final summary
 console.log("\n── Final Steps ──")
 console.log("  1. Make sure PostgreSQL is running with database 'crm'")
 console.log("  2. Pull the Ollama model:  ollama pull dolphin-llama3:8b")
@@ -1,7 +1,7 @@
 "use client"

-import { useState, useCallback, useRef, useEffect } from "react"
-import { AIChat } from "@/components/ai/ai-chat"
+import { useState, useCallback, useRef } from "react"
+import { AIChat, type AIChatHandle } from "@/components/ai/ai-chat"
 import { JobSelector } from "@/components/ai/job-selector"
 import { Bot, ChevronRight } from "lucide-react"

@@ -16,12 +16,50 @@ const tips = [
 export default function AIAssistantPage() {
  const [selectedJob, setSelectedJob] = useState<{ job_title: string; keywords: string[]; industry: string; description: string } | null>(null)
  const [recentPrompts, setRecentPrompts] = useState<string[]>([])
-  const aiChatRef = useRef<{ fillInput: (text: string) => void } | null>(null)
+  const [searching, setSearching] = useState(false)
+  const aiChatRef = useRef<AIChatHandle | null>(null)

  const handleJobSelect = useCallback((job: typeof selectedJob) => {
    setSelectedJob(job)
  }, [])

+  const handleSearch = useCallback(async (job: NonNullable<typeof selectedJob>) => {
+    setSearching(true)
+    const keyword = job.keywords[0]
+    aiChatRef.current?.addAssistantMessage(`🔍 Searching Facebook for **${job.job_title}** leads...`)
+
+    const controller = new AbortController()
+    const timeoutId = setTimeout(() => controller.abort(), 360000)
+    const statusId = setTimeout(() => {
+      aiChatRef.current?.addAssistantMessage("⏳ Still searching Facebook (this can take up to 5 minutes)...")
+    }, 45000)
+
+    try {
+      const res = await fetch(`http://localhost:3008/scrape/facebook?force=true&query=${encodeURIComponent(keyword)}`, { method: "POST", signal: controller.signal })
+      clearTimeout(timeoutId)
+      clearTimeout(statusId)
+      const data = await res.json()
+      if (data.success && data.leads?.length > 0) {
+        const leadsText = data.leads.map((lead: any, i: number) =>
+          `**${i + 1}.** ${lead.author || "Unknown"}\n> ${(lead.content || "").slice(0, 300)}\n> 🔗 ${lead.url || "(no link available)"}`
+        ).join("\n\n")
+        aiChatRef.current?.addAssistantMessage(`✅ Found **${data.leads.length}** leads:\n\n${leadsText}`)
+      } else {
+        const reason = data.error || data.flag_reason || "No leads found this time"
+        aiChatRef.current?.addAssistantMessage(`⚠️ ${reason}`)
+      }
+    } catch (err: any) {
+      clearTimeout(timeoutId)
+      clearTimeout(statusId)
+      const msg = err.name === "AbortError"
+        ? "❌ Scraper timed out after 5 minutes. Try again or check the scraper service."
+        : `❌ ${err instanceof Error ? err.message : "Search failed"}`
+      aiChatRef.current?.addAssistantMessage(msg)
+    } finally {
+      setSearching(false)
+    }
+  }, [])
+
  const handleTipClick = useCallback((tip: string) => {
    aiChatRef.current?.fillInput(tip)
  }, [])
@@ -71,7 +109,7 @@ export default function AIAssistantPage() {
            <span className="w-1.5 h-1.5 rounded-full bg-primary" />
            <span className="text-primary text-[10px] font-bold uppercase tracking-[0.15em]">Target Job</span>
          </div>
-          <JobSelector onSelect={handleJobSelect} />
+          <JobSelector onSelect={handleJobSelect} onSearch={handleSearch} searching={searching} />
          {selectedJob && (
            <div className="bg-card/50 border border-border rounded-xl p-3.5 mt-3 space-y-2">
              <h4 className="text-sm font-semibold text-foreground">{selectedJob.job_title}</h4>
@@ -48,7 +48,12 @@ interface AIChatProps {
 }


-export const AIChat = forwardRef<{ fillInput: (text: string) => void }, AIChatProps>(({ onMessageSent }, ref) => {
+export interface AIChatHandle {
+  fillInput: (text: string) => void
+  addAssistantMessage: (content: string) => void
+}
+
+export const AIChat = forwardRef<AIChatHandle, AIChatProps>(({ onMessageSent }, ref) => {
  const [messages, setMessages] = useState<ChatMessage[]>([])
  const [input, setInput] = useState("")
  const [loading, setLoading] = useState(false)
@@ -66,6 +71,17 @@ export const AIChat = forwardRef<{ fillInput: (text: string) => void }, AIChatPr
      setInput(text)
      setTimeout(() => textareaRef.current?.focus(), 50)
    },
+    addAssistantMessage(content: string) {
+      setMessages((prev) => {
+        if (!prev.some((m) => m.role === "user")) {
+          return [
+            { role: "user", content: "Search Facebook" },
+            { role: "assistant", content },
+          ]
+        }
+        return [...prev, { role: "assistant", content }]
+      })
+    },
  }), [])

  const handleGifSelect = useCallback((gif: { url: string; previewUrl: string; title: string }) => {
@@ -1,7 +1,7 @@
 "use client"

 import { useState, useEffect } from "react"
-import { Briefcase, ChevronDown, Loader2 } from "lucide-react"
+import { Briefcase, ChevronDown, Loader2, Search } from "lucide-react"

 interface Job {
  job_title: string
@@ -12,9 +12,11 @@ interface Job {

 interface JobSelectorProps {
  onSelect: (job: Job | null) => void
+  onSearch?: (job: Job) => void
+  searching?: boolean
 }

-export function JobSelector({ onSelect }: JobSelectorProps) {
+export function JobSelector({ onSelect, onSearch, searching }: JobSelectorProps) {
  const [jobs, setJobs] = useState<Job[]>([])
  const [loading, setLoading] = useState(true)
  const [open, setOpen] = useState(false)
@@ -39,36 +41,47 @@ export function JobSelector({ onSelect }: JobSelectorProps) {
      <button
        type="button"
        onClick={() => setOpen(!open)}
-        className="w-full flex items-center gap-2.5 bg-[#1a1d2e] border border-[#ffffff0f] hover:border-primary/30 rounded-xl px-4 py-3 text-sm text-[#9ca3af] hover:text-[#e5e7eb] transition-all duration-200"
+        className="w-full flex items-center gap-2.5 bg-card/50 border border-border hover:border-primary/20 rounded-xl px-4 py-3 text-sm text-muted-foreground hover:text-foreground transition-all duration-200"
      >
-        <Briefcase className="h-4 w-4 text-primary flex-none" />
+        <Briefcase className="h-4 w-4 text-[#f97316] flex-none" />
        <span className="flex-1 text-left truncate">
          {selected ? selected.job_title : loading ? "Loading jobs..." : "Select a job category"}
        </span>
-        {loading ? <Loader2 className="h-3.5 w-3.5 animate-spin text-primary" /> : <ChevronDown className={`h-3.5 w-3.5 text-[#4b5563] transition-transform duration-200 ${open ? "rotate-180" : ""}`} />}
+        {loading ? <Loader2 className="h-3.5 w-3.5 animate-spin text-primary" /> : <ChevronDown className={`h-3.5 w-3.5 text-muted-foreground/60 transition-transform duration-200 ${open ? "rotate-180" : ""}`} />}
      </button>

      {open && (
        <>
          <div className="fixed inset-0 z-10" onClick={() => setOpen(false)} />
-          <div className="absolute top-full left-0 right-0 mt-1.5 z-20 bg-[#1a1d2e] border border-[#ffffff0f] rounded-xl shadow-xl shadow-black/40 max-h-60 overflow-y-auto">
+          <div className="absolute top-full left-0 right-0 mt-1.5 z-20 bg-card border border-border rounded-xl shadow-xl max-h-60 overflow-y-auto">
            {jobs.map((job, i) => (
              <button
                key={i}
                type="button"
                onClick={() => handleSelect(job)}
-                className="w-full text-left px-4 py-3 text-sm text-[#9ca3af] hover:bg-[#1f2437] hover:text-[#e5e7eb] transition-all duration-150 border-b border-[#ffffff08] last:border-0 border-l-2 border-l-transparent hover:border-l-primary/40"
+                className="w-full text-left px-4 py-3 text-sm text-muted-foreground hover:bg-muted hover:text-foreground transition-all duration-150 border-b border-border/20 last:border-0 border-l-2 border-l-transparent hover:border-l-primary/40"
              >
                <div className="font-medium">{job.job_title}</div>
-                <div className="text-xs text-[#4b5563] mt-0.5">{job.industry} &mdash; {job.description}</div>
+                <div className="text-xs text-muted-foreground/60 mt-0.5">{job.industry} &mdash; {job.description}</div>
              </button>
            ))}
            {jobs.length === 0 && !loading && (
-              <div className="px-4 py-4 text-xs text-[#4b5563] text-center">No job categories loaded</div>
+              <div className="px-4 py-4 text-xs text-muted-foreground/60 text-center">No job categories loaded</div>
            )}
          </div>
        </>
      )}
+      {selected && onSearch && (
+        <button
+          type="button"
+          onClick={() => onSearch(selected)}
+          disabled={searching}
+          className="w-full flex items-center justify-center gap-2 mt-3 bg-gradient-to-r from-[#f97316] to-[#ea580c] hover:from-[#ea580c] hover:to-[#dc2626] disabled:opacity-50 disabled:cursor-not-allowed text-white text-sm font-semibold rounded-xl px-4 py-3 transition-all duration-200 shadow-lg shadow-[#f97316]/20"
+        >
+          {searching ? <Loader2 className="h-4 w-4 animate-spin" /> : <Search className="h-4 w-4" />}
+          {searching ? "Searching..." : "Search Facebook"}
+        </button>
+      )}
    </div>
  )
 }