Added Ai load fix

2026-07-10 11:15:43 +02:00 · 2026-07-02 13:26:56 +02:00
parent 1269f6cce8
commit 37679a7a60
2 changed files with 144 additions and 13 deletions
@@ -372,6 +372,7 @@ OFFER_PATTERNS = [
    r"\bmy\s+portfolio\b",
    r"\breasonable\s+(price|pricing|rate)\b",
    r"\bwhatsapp\s+(me|us|at)\b",
+    r"\bwhatsapp\b",
    r"\blooking\s+for\s+(a\s+)?(business|client|customer)\b",
    r"\bhelp\s+your\s+business\b",
    r"\bi\s+am\s+a\s+(web|freelance|designer|developer)\b",
@@ -407,6 +408,46 @@ OFFER_PATTERNS = [
    r"\bselling\s+(my|a|the|this)\b",
    r"\bpremium\b",
    r"\bi'm\s+selling\b",
+    r"\bgroup\b",
+    r"\bi\s+can\s+help\b",
+    r"\binbox\s+me\b",
+    r"\bpm\s+me\b",
+    r"\bdm\s+me\b",
+    r"\bmessage\s+me\s+for\b",
+    r"\bquote\b",
+    r"\bdiscount\b",
+    r"\bbest\s+(deal|price|offer|service)\b",
+    r"\bprice\s+(start|begin|include|range)\b",
+    r"\breach\s+out\b",
+    r"\bclick\s+(the\s+)?link\b",
+    r"\bcheck\s+(out|this|my)\b",
+    r"\bwebsite\s+for\s+your\b",
+    r"\bprobono\b",
+    r"\bfreelance\s+opportunit",
+    r"\bfull.?stack\b",
+    r"\bresponsive\s+(web)?sites?\b",
+    r"\blooking\s+for\s+(new\s+)?(projects?|work)\b",
+    r"\bmern\s+stack\b",
+    r"\bexpress\s+js\b",
+    r"\breact\s+js\b",
+    r"\bnode\s+js\b",
+    r"\bwebsite\s+for\s+\$\b",
+    r"\bi\s+build\s+(websites?|shopify|wordpress)\b",
+    r"\bwe\s+build\s+(websites?|shopify|wordpress)\b",
+    r"\bi\s+create\s+(websites?|shopify|wordpress)\b",
+    r"\bwe\s+create\s+(websites?|shopify|wordpress)\b",
+    r"\bfull.?time\s+(position|job|role|work)\b",
+    r"\bremote\s+(position|job|role|work)\b",
+    r"\bhelp\s+wanted\b",
+    r"\bjob\s+(opening|vacancy|opportunity)\b",
+    r"\bnow\s+hiring\b",
+    r"\bpart.?time\s+(position|job|role|work)\b",
+    r"\byears?\s+of\s+(teaching|experience)\b",
+    r"\bsend\s+(you|me)\s+(the\s+)?link\b",
+    r"\bcomment\s+.*\bsend\b",
+    r"\bfor\s+free\b",
+    r"\bmake\s+money\b",
+    r"\bno\s+coding\b",
 ]

 REQUEST_PATTERNS = [
@@ -748,7 +789,7 @@ def _clean_fb_text(text: str) -> str:
    return '\n'.join(cleaned_lines)

 # Words that should never be treated as person names
-_NON_NAMES = {"online","tutor","tutoring","school","academy","learning","urgently","looking","south","africa","philippines","australia","creative","professional","digital","services","solutions","statistics","actuarial","homeschooling","homeschool","reading","math","english","science","grade","group","page","website","design","development","agency","company","studio","graphic","technical","support","customer","guest","post","fashion","wordpress","shopify","ecommerce","business","marketing","social","media","content","strategy","seo","free","domain","hosting","sponsored","promoted","advertisement","need","want","help","please","contact","send","message","whatsapp","hire","freelance","writer","blogger","expert","specialist","consultant","freelancer","software","creators","village","town","city","university","college","institute","evolve","aesthetics","sale","selling","premium","builder","pro","people","ecommerce"}
+_NON_NAMES = {"online","tutor","tutoring","school","academy","learning","urgently","looking","south","africa","philippines","australia","creative","professional","digital","services","solutions","statistics","actuarial","homeschooling","homeschool","reading","math","english","science","grade","group","page","website","design","development","agency","company","studio","graphic","technical","support","customer","guest","post","fashion","wordpress","shopify","ecommerce","business","marketing","social","media","content","strategy","seo","free","domain","hosting","sponsored","promoted","advertisement","need","want","help","please","contact","send","message","whatsapp","hire","freelance","writer","blogger","expert","specialist","consultant","freelancer","software","creators","village","town","city","university","college","institute","evolve","aesthetics","sale","selling","premium","builder","pro","people","ecommerce","press","anonymous","tuition","parents","tutors","advanced","custom","fields","speed","optimization","elementor","woocommerce","acf","availability","january","february","march","april","may","june","july","august","september","october","november","december","mums","dads","uae","sharjah","dubai","abu","dhabi","hong","kong","canada","usa","united","kingdom","aunty","piano","plus","recommendations","needed"}

 # ── Post Extraction ──────────────────────────────────────────────────
 # Two strategies for extracting posts from page content:
@@ -756,10 +797,22 @@ _NON_NAMES = {"online","tutor","tutoring","school","academy","learning","urgentl
 #   2. _extract_posts_from_text     — fallback that parses raw page text line by line
 # Both apply dedup (seen_texts), offer filtering, and request scoring.

+_GROUP_SUFFIXES = [" - South Africa", " - Australia", " - Philippines", " PH", "Group", "help group", "info group", "public group", "private group", "group for", "community", "creators", "marketplace"]
+
 def _extract_posts_from_elements(elements: list[dict], base_url: str) -> list[dict]:
    posts = []
    seen_texts = set()
+    now = datetime.utcnow()
    for el in elements:
+        # Reject group posts immediately
+        if el.get('isGroup'):
+            continue
+        # Reject posts older than 2 days
+        ts = el.get('ts', 0)
+        if ts:
+            age_seconds = (now.timestamp() * 1000 - ts) / 1000
+            if age_seconds > 172800:  # 2 days
+                continue
        raw_text = el.get('text', '')
        if len(raw_text) < 40:
            continue
@@ -773,13 +826,23 @@ def _extract_posts_from_elements(elements: list[dict], base_url: str) -> list[di
        if dekey in seen_texts:
            continue
        seen_texts.add(dekey)
+        # Reject if text contains group-like patterns
+        lower = text.lower()
+        if ' / ' in lower:
+            continue
+        skip = False
+        for suff in _GROUP_SUFFIXES:
+            if suff.lower() in lower:
+                skip = True
+                break
+        if skip:
+            continue
        request_score = 2 if is_request(text) else 0
        post_url = el.get('url') or base_url

        # Prefer JS-extracted date, fall back to text parsing
        js_date = (el.get('date') or '').strip()
        if js_date:
-            # Try ISO datetime from <time datetime>
            try:
                dt = datetime.fromisoformat(js_date.replace('Z', '+00:00'))
                date_str = dt.strftime('%Y-%m-%d')
@@ -883,7 +946,6 @@ def _extract_posts_from_text(raw: str, url: str) -> list[dict]:
            if name:
                p['author'] = name
    # Remove group posts: any post whose text looks like it came from a group
-    _group_suffixes = [" - South Africa", " - Australia", " PH", "Group", "help group", "info group"]
    filtered = []
    for p in posts:
        t = (p.get('title') or p.get('content') or '')
@@ -891,14 +953,36 @@ def _extract_posts_from_text(raw: str, url: str) -> list[dict]:
            continue
        lower = t.lower()
        skip = False
-        for suff in _group_suffixes:
+        for suff in _GROUP_SUFFIXES:
            if suff.lower() in lower:
                skip = True
                break
        if skip:
            continue
        filtered.append(p)
-    return filtered
+    # Dedup: merge posts where one is a suffix of another (same post split)
+    merged = []
+    for p in filtered:
+        t = (p.get('title') or p.get('content') or '').lower()
+        # Check if this post is substantially contained in an already-merged post
+        found = False
+        for i, m in enumerate(merged):
+            mt = (m.get('title') or m.get('content') or '').lower()
+            # Word-level overlap: count shared words
+            twords = set(t.split())
+            mwords = set(mt.split())
+            if len(twords) > 3 and len(mwords) > 3:
+                overlap = len(twords & mwords)
+                smaller = min(len(twords), len(mwords))
+                if overlap / smaller >= 0.6:
+                    # Keep the longer one
+                    if len(t) > len(mt):
+                        merged[i] = p
+                    found = True
+                    break
+        if not found:
+            merged.append(p)
+    return merged

 # ── Human-like Behavior Simulation ──────────────────────────────────
 # These functions add random delays, mouse movements, and scroll patterns
@@ -950,7 +1034,9 @@ async def _get_article_elements(page) -> list[dict]:
    return await page.evaluate('''() => {
        const results = [];
        const seenTexts = new Set();
-        const terms = ["website","web design","web develop","need a","looking for","build my","create a","wordpress","landing page","ecommerce"];
+        const terms = ["website","web design","web develop","need a","looking for","build my","create a","wordpress","landing page","ecommerce","tutor","tutoring","homeschool","math","reading","english","science","grade"];
+        const now = Date.now();
+        const DAY_MS = 86400000;
        const anchors = document.querySelectorAll('a[href*="/posts/"], a[href*="/story.php"], a[href*="/photo.php"], a[href*="/watch"], a[href*="/reel/"], a[href*="/permalink/"]');
        for (const a of anchors) {
            const h = a.getAttribute('href') || '';
@@ -966,15 +1052,22 @@ async def _get_article_elements(page) -> list[dict]:
            const key = txt.substring(0, 80);
            if (seenTexts.has(key)) continue;
            seenTexts.add(key);
+
+            const isGroup = h.includes('/groups/');
            let author = '';
            const nameEl = cell.querySelector('h4, strong, a[role="link"], span[dir="auto"]');
            if (nameEl) author = (nameEl.innerText || '').trim();
            if (!author || author.length > 60 || author.length < 2) author = '';
            let postUrl = h.startsWith('http') ? h : 'https://www.facebook.com' + h;
            let date = '';
+            let ts = 0;
            const timeEl = cell.querySelector('time');
-            if (timeEl) date = timeEl.getAttribute('datetime') || timeEl.getAttribute('aria-label') || '';
-            results.push({ text: txt, author, url: postUrl, date });
+            if (timeEl) {
+                date = timeEl.getAttribute('datetime') || timeEl.getAttribute('aria-label') || '';
+                const dtVal = timeEl.getAttribute('datetime');
+                if (dtVal) { ts = new Date(dtVal).getTime(); }
+            }
+            results.push({ text: txt, author, url: postUrl, date, isGroup, ts });
        }
        return results;
    }''')
@@ -1119,7 +1212,9 @@ async def search_facebook(page, context, query: str):
                    p['author'] = ''
        # Final filter: strip any remaining group posts
        posts = [p for p in posts if not (
-            '/groups/' in p.get('url', '') or ' / ' in (p.get('title') or p.get('content') or '')
+            '/groups/' in p.get('url', '') or '/group/' in p.get('url', '')
+            or '/pages/' in p.get('url', '')
+            or ' / ' in (p.get('title') or p.get('content') or '')
        )]
    except Exception as e:
        logger.warning("Facebook search '%s' failed: %s", query, e)
@@ -1600,6 +1695,8 @@ NOT LEAD:
 - Recruiting employees, hiring staff, looking for business partners
 - Selling products, promoting services, affiliate offers
 - "Need web hosting", "Looking for a partner", "Looking for content writer", "Video spokesperson"
+- Posts from groups, communities, or pages (group announcements, group posts, page posts)
+- Posts containing the word "group", "page", "community", "creators" — these are NEVER individual leads

 For each numbered post, answer ONLY "yes" (LEAD) or "no" (NOT LEAD):
 {chr(10).join(f'{i+1}. {t}' for i, t in enumerate(briefs))}
@@ -1683,6 +1780,28 @@ Return a JSON array like ["yes","no","yes"] matching the order above."""
        'i\'m selling', 'i\'m offering', 'we\'re offering',
        'free ecommerce', 'free website design',
        'starting a', 'looking for a few businesses',
+        # Group-related rejections
+        'group', ' i need a website group', 'south africa web', 'philippines web', 'australia web',
+        'i can help', 'inbox me', 'dm me', 'pm me', 'message me for',
+        'best price', 'discount', 'reach out', 'check out my', 'check this',
+        'website for your', 'price start', 'price begin', 'website creator',
+        'website & software', 'creators &', 'creators marketplace',
+        'website group', 'page group',
+        # Self-promotion rejections
+        'i\'m a web', "i'm a web", 'i am a full stack', "i'm a full stack", 'i\'m a full stack',
+        'freelance opportunity', 'looking for new project', 'looking for new work',
+        'full stack web', 'mern stack', 'responsive business website',
+        'i build website', 'i build shopify', 'i build wordpress',
+        'we build website', 'we build shopify', 'we build wordpress',
+        'i create website', 'we create website',
+        'full time position', 'full time job', 'remote position', 'remote job',
+        'help wanted', 'now hiring', 'job opening',
+        'website speed', 'speed optimization', 'on page seo', 'off page seo',
+        'elementor pro', 'woocommerce', 'acf', 'custom post type',
+        'send you the link', 'comment website',
+        'for free', 'no coding', 'make money', 'website for free',
+        'part time job', 'part time position',
+        'years of experience', 'years of teaching',
    ]
    for r in results:
        t = r['title'].lower()
@@ -1703,7 +1822,9 @@ Return a JSON array like ["yes","no","yes"] matching the order above."""
            seen_titles.add(key)
            merged.append(r)
    # Final sweep: strip any remaining offers or group posts from merged
+    group_words = ['group', ' groups', 'page:', 'page |', 'community', 'creators', 'marketplace']
    merged = [r for r in merged if not any(kw in (r.get('title','') or '').lower() for kw in offer_reject)]
+    merged = [r for r in merged if not any(gw in (r.get('title','') or '').lower() for gw in group_words)]

    # Fill to 5 with loose keyword matches (at least web OR request term)
    if len(merged) < 5:
@@ -1716,6 +1837,8 @@ Return a JSON array like ["yes","no","yes"] matching the order above."""
                continue
            if any(kw in t for kw in offer_reject):
                continue
+            if any(gw in t for gw in group_words):
+                continue
            seen_titles.add(key)
            merged.append(r)
            if len(merged) >= 5: