From d77ff2b9653e63be745c68ca3a0254f0a624fa3f Mon Sep 17 00:00:00 2001 From: Ace Date: Tue, 7 Jul 2026 10:08:21 +0200 Subject: [PATCH] Added logic to the 4 languages so it can search each language at a time --- browser-use-service/main.py | 40 ++++++++++++++++++++++++------------- data/ai/ai.md | 13 +++++++++--- rust-ai/src/AI.md | 15 ++++++++++---- 3 files changed, 47 insertions(+), 21 deletions(-) diff --git a/browser-use-service/main.py b/browser-use-service/main.py index 7405cfc..5ca213d 100644 --- a/browser-use-service/main.py +++ b/browser-use-service/main.py @@ -711,11 +711,17 @@ async def _quick_search(page, context, query: str) -> tuple: if '/login' in current_url.lower(): logger.warning("Quick search redirected to login for '%s'", query[:40]) return page, [] - await page.wait_for_timeout(random.randint(4000, 6000)) - await page.evaluate("window.scrollBy(0, 600)") - await page.wait_for_timeout(random.randint(2000, 4000)) - await page.evaluate("window.scrollBy(0, 600)") - await page.wait_for_timeout(random.randint(2000, 3000)) + await page.wait_for_timeout(random.randint(3000, 7000)) + await page.evaluate(f"window.scrollBy(0, {random.randint(400, 900)})") + await page.wait_for_timeout(random.randint(2000, 5000)) + if random.random() < 0.35: + await page.evaluate(f"window.scrollBy(0, -{random.randint(100, 400)})") + await page.wait_for_timeout(random.randint(1500, 3500)) + await page.evaluate(f"window.scrollBy(0, {random.randint(400, 900)})") + await page.wait_for_timeout(random.randint(2000, 5000)) + if random.random() < 0.25: + await page.evaluate("window.scrollTo(0, 0)") + await page.wait_for_timeout(random.randint(1000, 3000)) raw_articles = await _get_article_elements(page) posts = _extract_posts_from_elements(raw_articles, url) if raw_articles else [] raw = await page.evaluate('document.body.innerText') @@ -865,7 +871,7 @@ def _parse_fb_date(block: list[str]) -> str: return datetime.now().strftime('%Y-%m-%d') -def _is_within_days(date_str: str, max_days: int = 3) -> bool: +def _is_within_days(date_str: str, max_days: int = 2) -> bool: """Check if date is within max_days from now. Empty/unparseable = keep.""" if not date_str: return True @@ -1442,6 +1448,7 @@ async def _scrape_with_firefox(profile_path: str, force: bool, query: str | None context = await pw.firefox.launch_persistent_context( user_data_dir=profile_dir, headless=True, + viewport=random.choice(VIEWPORTS), firefox_user_prefs={ "dom.webdriver.enabled": False, "dom.webdriver.timeout": 0, @@ -1503,7 +1510,9 @@ async def _scrape_with_firefox(profile_path: str, force: bool, query: str | None if not posts: continue if i > 0: - await page.wait_for_timeout(random.uniform(3000, 7000)) + await page.wait_for_timeout(random.uniform(5000, 12000)) + if random.random() < 0.2: + await random_idle(page) continue if random.random() < 0.4: await page.evaluate(f"window.scrollBy(0, {random.randint(-300, 300)})") @@ -1532,8 +1541,8 @@ async def _scrape_with_firefox(profile_path: str, force: bool, query: str | None seen.add(key) deduped.append(p) - # Filter to last 3 days only - deduped = [p for p in deduped if _is_within_days(p.get('date', ''), 7)] + # Filter to last 2 days only + deduped = [p for p in deduped if _is_within_days(p.get('date', ''), 2)] leads = deduped[:20] if leads: @@ -1589,6 +1598,7 @@ async def _scrape_with_chromium(profile_path: str, browser: str, force: bool = F launch_kwargs = dict( user_data_dir=profile_dir, headless=True, + viewport=random.choice(VIEWPORTS), args=CHROME_LAUNCH_ARGS, ) if channel: @@ -1649,7 +1659,9 @@ async def _scrape_with_chromium(profile_path: str, browser: str, force: bool = F if not posts: continue if i > 0: - await page.wait_for_timeout(random.uniform(3000, 7000)) + await page.wait_for_timeout(random.uniform(5000, 12000)) + if random.random() < 0.2: + await random_idle(page) continue if random.random() < 0.4: await page.evaluate(f"window.scrollBy(0, {random.randint(-300, 300)})") @@ -1678,7 +1690,7 @@ async def _scrape_with_chromium(profile_path: str, browser: str, force: bool = F seen.add(key) deduped.append(p) - deduped = [p for p in deduped if _is_within_days(p.get('date', ''), 7)] + deduped = [p for p in deduped if _is_within_days(p.get('date', ''), 2)] leads = deduped[:20] if leads: leads = await classify_leads(leads, tutoring=tutoring) @@ -1734,7 +1746,7 @@ async def _scrape_with_agent(force: bool = False) -> dict: - The post text content - The post URL (if visible) - The post date -5. ONLY include posts from the last 3 days +5. ONLY include posts from the last 2 days 6. Collect as many posts as you can (aim for 5-10 per search) When done, return the data as a JSON list with keys: content, author, url, date.""", @@ -1763,8 +1775,8 @@ When done, return the data as a JSON list with keys: content, author, url, date. seen.add(key) deduped.append(p) - # Filter to last 3 days only - deduped = [p for p in deduped if _is_within_days(p.get('date', ''), 7)] + # Filter to last 2 days only + deduped = [p for p in deduped if _is_within_days(p.get('date', ''), 2)] leads = deduped[:20] if leads: diff --git a/data/ai/ai.md b/data/ai/ai.md index 5717a9f..e8909e7 100644 --- a/data/ai/ai.md +++ b/data/ai/ai.md @@ -40,12 +40,19 @@ The scraper lives at `browser-use-service/main.py` port 3008. - 6-7 non-English quick searches (Afrikaans, isiXhosa, isiZulu — 2 queries each per category) - Total: ~14 searches per scrape, completed in 2-4 minutes 4. **Quick searches** — load page, double-scroll, extract visible posts (~12-18s each) -5. **Date filter** — only posts within 7 days are considered -6. **2-pass classification (dead-accurate)**: +5. **Date filter** — only posts within **2 days** are considered. Anything older is discarded. Fresh leads only. +6. **Stealth mechanics**: + - Random viewport dimensions (1280×800 to 1920×1080) — never the same size twice + - Variable delays between searches (5-12 seconds) with mouse idle actions mixed in + - Human-like scroll patterns: scroll down, pause, sometimes scroll back up, sometimes return to top + - Canvas/WebGL/audio fingerprint spoofing via injected init scripts + - Random decoy page visits (e.g., Facebook Groups) between searches + - Profile directory is temp-copied and cleaned up after each scrape + - Detection signal monitoring (checkpoint, login pages, security challenges) +7. **2-pass classification (dead-accurate)**: - **Pass 1 (AI)**: Ollama classifies each post as LEAD or NOT using a strict prompt per category. This is the primary filter and most accurate. - **Pass 2 (Keyword)**: Only posts matching BOTH a target term AND a request term are kept. Requires multi-word phrases — standalone words like "need", "want", "help" are NOT used as they cause false positives. Aggressive reject list catches service offers, self-promotions, portfolio posts, learning-requests, and existing-site issues. - **No loose fill**: Unlike the old approach, there is NO third pass that accepts posts matching EITHER term. Every returned lead has passed both AI and/or strict keyword validation. If fewer than 5 posts pass, that means only genuine leads are returned — no noise to pad the count. -7. **Anti-detection** — random delays, human-like scrolling, user-agent rotation, proxy support 8. **Scrape timing** — 3-6 minutes for a complete run. Returns 5-10 leads with high confidence. ### Lead Categories diff --git a/rust-ai/src/AI.md b/rust-ai/src/AI.md index 5717a9f..624ce7d 100644 --- a/rust-ai/src/AI.md +++ b/rust-ai/src/AI.md @@ -40,12 +40,19 @@ The scraper lives at `browser-use-service/main.py` port 3008. - 6-7 non-English quick searches (Afrikaans, isiXhosa, isiZulu — 2 queries each per category) - Total: ~14 searches per scrape, completed in 2-4 minutes 4. **Quick searches** — load page, double-scroll, extract visible posts (~12-18s each) -5. **Date filter** — only posts within 7 days are considered -6. **2-pass classification (dead-accurate)**: +5. **Date filter** — only posts within **2 days** are considered. Anything older is discarded. Fresh leads only. +6. **Stealth mechanics**: + - Random viewport dimensions (1280×800 to 1920×1080) — never the same size twice + - Variable delays between searches (5-12 seconds) with mouse idle actions mixed in + - Human-like scroll patterns: scroll down, pause, sometimes scroll back up, sometimes return to top + - Canvas/WebGL/audio fingerprint spoofing via injected init scripts + - Random decoy page visits (e.g., Facebook Groups) between searches + - Profile directory is temp-copied and cleaned up after each scrape + - Detection signal monitoring (checkpoint, login pages, security challenges) +7. **2-pass classification (dead-accurate)**: - **Pass 1 (AI)**: Ollama classifies each post as LEAD or NOT using a strict prompt per category. This is the primary filter and most accurate. - **Pass 2 (Keyword)**: Only posts matching BOTH a target term AND a request term are kept. Requires multi-word phrases — standalone words like "need", "want", "help" are NOT used as they cause false positives. Aggressive reject list catches service offers, self-promotions, portfolio posts, learning-requests, and existing-site issues. - **No loose fill**: Unlike the old approach, there is NO third pass that accepts posts matching EITHER term. Every returned lead has passed both AI and/or strict keyword validation. If fewer than 5 posts pass, that means only genuine leads are returned — no noise to pad the count. -7. **Anti-detection** — random delays, human-like scrolling, user-agent rotation, proxy support 8. **Scrape timing** — 3-6 minutes for a complete run. Returns 5-10 leads with high confidence. ### Lead Categories @@ -76,7 +83,7 @@ Each lead returned includes: - `author` — poster's name (may include location in name) - `content` — extracted post text - `url` — direct link to the post -- `date` — when posted (filtered within 7 days) +- `date` — when posted (filtered within 2 days) - `category` — "website" or "tutor" Target is 5-10 dead-accurate leads per scrape. Quality over quantity — no loose padding.