mirror of
https://git.coastit.co.za/caitlin/CRM_ENVR.git
synced 2026-07-10 03:05:43 +02:00
Added logic to the 4 languages so it can search each language at a time
This commit is contained in:
+26
-14
@@ -711,11 +711,17 @@ async def _quick_search(page, context, query: str) -> tuple:
|
||||
if '/login' in current_url.lower():
|
||||
logger.warning("Quick search redirected to login for '%s'", query[:40])
|
||||
return page, []
|
||||
await page.wait_for_timeout(random.randint(4000, 6000))
|
||||
await page.evaluate("window.scrollBy(0, 600)")
|
||||
await page.wait_for_timeout(random.randint(2000, 4000))
|
||||
await page.evaluate("window.scrollBy(0, 600)")
|
||||
await page.wait_for_timeout(random.randint(2000, 3000))
|
||||
await page.wait_for_timeout(random.randint(3000, 7000))
|
||||
await page.evaluate(f"window.scrollBy(0, {random.randint(400, 900)})")
|
||||
await page.wait_for_timeout(random.randint(2000, 5000))
|
||||
if random.random() < 0.35:
|
||||
await page.evaluate(f"window.scrollBy(0, -{random.randint(100, 400)})")
|
||||
await page.wait_for_timeout(random.randint(1500, 3500))
|
||||
await page.evaluate(f"window.scrollBy(0, {random.randint(400, 900)})")
|
||||
await page.wait_for_timeout(random.randint(2000, 5000))
|
||||
if random.random() < 0.25:
|
||||
await page.evaluate("window.scrollTo(0, 0)")
|
||||
await page.wait_for_timeout(random.randint(1000, 3000))
|
||||
raw_articles = await _get_article_elements(page)
|
||||
posts = _extract_posts_from_elements(raw_articles, url) if raw_articles else []
|
||||
raw = await page.evaluate('document.body.innerText')
|
||||
@@ -865,7 +871,7 @@ def _parse_fb_date(block: list[str]) -> str:
|
||||
return datetime.now().strftime('%Y-%m-%d')
|
||||
|
||||
|
||||
def _is_within_days(date_str: str, max_days: int = 3) -> bool:
|
||||
def _is_within_days(date_str: str, max_days: int = 2) -> bool:
|
||||
"""Check if date is within max_days from now. Empty/unparseable = keep."""
|
||||
if not date_str:
|
||||
return True
|
||||
@@ -1442,6 +1448,7 @@ async def _scrape_with_firefox(profile_path: str, force: bool, query: str | None
|
||||
context = await pw.firefox.launch_persistent_context(
|
||||
user_data_dir=profile_dir,
|
||||
headless=True,
|
||||
viewport=random.choice(VIEWPORTS),
|
||||
firefox_user_prefs={
|
||||
"dom.webdriver.enabled": False,
|
||||
"dom.webdriver.timeout": 0,
|
||||
@@ -1503,7 +1510,9 @@ async def _scrape_with_firefox(profile_path: str, force: bool, query: str | None
|
||||
if not posts:
|
||||
continue
|
||||
if i > 0:
|
||||
await page.wait_for_timeout(random.uniform(3000, 7000))
|
||||
await page.wait_for_timeout(random.uniform(5000, 12000))
|
||||
if random.random() < 0.2:
|
||||
await random_idle(page)
|
||||
continue
|
||||
if random.random() < 0.4:
|
||||
await page.evaluate(f"window.scrollBy(0, {random.randint(-300, 300)})")
|
||||
@@ -1532,8 +1541,8 @@ async def _scrape_with_firefox(profile_path: str, force: bool, query: str | None
|
||||
seen.add(key)
|
||||
deduped.append(p)
|
||||
|
||||
# Filter to last 3 days only
|
||||
deduped = [p for p in deduped if _is_within_days(p.get('date', ''), 7)]
|
||||
# Filter to last 2 days only
|
||||
deduped = [p for p in deduped if _is_within_days(p.get('date', ''), 2)]
|
||||
|
||||
leads = deduped[:20]
|
||||
if leads:
|
||||
@@ -1589,6 +1598,7 @@ async def _scrape_with_chromium(profile_path: str, browser: str, force: bool = F
|
||||
launch_kwargs = dict(
|
||||
user_data_dir=profile_dir,
|
||||
headless=True,
|
||||
viewport=random.choice(VIEWPORTS),
|
||||
args=CHROME_LAUNCH_ARGS,
|
||||
)
|
||||
if channel:
|
||||
@@ -1649,7 +1659,9 @@ async def _scrape_with_chromium(profile_path: str, browser: str, force: bool = F
|
||||
if not posts:
|
||||
continue
|
||||
if i > 0:
|
||||
await page.wait_for_timeout(random.uniform(3000, 7000))
|
||||
await page.wait_for_timeout(random.uniform(5000, 12000))
|
||||
if random.random() < 0.2:
|
||||
await random_idle(page)
|
||||
continue
|
||||
if random.random() < 0.4:
|
||||
await page.evaluate(f"window.scrollBy(0, {random.randint(-300, 300)})")
|
||||
@@ -1678,7 +1690,7 @@ async def _scrape_with_chromium(profile_path: str, browser: str, force: bool = F
|
||||
seen.add(key)
|
||||
deduped.append(p)
|
||||
|
||||
deduped = [p for p in deduped if _is_within_days(p.get('date', ''), 7)]
|
||||
deduped = [p for p in deduped if _is_within_days(p.get('date', ''), 2)]
|
||||
leads = deduped[:20]
|
||||
if leads:
|
||||
leads = await classify_leads(leads, tutoring=tutoring)
|
||||
@@ -1734,7 +1746,7 @@ async def _scrape_with_agent(force: bool = False) -> dict:
|
||||
- The post text content
|
||||
- The post URL (if visible)
|
||||
- The post date
|
||||
5. ONLY include posts from the last 3 days
|
||||
5. ONLY include posts from the last 2 days
|
||||
6. Collect as many posts as you can (aim for 5-10 per search)
|
||||
|
||||
When done, return the data as a JSON list with keys: content, author, url, date.""",
|
||||
@@ -1763,8 +1775,8 @@ When done, return the data as a JSON list with keys: content, author, url, date.
|
||||
seen.add(key)
|
||||
deduped.append(p)
|
||||
|
||||
# Filter to last 3 days only
|
||||
deduped = [p for p in deduped if _is_within_days(p.get('date', ''), 7)]
|
||||
# Filter to last 2 days only
|
||||
deduped = [p for p in deduped if _is_within_days(p.get('date', ''), 2)]
|
||||
|
||||
leads = deduped[:20]
|
||||
if leads:
|
||||
|
||||
+10
-3
@@ -40,12 +40,19 @@ The scraper lives at `browser-use-service/main.py` port 3008.
|
||||
- 6-7 non-English quick searches (Afrikaans, isiXhosa, isiZulu — 2 queries each per category)
|
||||
- Total: ~14 searches per scrape, completed in 2-4 minutes
|
||||
4. **Quick searches** — load page, double-scroll, extract visible posts (~12-18s each)
|
||||
5. **Date filter** — only posts within 7 days are considered
|
||||
6. **2-pass classification (dead-accurate)**:
|
||||
5. **Date filter** — only posts within **2 days** are considered. Anything older is discarded. Fresh leads only.
|
||||
6. **Stealth mechanics**:
|
||||
- Random viewport dimensions (1280×800 to 1920×1080) — never the same size twice
|
||||
- Variable delays between searches (5-12 seconds) with mouse idle actions mixed in
|
||||
- Human-like scroll patterns: scroll down, pause, sometimes scroll back up, sometimes return to top
|
||||
- Canvas/WebGL/audio fingerprint spoofing via injected init scripts
|
||||
- Random decoy page visits (e.g., Facebook Groups) between searches
|
||||
- Profile directory is temp-copied and cleaned up after each scrape
|
||||
- Detection signal monitoring (checkpoint, login pages, security challenges)
|
||||
7. **2-pass classification (dead-accurate)**:
|
||||
- **Pass 1 (AI)**: Ollama classifies each post as LEAD or NOT using a strict prompt per category. This is the primary filter and most accurate.
|
||||
- **Pass 2 (Keyword)**: Only posts matching BOTH a target term AND a request term are kept. Requires multi-word phrases — standalone words like "need", "want", "help" are NOT used as they cause false positives. Aggressive reject list catches service offers, self-promotions, portfolio posts, learning-requests, and existing-site issues.
|
||||
- **No loose fill**: Unlike the old approach, there is NO third pass that accepts posts matching EITHER term. Every returned lead has passed both AI and/or strict keyword validation. If fewer than 5 posts pass, that means only genuine leads are returned — no noise to pad the count.
|
||||
7. **Anti-detection** — random delays, human-like scrolling, user-agent rotation, proxy support
|
||||
8. **Scrape timing** — 3-6 minutes for a complete run. Returns 5-10 leads with high confidence.
|
||||
|
||||
### Lead Categories
|
||||
|
||||
+11
-4
@@ -40,12 +40,19 @@ The scraper lives at `browser-use-service/main.py` port 3008.
|
||||
- 6-7 non-English quick searches (Afrikaans, isiXhosa, isiZulu — 2 queries each per category)
|
||||
- Total: ~14 searches per scrape, completed in 2-4 minutes
|
||||
4. **Quick searches** — load page, double-scroll, extract visible posts (~12-18s each)
|
||||
5. **Date filter** — only posts within 7 days are considered
|
||||
6. **2-pass classification (dead-accurate)**:
|
||||
5. **Date filter** — only posts within **2 days** are considered. Anything older is discarded. Fresh leads only.
|
||||
6. **Stealth mechanics**:
|
||||
- Random viewport dimensions (1280×800 to 1920×1080) — never the same size twice
|
||||
- Variable delays between searches (5-12 seconds) with mouse idle actions mixed in
|
||||
- Human-like scroll patterns: scroll down, pause, sometimes scroll back up, sometimes return to top
|
||||
- Canvas/WebGL/audio fingerprint spoofing via injected init scripts
|
||||
- Random decoy page visits (e.g., Facebook Groups) between searches
|
||||
- Profile directory is temp-copied and cleaned up after each scrape
|
||||
- Detection signal monitoring (checkpoint, login pages, security challenges)
|
||||
7. **2-pass classification (dead-accurate)**:
|
||||
- **Pass 1 (AI)**: Ollama classifies each post as LEAD or NOT using a strict prompt per category. This is the primary filter and most accurate.
|
||||
- **Pass 2 (Keyword)**: Only posts matching BOTH a target term AND a request term are kept. Requires multi-word phrases — standalone words like "need", "want", "help" are NOT used as they cause false positives. Aggressive reject list catches service offers, self-promotions, portfolio posts, learning-requests, and existing-site issues.
|
||||
- **No loose fill**: Unlike the old approach, there is NO third pass that accepts posts matching EITHER term. Every returned lead has passed both AI and/or strict keyword validation. If fewer than 5 posts pass, that means only genuine leads are returned — no noise to pad the count.
|
||||
7. **Anti-detection** — random delays, human-like scrolling, user-agent rotation, proxy support
|
||||
8. **Scrape timing** — 3-6 minutes for a complete run. Returns 5-10 leads with high confidence.
|
||||
|
||||
### Lead Categories
|
||||
@@ -76,7 +83,7 @@ Each lead returned includes:
|
||||
- `author` — poster's name (may include location in name)
|
||||
- `content` — extracted post text
|
||||
- `url` — direct link to the post
|
||||
- `date` — when posted (filtered within 7 days)
|
||||
- `date` — when posted (filtered within 2 days)
|
||||
- `category` — "website" or "tutor"
|
||||
|
||||
Target is 5-10 dead-accurate leads per scrape. Quality over quantity — no loose padding.
|
||||
|
||||
Reference in New Issue
Block a user