CRM_ENVIROMENT/CRM_ENVR
7
0
Fork 0
mirror of https://git.coastit.co.za/caitlin/CRM_ENVR.git synced 2026-07-10 03:05:43 +02:00
Code Issues Packages Projects Releases Wiki Activity

Added logic to the 4 languages so it can search each language at a time

Browse Source
This commit is contained in:
Ace
2026-07-07 10:08:21 +02:00
parent 0bc3ca58ed
commit d77ff2b965
3 changed files with 47 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files
browser-use-service/main.py
+26 -14
View File
@@ -711,11 +711,17 @@ async def _quick_search(page, context, query: str) -> tuple:
if '/login' in current_url.lower():
logger.warning("Quick search redirected to login for '%s'", query[:40])
return page, []
await page.wait_for_timeout(random.randint(4000, 6000))
await page.evaluate("window.scrollBy(0, 600)")
await page.wait_for_timeout(random.randint(2000, 4000))
await page.evaluate("window.scrollBy(0, 600)")
await page.wait_for_timeout(random.randint(2000, 3000))
await page.wait_for_timeout(random.randint(3000, 7000))
await page.evaluate(f"window.scrollBy(0, {random.randint(400, 900)})")
await page.wait_for_timeout(random.randint(2000, 5000))
if random.random() < 0.35:
await page.evaluate(f"window.scrollBy(0, -{random.randint(100, 400)})")
await page.wait_for_timeout(random.randint(1500, 3500))
await page.evaluate(f"window.scrollBy(0, {random.randint(400, 900)})")
await page.wait_for_timeout(random.randint(2000, 5000))
if random.random() < 0.25:
await page.evaluate("window.scrollTo(0, 0)")
await page.wait_for_timeout(random.randint(1000, 3000))
raw_articles = await _get_article_elements(page)
posts = _extract_posts_from_elements(raw_articles, url) if raw_articles else []
raw = await page.evaluate('document.body.innerText')
@@ -865,7 +871,7 @@ def _parse_fb_date(block: list[str]) -> str:
return datetime.now().strftime('%Y-%m-%d')
def _is_within_days(date_str: str, max_days: int = 3) -> bool:
def _is_within_days(date_str: str, max_days: int = 2) -> bool:
"""Check if date is within max_days from now. Empty/unparseable = keep."""
if not date_str:
return True
@@ -1442,6 +1448,7 @@ async def _scrape_with_firefox(profile_path: str, force: bool, query: str | None
context = await pw.firefox.launch_persistent_context(
user_data_dir=profile_dir,
headless=True,
viewport=random.choice(VIEWPORTS),
firefox_user_prefs={
"dom.webdriver.enabled": False,
"dom.webdriver.timeout": 0,
@@ -1503,7 +1510,9 @@ async def _scrape_with_firefox(profile_path: str, force: bool, query: str | None
if not posts:
continue
if i > 0:
await page.wait_for_timeout(random.uniform(3000, 7000))
await page.wait_for_timeout(random.uniform(5000, 12000))
if random.random() < 0.2:
await random_idle(page)
continue
if random.random() < 0.4:
await page.evaluate(f"window.scrollBy(0, {random.randint(-300, 300)})")
@@ -1532,8 +1541,8 @@ async def _scrape_with_firefox(profile_path: str, force: bool, query: str | None
seen.add(key)
deduped.append(p)
# Filter to last 3 days only
deduped = [p for p in deduped if _is_within_days(p.get('date', ''), 7)]
# Filter to last 2 days only
deduped = [p for p in deduped if _is_within_days(p.get('date', ''), 2)]
leads = deduped[:20]
if leads:
@@ -1589,6 +1598,7 @@ async def _scrape_with_chromium(profile_path: str, browser: str, force: bool = F
launch_kwargs = dict(
user_data_dir=profile_dir,
headless=True,
viewport=random.choice(VIEWPORTS),
args=CHROME_LAUNCH_ARGS,
)
if channel:
@@ -1649,7 +1659,9 @@ async def _scrape_with_chromium(profile_path: str, browser: str, force: bool = F
if not posts:
continue
if i > 0:
await page.wait_for_timeout(random.uniform(3000, 7000))
await page.wait_for_timeout(random.uniform(5000, 12000))
if random.random() < 0.2:
await random_idle(page)
continue
if random.random() < 0.4:
await page.evaluate(f"window.scrollBy(0, {random.randint(-300, 300)})")
@@ -1678,7 +1690,7 @@ async def _scrape_with_chromium(profile_path: str, browser: str, force: bool = F
seen.add(key)
deduped.append(p)
deduped = [p for p in deduped if _is_within_days(p.get('date', ''), 7)]
deduped = [p for p in deduped if _is_within_days(p.get('date', ''), 2)]
leads = deduped[:20]
if leads:
leads = await classify_leads(leads, tutoring=tutoring)
@@ -1734,7 +1746,7 @@ async def _scrape_with_agent(force: bool = False) -> dict:
- The post text content
- The post URL (if visible)
- The post date
5. ONLY include posts from the last 3 days
5. ONLY include posts from the last 2 days
6. Collect as many posts as you can (aim for 5-10 per search)
When done, return the data as a JSON list with keys: content, author, url, date.""",
@@ -1763,8 +1775,8 @@ When done, return the data as a JSON list with keys: content, author, url, date.
seen.add(key)
deduped.append(p)
# Filter to last 3 days only
deduped = [p for p in deduped if _is_within_days(p.get('date', ''), 7)]
# Filter to last 2 days only
deduped = [p for p in deduped if _is_within_days(p.get('date', ''), 2)]
leads = deduped[:20]
if leads: