caitlin/CRM_ENVR
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Updated the to use targeted jobs but its weird

Browse Source
This commit is contained in:
Ace
2026-07-01 12:15:37 +02:00
parent 878627a6ba
commit e920d4925a
5 changed files with 434 additions and 134 deletions
Download Patch File Download Diff File Expand all files Collapse all files
browser-use-service/main.py
+323 -85
View File
@@ -341,6 +341,7 @@ OFFER_PATTERNS = [
r"\bfree\s+domain\b",
r"\bweb\s+hosting\b",
r"\bfree\s+website\b",
r"\bfree\s+\w+\s+website\b",
r"\bprofessional\s+website",
r"\baffordable\s+web\b",
r"\bget\s+a\s+free\b",
@@ -365,6 +366,47 @@ OFFER_PATTERNS = [
r"\bsponsored\b",
r"\bpromoted\b",
r"\badvertisement\b",
r"\bdo\s+you\s+(need|want)\b",
r"\bwe\s+(can|will)\s+(build|design|create|develop|do)\b",
r"\bi\s+(can|will)\s+(build|design|create|develop|do)\b",
r"\bmy\s+portfolio\b",
r"\breasonable\s+(price|pricing|rate)\b",
r"\bwhatsapp\s+(me|us|at)\b",
r"\blooking\s+for\s+(a\s+)?(business|client|customer)\b",
r"\bhelp\s+your\s+business\b",
r"\bi\s+am\s+a\s+(web|freelance|designer|developer)\b",
r"\bcontact\s+me\b",
r"\bwe\s+(are|offer|provide)\s+(web|website|design|service)\b",
r"\btake\s+(the\s+|this\s+)?quiz\b",
r"\bhomeschool\b",
r"\byour\s+(home\s+)?tutor\b",
r"\blink\s+(in\s+)?(bio|comment|below)\b",
r"\bapply\s+now\b",
r"\bget\s+started\b",
r"\bfor\s+only\b",
r"\blow\s+(price|cost|rate)\b",
r"\bhit\s+me\s+up\b",
r"\bsend\s+me\s+(a\s+)?(message|dm|pm)\b",
r"\bi\s+do\s+(web|website|design|development)\b",
r"\bwe\s+do\s+(web|website|design|development)\b",
r"\bi'm\s+offering\b",
r"\bwe're\s+offering\b",
r"\bstarting\s+a\s+\w+\s+(service|business)\b",
r"\blooking\s+for\s+a\s+few\s+businesses?\b",
r"\blet('?)s\s+connect\b",
r"\bi\s+have\s+projects?\b",
r"\byour\s+business\b",
r"\bwordpress\s+development\b",
r"\be.?commerce\s+website\b",
r"\bwebsite\s+builder\b",
r"\bai\s+studio\b",
r"\bpage\s+speed\b",
r"\bguest\s+post",
r"\bguest\s+blog",
r"\bfor\s+sale\b",
r"\bselling\s+(my|a|the|this)\b",
r"\bpremium\b",
r"\bi'm\s+selling\b",
]
REQUEST_PATTERNS = [
@@ -549,6 +591,48 @@ FB_SEARCHES = [
"want to build a website for my",
]
TUTORING_SEARCHES = [
"need a tutor",
"looking for a tutor",
"tutor for my child",
"need help with homework",
"private tutor needed",
"looking for tutoring",
"need a math tutor",
"english tutor needed",
"online tutor for",
"lessons for my child",
"help my child with",
"need someone to tutor",
"looking for someone to teach",
"tutoring for my",
"need help learning",
"recommend a tutor",
"anyone know a tutor",
"need a private tutor",
"tutoring services for my child",
"need academic help",
"need a reading tutor",
"looking for piano teacher",
"need help with maths",
"need a science tutor",
"looking for language tutor",
"need programming tutor",
"coding tutor for my",
"sat tutor needed",
"exam preparation tutor",
"need a homeschool tutor",
"tutor near me for",
]
def _search_list_for_query(query: str) -> list[str]:
"""Pick the appropriate search query pool based on the search term."""
tl = query.lower()
tutoring_terms = ["tutor", "tutoring", "lessons", "homework", "teach", "learning", "child", "math", "english", "science", "exam", "homeschool", "coding", "programming", "piano", "reading"]
if any(t in tl for t in tutoring_terms):
return TUTORING_SEARCHES
return FB_SEARCHES
VIEWPORTS = [
{'width': 1280, 'height': 800},
{'width': 1366, 'height': 768},
@@ -663,6 +747,9 @@ def _clean_fb_text(text: str) -> str:
cleaned_lines.append(stripped)
return '\n'.join(cleaned_lines)
# Words that should never be treated as person names
_NON_NAMES = {"online","tutor","tutoring","school","academy","learning","urgently","looking","south","africa","philippines","australia","creative","professional","digital","services","solutions","statistics","actuarial","homeschooling","homeschool","reading","math","english","science","grade","group","page","website","design","development","agency","company","studio","graphic","technical","support","customer","guest","post","fashion","wordpress","shopify","ecommerce","business","marketing","social","media","content","strategy","seo","free","domain","hosting","sponsored","promoted","advertisement","need","want","help","please","contact","send","message","whatsapp","hire","freelance","writer","blogger","expert","specialist","consultant","freelancer","software","creators","village","town","city","university","college","institute","evolve","aesthetics","sale","selling","premium","builder","pro","people","ecommerce"}
# ── Post Extraction ──────────────────────────────────────────────────
# Two strategies for extracting posts from page content:
# 1. _extract_posts_from_elements — uses structured DOM data from _get_article_elements()
@@ -720,6 +807,25 @@ def _extract_posts_from_text(raw: str, url: str) -> list[dict]:
posts = []
seen_texts = set()
cur = []
def _find_author(candidate_lines: list[str]) -> str:
for line in reversed(candidate_lines):
ln = line.strip()
if not ln or len(ln) > 60 or len(ln) < 3:
continue
words = ln.split()
if not (1 < len(words) < 8):
continue
if not all(w[0].isalpha() and w[0].isupper() for w in words if w):
continue
lower_ln = ln.lower()
if any(kw in lower_ln for kw in BROAD_KEYWORDS) or is_offer(ln):
continue
if 'http' in ln or '/' in ln or '@' in ln:
continue
return ln
return ''
for l in lines:
words = re.findall(r'[A-Za-z]{2,}', l)
if len(words) < 2:
@@ -735,7 +841,7 @@ def _extract_posts_from_text(raw: str, url: str) -> list[dict]:
posts.append({
"title": snippet[:300],
"content": snippet[:1000],
"author": '',
"author": _find_author(cur[-3:]),
"url": url,
"source": "facebook",
"date": _parse_fb_date(cur + [l]),
@@ -759,7 +865,40 @@ def _extract_posts_from_text(raw: str, url: str) -> list[dict]:
cur.append(l)
if len(cur) > 10:
cur.pop(0)
return posts
# Post-process: scan each post's text for inline names
def _scan_inline_name(text: str) -> str:
for m in re.finditer(r'([A-Z][a-z]{2,}(?:\s+[A-Z][a-z]{2,}){1,3})[\s,;:]', text):
cand = m.group(1).strip()
if cand and 3 <= len(cand) <= 60:
lower = cand.lower()
words = lower.split()
if not any(kw in lower for kw in BROAD_KEYWORDS) and not is_offer(cand):
if not any(w in _NON_NAMES for w in words):
return cand
return ''
for p in posts:
if not p.get('author'):
txt = p.get('content') or p.get('title') or ''
name = _scan_inline_name(txt)
if name:
p['author'] = name
# Remove group posts: any post whose text looks like it came from a group
_group_suffixes = [" - South Africa", " - Australia", " PH", "Group", "help group", "info group"]
filtered = []
for p in posts:
t = (p.get('title') or p.get('content') or '')
if ' / ' in t:
continue
lower = t.lower()
skip = False
for suff in _group_suffixes:
if suff.lower() in lower:
skip = True
break
if skip:
continue
filtered.append(p)
return filtered
# ── Human-like Behavior Simulation ──────────────────────────────────
# These functions add random delays, mouse movements, and scroll patterns
@@ -811,58 +950,31 @@ async def _get_article_elements(page) -> list[dict]:
return await page.evaluate('''() => {
const results = [];
const seenTexts = new Set();
const selectors = [
'div[role="article"]',
'div[role="feed"] > div',
'div.x1yztbdb',
'div[data-pagelet]',
];
for (const sel of selectors) {
const els = document.querySelectorAll(sel);
for (const el of els) {
const text = (el.innerText || '').trim();
if (text.length < 40) continue;
const key = text.substring(0, 80);
if (seenTexts.has(key)) continue;
seenTexts.add(key);
// --- Publisher name ---
let author = '';
const nameEl = el.querySelector('h4, strong, a[role="link"]');
if (nameEl) {
author = (nameEl.innerText || '').trim();
}
if (!author) {
const links = el.querySelectorAll('a');
for (const a of links) {
const t = (a.innerText || '').trim();
if (t && t.length > 1 && t.length < 60 && /^[A-Za-zÀ-ÿ]/.test(t) && !t.includes('facebook') && !t.includes('/')) {
author = t;
break;
}
}
}
// --- Post URL ---
let postUrl = '';
for (const a of el.querySelectorAll('a')) {
const h = (a.getAttribute('href') || '');
if (h.includes('/posts/') || h.includes('/photo/') || h.includes('/video/') || h.includes('/groups/') || h.includes('/permalink/')) {
postUrl = h.startsWith('http') ? h : 'https://www.facebook.com' + h;
break;
}
}
// --- Date from <time datetime> ---
let date = '';
const timeEl = el.querySelector('time');
if (timeEl) {
date = timeEl.getAttribute('datetime') || timeEl.getAttribute('aria-label') || '';
}
results.push({ text, author, url: postUrl, date });
}
if (results.length > 0) break;
const terms = ["website","web design","web develop","need a","looking for","build my","create a","wordpress","landing page","ecommerce"];
const anchors = document.querySelectorAll('a[href*="/posts/"], a[href*="/story.php"], a[href*="/photo.php"], a[href*="/watch"], a[href*="/reel/"], a[href*="/permalink/"]');
for (const a of anchors) {
const h = a.getAttribute('href') || '';
if (!h) continue;
const cell = a.closest('div[style]') || a.parentElement;
if (!cell) continue;
const txt = (cell.innerText || '').trim();
if (txt.length < 40) continue;
const lower = txt.toLowerCase();
let matched = false;
for (const t of terms) { if (lower.includes(t)) { matched = true; break; } }
if (!matched) continue;
const key = txt.substring(0, 80);
if (seenTexts.has(key)) continue;
seenTexts.add(key);
let author = '';
const nameEl = cell.querySelector('h4, strong, a[role="link"], span[dir="auto"]');
if (nameEl) author = (nameEl.innerText || '').trim();
if (!author || author.length > 60 || author.length < 2) author = '';
let postUrl = h.startsWith('http') ? h : 'https://www.facebook.com' + h;
let date = '';
const timeEl = cell.querySelector('time');
if (timeEl) date = timeEl.getAttribute('datetime') || timeEl.getAttribute('aria-label') || '';
results.push({ text: txt, author, url: postUrl, date });
}
return results;
}''')
@@ -896,15 +1008,11 @@ async def search_facebook(page, context, query: str):
url = f'https://www.facebook.com/search/posts/?q={urllib.parse.quote(query)}'
try:
await page.goto(url, wait_until='domcontentloaded', timeout=30000)
try:
await page.wait_for_selector('div[role="article"], div[role="feed"]', timeout=15000)
except Exception:
pass
await page.wait_for_timeout(random.randint(3000, 8000))
await page.wait_for_timeout(random.randint(5000, 10000))
await human_scroll(page, steps=random.randint(2, 4), total_delay=random.uniform(6, 15))
await human_scroll(page, steps=random.randint(4, 7), total_delay=random.uniform(12, 25))
if random.random() < 0.1:
if random.random() < 0.3:
await page.evaluate("window.scrollTo(0, document.body.scrollHeight)")
await page.wait_for_timeout(random.randint(1000, 3000))
await page.evaluate("window.scrollBy(0, -300)")
@@ -919,9 +1027,100 @@ async def search_facebook(page, context, query: str):
raw_articles = await _get_article_elements(page)
posts = _extract_posts_from_elements(raw_articles, url) if raw_articles else []
if not posts:
raw = await page.evaluate('document.body.innerText')
posts = _extract_posts_from_text(raw, url)
raw = await page.evaluate('document.body.innerText')
text_posts = _extract_posts_from_text(raw, url)
existing = {(p.get('title') or p.get('content',''))[:80] for p in posts}
for tp in text_posts:
key = (tp.get('title') or tp.get('content',''))[:80]
if key not in existing:
posts.append(tp)
if posts:
# Extract author names from profile links in the DOM
try:
profiles = await page.evaluate(r'''() => {
const out = [];
const seenTxt = new Set();
for (const a of document.querySelectorAll('a[href*="/profile.php"], a[href*="/user/"], a[href*="/people/"], a[href*="/me/"]')) {
const name = (a.innerText || '').trim();
if (!name || name.length < 3 || name.length > 60) continue;
const words = name.split(' ');
if (words.length < 2 || words.length > 6) continue;
if (!/^[A-Z]/.test(name)) continue;
if (name.includes('facebook') || name.includes('/')) continue;
const cell = a.closest('div[style]') || a.parentElement;
const txt = cell ? (cell.innerText || '').substring(0, 200) : '';
if (!txt) continue;
const key = txt.substring(0, 80);
if (seenTxt.has(key)) continue;
seenTxt.add(key);
out.push({ name, textKey: key });
}
return out;
}''')
if profiles:
for p in posts:
pk = (p.get('content') or p.get('title') or '')[:80].strip()
if not pk: continue
for pr in profiles:
if pk[:30] in pr['textKey'] or pr['textKey'][:30] in pk:
if not p.get('author'):
p['author'] = pr['name']
break
except Exception:
pass
try:
meta = await page.evaluate(r'''() => {
const out = [];
const seenHref = new Set();
const all = document.querySelectorAll('a[href]');
for (const a of all) {
const h = a.href;
if (!h || seenHref.has(h)) continue;
const u = new URL(h);
if (!u.hostname.includes('facebook.com') && !u.hostname.includes('fb.com')) continue;
const path = u.pathname;
if (path === '/' || path === '/search/' || path === '/marketplace/' || path.startsWith('/messages/') || path.startsWith('/notifications/') || path.startsWith('/settings/')) continue;
const isPost = path.includes('/posts/') || path.includes('/videos/') || path.includes('/photos/') || path.startsWith('/story.php') || path.startsWith('/photo.php') || path.startsWith('/watch') || path.includes('/reel/') || path.startsWith('/permalink/');
if (!isPost) continue;
const cell = a.closest('[style*="position"], [role="article"], div[data-pagelet], div.x1yztbdb, div.x78zum5, div.x1n2onr6') || a.parentElement;
if (!cell) continue;
const txt = (cell.innerText || '').trim();
if (txt.length < 40) continue;
let author = '';
const nameEl = cell.querySelector('h4, strong, a[role="link"], span[dir="auto"]');
if (nameEl) author = (nameEl.innerText || '').trim();
if (!author || author.length > 60) author = '';
const key = txt.substring(0, 100);
seenHref.add(h);
out.push({ key, href: h, author, textMap: txt.substring(0,200) });
}
return out;
}''')
if meta:
for p in posts:
pk = (p.get('content') or p.get('title') or '')[:100].strip()
if not pk: continue
for m in meta:
mk = m.get('key') or ''
if pk[:40] in mk or mk[:40] in pk:
if m.get('href') and 'search/' not in m['href']:
p['url'] = m['href']
if m.get('author') and not p.get('author'):
p['author'] = m['author']
break
except Exception:
pass
# Clean up: remove group names passed off as authors
for p in posts:
if p.get('author'):
a = p['author']
al = a.lower()
if any(kw in al for kw in BROAD_KEYWORDS) or is_offer(a) or len(a.split()) < 2 or any(w in _NON_NAMES for w in al.split()):
p['author'] = ''
# Final filter: strip any remaining group posts
posts = [p for p in posts if not (
'/groups/' in p.get('url', '') or ' / ' in (p.get('title') or p.get('content') or '')
)]
except Exception as e:
logger.warning("Facebook search '%s' failed: %s", query, e)
return page, []
@@ -971,7 +1170,7 @@ def cleanup_chrome():
# - Auto-detect: Firefox → Opera → Chrome → Edge (first found wins)
# - If no profile found → Agent (browser-use + ChatOllama) fallback
async def scrape_facebook(profile_path: str | None = None, force: bool = False) -> dict:
async def scrape_facebook(profile_path: str | None = None, force: bool = False, query: str | None = None) -> dict:
"""Dispatcher — respect SELECTED_BROWSER, fall through on flag, then Agent."""
effective_path = profile_path or ""
browser_type = ""
@@ -1013,14 +1212,14 @@ async def scrape_facebook(profile_path: str | None = None, force: bool = False)
# Firefox path
if browser_type == "firefox":
result = await _scrape_with_firefox(effective_path, force)
result = await _scrape_with_firefox(effective_path, force, query)
if result.get("success") or not result.get("flagged"):
return result
logger.warning("Firefox flagged (%s), trying Agent", result.get("flag_reason", "unknown"))
return await _scrape_with_agent(force)
# Chromium-based (chrome / opera / edge)
result = await _scrape_with_chromium(effective_path, browser_type, force)
result = await _scrape_with_chromium(effective_path, browser_type, force, query)
if result.get("success") or not result.get("flagged"):
return result
logger.warning("%s flagged (%s), trying Agent", browser_type, result.get("flag_reason", "unknown"))
@@ -1035,7 +1234,7 @@ async def scrape_facebook(profile_path: str | None = None, force: bool = False)
# human browsing). If flagged by Facebook, returns flagged=True for the
# dispatcher to fall through to the Agent.
async def _scrape_with_firefox(profile_path: str, force: bool) -> dict:
async def _scrape_with_firefox(profile_path: str, force: bool, query: str | None = None) -> dict:
"""Scrape Facebook using Firefox + persistent real profile (no cookie injection)."""
if not profile_path:
return {"success": False, "leads": [], "flagged": False, "flag_reason": None, "error": "No profile path"}
@@ -1089,9 +1288,13 @@ async def _scrape_with_firefox(profile_path: str, force: bool) -> dict:
return {"success": True, "leads": [], "flagged": False, "flag_reason": None, "error": None}
all_posts = []
searches = random.sample(FB_SEARCHES, k=random.randint(2, 4))
for i, query in enumerate(searches):
page, posts = await search_facebook(page, context, query)
if query:
query_pool = _search_list_for_query(query)
searches = [query] + random.sample(query_pool, k=random.randint(1, 2))
else:
searches = random.sample(FB_SEARCHES, k=random.randint(2, 4))
for i, sq in enumerate(searches):
page, posts = await search_facebook(page, context, sq)
all_posts.extend(posts)
if not posts:
continue
@@ -1142,6 +1345,12 @@ async def _scrape_with_firefox(profile_path: str, force: bool) -> dict:
pass
# ── Chromium Scraper ─────────────────────────────────────────────────
# ── Chromium Scraper ─────────────────────────────────────────────────
# ── Chromium Scraper ─────────────────────────────────────────────────
# Generic scraper for Chrome/Edge/Opera. Uses the same structure as the
# Firefox scraper but with Chromium-specific launch config:
@@ -1152,7 +1361,7 @@ async def _scrape_with_firefox(profile_path: str, force: bool) -> dict:
# Anti-detection script differs slightly from Firefox variant.
# Same decoy-skip and flagged-fallback behavior as Firefox.
async def _scrape_with_chromium(profile_path: str, browser: str, force: bool = False) -> dict:
async def _scrape_with_chromium(profile_path: str, browser: str, force: bool = False, query: str | None = None) -> dict:
"""Scrape Facebook using a Chromium-based browser profile (Chrome/Edge/Opera)."""
if not profile_path:
return {"success": False, "leads": [], "flagged": False, "flag_reason": None, "error": "No profile path"}
@@ -1214,9 +1423,13 @@ async def _scrape_with_chromium(profile_path: str, browser: str, force: bool = F
return {"success": True, "leads": [], "flagged": False, "flag_reason": None, "error": None}
all_posts = []
searches = random.sample(FB_SEARCHES, k=random.randint(2, 4))
for i, query in enumerate(searches):
page, posts = await search_facebook(page, context, query)
if query:
query_pool = _search_list_for_query(query)
searches = [query] + random.sample(query_pool, k=random.randint(1, 2))
else:
searches = random.sample(FB_SEARCHES, k=random.randint(2, 4))
for i, sq in enumerate(searches):
page, posts = await search_facebook(page, context, sq)
all_posts.extend(posts)
if not posts:
continue
@@ -1449,15 +1662,35 @@ Return a JSON array like ["yes","no","yes"] matching the order above."""
"price for",
]
keyword_leads: list[dict] = []
offer_reject = [
'i build website', 'i offer web', 'i am a web developer',
'affordable web design package', 'web hosting',
'i do web design', 'i develop websites',
'do you need a', 'do you want a', 'we can build',
'i can build', 'my portfolio', 'reasonable price',
'whatsapp me', 'looking for a business', 'looking for client',
'help your business', 'i am a web', 'contact me',
'we offer web', 'we provide web',
'take the quiz', 'homeschool', 'your home tutor',
'link in bio', 'apply now', 'get started',
'for only', 'low price', 'hit me up',
'send me a message', 'i do website', 'we do website',
'we do web', 'i do web',
'website designer / web developer', 'website & software creators',
'website builders for small businesses', 'australia web designers',
'south africa', 'wix website design',
'for sale', 'selling my', 'premium',
'i\'m selling', 'i\'m offering', 'we\'re offering',
'free ecommerce', 'free website design',
'starting a', 'looking for a few businesses',
]
for r in results:
t = r['title'].lower()
has_web = any(kw in t for kw in web_terms)
has_request = any(kw in t for kw in request_terms)
if not has_web or not has_request:
continue
if any(kw in t for kw in ['i build website', 'i offer web', 'i am a web developer',
'affordable web design package', 'web hosting',
'i do web design', 'i develop websites']):
if any(kw in t for kw in offer_reject):
continue
keyword_leads.append(r)
@@ -1469,6 +1702,8 @@ Return a JSON array like ["yes","no","yes"] matching the order above."""
if key not in seen_titles:
seen_titles.add(key)
merged.append(r)
# Final sweep: strip any remaining offers or group posts from merged
merged = [r for r in merged if not any(kw in (r.get('title','') or '').lower() for kw in offer_reject)]
# Fill to 5 with loose keyword matches (at least web OR request term)
if len(merged) < 5:
@@ -1477,11 +1712,14 @@ Return a JSON array like ["yes","no","yes"] matching the order above."""
if key in seen_titles:
continue
t = r['title'].lower()
if any(kw in t for kw in web_terms) or any(kw in t for kw in request_terms):
seen_titles.add(key)
merged.append(r)
if len(merged) >= 5:
break
if not (any(kw in t for kw in web_terms) or any(kw in t for kw in request_terms)):
continue
if any(kw in t for kw in offer_reject):
continue
seen_titles.add(key)
merged.append(r)
if len(merged) >= 5:
break
logger.info("classify_leads: %d merged (%d AI + %d keyword) from %d raw", len(merged), len(ai_leads), len(keyword_leads), len(results))
return merged[:10]
@@ -1601,8 +1839,8 @@ async def agent_run(task: str = Body(..., embed=True)):
return {"success": False, "error": str(e)}
@app.post("/scrape/facebook")
async def scrape_facebook_endpoint(profile_path: str | None = Query(None), force: bool = Query(False)):
result = await scrape_facebook(profile_path, force)
async def scrape_facebook_endpoint(profile_path: str | None = Query(None), force: bool = Query(False), query: str | None = Query(None)):
result = await scrape_facebook(profile_path, force, query)
return result
if __name__ == "__main__":