CRM_ENVIROMENT/NewStrcuture_Backup
7
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Checks done

Browse Source
This commit is contained in:
Ace
2026-06-25 16:04:05 +02:00
parent 5668a63370
commit 9bbaf70145
1 changed files with 78 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files
browser-use-service/main.py
+74 -14
View File
@@ -147,7 +147,7 @@ def extract_agent_posts(agent_result: str, page_text: str) -> list[dict]:
if isinstance(item, dict) and item.get('content'):
posts.append({
'content': item.get('content', ''),
'author': item.get('author', ''),
'author': item.get('author', '') or item.get('publisher', '') or '',
'url': item.get('url', ''),
'date': item.get('date', ''),
})
@@ -155,11 +155,20 @@ def extract_agent_posts(agent_result: str, page_text: str) -> list[dict]:
pass
if not posts:
lines = [l.strip() for l in page_text.split('\n') if l.strip()]
for line in lines:
if len(line) > 30 and not any(skip in line.lower() for skip in
['facebook', 'messenger', 'notification', 'comment', 'like', 'share']):
posts.append({'content': line[:500], 'author': '', 'url': '', 'date': ''})
lines = [l.strip() for l in agent_result.split('\n') if l.strip()]
cur = []
for l in lines:
if l.startswith(('1.', '2.', '3.', '4.', '5.', '6.', '7.', '8.', '9.', '- ', '* ')):
if cur:
combined = ' '.join(cur)
if len(combined) > 30:
posts.append({'content': combined[:500], 'author': '', 'url': '', 'date': ''})
cur = []
cur.append(l)
if cur:
combined = ' '.join(cur)
if len(combined) > 30:
posts.append({'content': combined[:500], 'author': '', 'url': '', 'date': ''})
return posts
@@ -443,6 +452,18 @@ def _parse_fb_date(block: list[str]) -> str:
pass
return datetime.now().strftime('%Y-%m-%d')
def _is_within_days(date_str: str, max_days: int = 3) -> bool:
"""Check if date is within max_days from now. Empty/unparseable = keep."""
if not date_str:
return True
try:
dt = datetime.strptime(date_str.strip(), '%Y-%m-%d')
return (datetime.now() - dt).days <= max_days
except ValueError:
return True
def _clean_fb_text(text: str) -> str:
cleaned_lines = []
for l in text.split('\n'):
@@ -475,13 +496,26 @@ def _extract_posts_from_elements(elements: list[dict], base_url: str) -> list[di
seen_texts.add(dekey)
request_score = 2 if is_request(text) else 0
post_url = el.get('url') or base_url
# Prefer JS-extracted date, fall back to text parsing
js_date = (el.get('date') or '').strip()
if js_date:
# Try ISO datetime from <time datetime>
try:
dt = datetime.fromisoformat(js_date.replace('Z', '+00:00'))
date_str = dt.strftime('%Y-%m-%d')
except (ValueError, TypeError):
date_str = _parse_fb_date([js_date])
else:
date_str = _parse_fb_date(lines)
posts.append({
"title": text[:300],
"content": text[:1000],
"author": el.get('author', ''),
"url": post_url,
"source": "facebook",
"date": _parse_fb_date(lines),
"date": date_str,
"_score": request_score,
})
posts.sort(key=lambda p: p.get('_score', 0), reverse=True)
@@ -588,24 +622,42 @@ async def _get_article_elements(page) -> list[dict]:
const key = text.substring(0, 80);
if (seenTexts.has(key)) continue;
seenTexts.add(key);
const links = el.querySelectorAll('a');
// --- Publisher name ---
let author = '';
const nameEl = el.querySelector('h4, strong, a[role="link"]');
if (nameEl) {
author = (nameEl.innerText || '').trim();
}
if (!author) {
const links = el.querySelectorAll('a');
for (const a of links) {
const t = (a.innerText || '').trim();
if (t && t.length > 2 && t.length < 80 && /^[a-zA-ZÀ-ÿ][a-zA-ZÀ-ÿ .'-]+$/.test(t)) {
if (t && t.length > 1 && t.length < 60 && /^[A-Za-zÀ-ÿ]/.test(t) && !t.includes('facebook') && !t.includes('/')) {
author = t;
break;
}
}
}
// --- Post URL ---
let postUrl = '';
for (const a of links) {
for (const a of el.querySelectorAll('a')) {
const h = (a.getAttribute('href') || '');
if (h.includes('/posts/') || h.includes('/photo/') || h.includes('/video/') || h.includes('/groups/')) {
if (h.includes('/posts/') || h.includes('/photo/') || h.includes('/video/') || h.includes('/groups/') || h.includes('/permalink/')) {
postUrl = h.startsWith('http') ? h : 'https://www.facebook.com' + h;
break;
}
}
results.push({ text, author, url: postUrl });
// --- Date from <time datetime> ---
let date = '';
const timeEl = el.querySelector('time');
if (timeEl) {
date = timeEl.getAttribute('datetime') || timeEl.getAttribute('aria-label') || '';
}
results.push({ text, author, url: postUrl, date });
}
if (results.length > 0) break;
}
@@ -804,6 +856,9 @@ async def _scrape_with_firefox(profile_path: str, force: bool) -> dict:
seen.add(key)
deduped.append(p)
# Filter to last 3 days only
deduped = [p for p in deduped if _is_within_days(p.get('date', ''), 3)]
leads = deduped[:20]
if leads:
leads = await classify_leads(leads)
@@ -847,10 +902,12 @@ async def _scrape_with_agent(force: bool = False) -> dict:
2. Use the Facebook search to find: {query}
3. Scroll through the results
4. For each relevant post, extract and list:
- The publisher/author name (who posted it)
- The post text content
- The author name
- The post URL (if visible)
5. Collect as many posts as you can (aim for 5-10 per search)
- The post date
5. ONLY include posts from the last 3 days
6. Collect as many posts as you can (aim for 5-10 per search)
When done, return the data as a JSON list with keys: content, author, url, date.""",
llm=make_ollama(num_ctx=32000, temperature=0.3),
@@ -878,6 +935,9 @@ When done, return the data as a JSON list with keys: content, author, url, date.
seen.add(key)
deduped.append(p)
# Filter to last 3 days only
deduped = [p for p in deduped if _is_within_days(p.get('date', ''), 3)]
leads = deduped[:20]
if leads:
leads = await classify_leads(leads)