From 9bbaf70145f3791c579288ce24e745dc2cd1322c Mon Sep 17 00:00:00 2001
From: Ace <dillen@coastit.co.za>
Date: Thu, 25 Jun 2026 16:04:05 +0200
Subject: [PATCH] Checks done

---
 browser-use-service/main.py | 96 ++++++++++++++++++++++++++++++-------
 1 file changed, 78 insertions(+), 18 deletions(-)

diff --git a/browser-use-service/main.py b/browser-use-service/main.py
index a6599a3..af098c5 100644
--- a/browser-use-service/main.py
+++ b/browser-use-service/main.py
@@ -147,7 +147,7 @@ def extract_agent_posts(agent_result: str, page_text: str) -> list[dict]:
                     if isinstance(item, dict) and item.get('content'):
                         posts.append({
                             'content': item.get('content', ''),
-                            'author': item.get('author', ''),
+                            'author': item.get('author', '') or item.get('publisher', '') or '',
                             'url': item.get('url', ''),
                             'date': item.get('date', ''),
                         })
@@ -155,11 +155,20 @@ def extract_agent_posts(agent_result: str, page_text: str) -> list[dict]:
             pass
 
     if not posts:
-        lines = [l.strip() for l in page_text.split('\n') if l.strip()]
-        for line in lines:
-            if len(line) > 30 and not any(skip in line.lower() for skip in
-                    ['facebook', 'messenger', 'notification', 'comment', 'like', 'share']):
-                posts.append({'content': line[:500], 'author': '', 'url': '', 'date': ''})
+        lines = [l.strip() for l in agent_result.split('\n') if l.strip()]
+        cur = []
+        for l in lines:
+            if l.startswith(('1.', '2.', '3.', '4.', '5.', '6.', '7.', '8.', '9.', '- ', '* ')):
+                if cur:
+                    combined = ' '.join(cur)
+                    if len(combined) > 30:
+                        posts.append({'content': combined[:500], 'author': '', 'url': '', 'date': ''})
+                    cur = []
+            cur.append(l)
+        if cur:
+            combined = ' '.join(cur)
+            if len(combined) > 30:
+                posts.append({'content': combined[:500], 'author': '', 'url': '', 'date': ''})
 
     return posts
 
@@ -443,6 +452,18 @@ def _parse_fb_date(block: list[str]) -> str:
                 pass
     return datetime.now().strftime('%Y-%m-%d')
 
+
+def _is_within_days(date_str: str, max_days: int = 3) -> bool:
+    """Check if date is within max_days from now. Empty/unparseable = keep."""
+    if not date_str:
+        return True
+    try:
+        dt = datetime.strptime(date_str.strip(), '%Y-%m-%d')
+        return (datetime.now() - dt).days <= max_days
+    except ValueError:
+        return True
+
+
 def _clean_fb_text(text: str) -> str:
     cleaned_lines = []
     for l in text.split('\n'):
@@ -475,13 +496,26 @@ def _extract_posts_from_elements(elements: list[dict], base_url: str) -> list[di
         seen_texts.add(dekey)
         request_score = 2 if is_request(text) else 0
         post_url = el.get('url') or base_url
+
+        # Prefer JS-extracted date, fall back to text parsing
+        js_date = (el.get('date') or '').strip()
+        if js_date:
+            # Try ISO datetime from <time datetime>
+            try:
+                dt = datetime.fromisoformat(js_date.replace('Z', '+00:00'))
+                date_str = dt.strftime('%Y-%m-%d')
+            except (ValueError, TypeError):
+                date_str = _parse_fb_date([js_date])
+        else:
+            date_str = _parse_fb_date(lines)
+
         posts.append({
             "title": text[:300],
             "content": text[:1000],
             "author": el.get('author', ''),
             "url": post_url,
             "source": "facebook",
-            "date": _parse_fb_date(lines),
+            "date": date_str,
             "_score": request_score,
         })
     posts.sort(key=lambda p: p.get('_score', 0), reverse=True)
@@ -588,24 +622,42 @@ async def _get_article_elements(page) -> list[dict]:
                 const key = text.substring(0, 80);
                 if (seenTexts.has(key)) continue;
                 seenTexts.add(key);
-                const links = el.querySelectorAll('a');
+
+                // --- Publisher name ---
                 let author = '';
-                for (const a of links) {
-                    const t = (a.innerText || '').trim();
-                    if (t && t.length > 2 && t.length < 80 && /^[a-zA-ZÀ-ÿ][a-zA-ZÀ-ÿ .'-]+$/.test(t)) {
-                        author = t;
-                        break;
+                const nameEl = el.querySelector('h4, strong, a[role="link"]');
+                if (nameEl) {
+                    author = (nameEl.innerText || '').trim();
+                }
+                if (!author) {
+                    const links = el.querySelectorAll('a');
+                    for (const a of links) {
+                        const t = (a.innerText || '').trim();
+                        if (t && t.length > 1 && t.length < 60 && /^[A-Za-zÀ-ÿ]/.test(t) && !t.includes('facebook') && !t.includes('/')) {
+                            author = t;
+                            break;
+                        }
                     }
                 }
+
+                // --- Post URL ---
                 let postUrl = '';
-                for (const a of links) {
+                for (const a of el.querySelectorAll('a')) {
                     const h = (a.getAttribute('href') || '');
-                    if (h.includes('/posts/') || h.includes('/photo/') || h.includes('/video/') || h.includes('/groups/')) {
+                    if (h.includes('/posts/') || h.includes('/photo/') || h.includes('/video/') || h.includes('/groups/') || h.includes('/permalink/')) {
                         postUrl = h.startsWith('http') ? h : 'https://www.facebook.com' + h;
                         break;
                     }
                 }
-                results.push({ text, author, url: postUrl });
+
+                // --- Date from <time datetime> ---
+                let date = '';
+                const timeEl = el.querySelector('time');
+                if (timeEl) {
+                    date = timeEl.getAttribute('datetime') || timeEl.getAttribute('aria-label') || '';
+                }
+
+                results.push({ text, author, url: postUrl, date });
             }
             if (results.length > 0) break;
         }
@@ -804,6 +856,9 @@ async def _scrape_with_firefox(profile_path: str, force: bool) -> dict:
                     seen.add(key)
                     deduped.append(p)
 
+            # Filter to last 3 days only
+            deduped = [p for p in deduped if _is_within_days(p.get('date', ''), 3)]
+
             leads = deduped[:20]
             if leads:
                 leads = await classify_leads(leads)
@@ -847,10 +902,12 @@ async def _scrape_with_agent(force: bool = False) -> dict:
 2. Use the Facebook search to find: {query}
 3. Scroll through the results
 4. For each relevant post, extract and list:
+   - The publisher/author name (who posted it)
    - The post text content
-   - The author name  
    - The post URL (if visible)
-5. Collect as many posts as you can (aim for 5-10 per search)
+   - The post date
+5. ONLY include posts from the last 3 days
+6. Collect as many posts as you can (aim for 5-10 per search)
 
 When done, return the data as a JSON list with keys: content, author, url, date.""",
                 llm=make_ollama(num_ctx=32000, temperature=0.3),
@@ -878,6 +935,9 @@ When done, return the data as a JSON list with keys: content, author, url, date.
                 seen.add(key)
                 deduped.append(p)
 
+        # Filter to last 3 days only
+        deduped = [p for p in deduped if _is_within_days(p.get('date', ''), 3)]
+
         leads = deduped[:20]
         if leads:
             leads = await classify_leads(leads)