CRM_ENVIROMENT/Newbie_CRM
7
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Updated Ai to pull from facebook, added a loading SVG to make user wait for server too boot up, and added commands that can be used to start off the AI exstraction

Browse Source
This commit is contained in:
Ace
2026-06-24 17:17:38 +02:00
parent 8a6ad5c6c6
commit b245b352e7
7 changed files with 307 additions and 88 deletions
Download Patch File Download Diff File Expand all files Collapse all files
browser-use-service/main.py
+43 -24
View File
@@ -369,7 +369,24 @@ async def _get_article_elements(page) -> list[dict]:
return results;
}''')
async def search_facebook(page, query: str) -> list[dict]:
async def _ensure_page(page, context):
try:
await page.evaluate('1')
return page
except Exception:
logger.warning("Page was closed mid-scrape, creating a fresh page")
page = await context.new_page()
try:
await page.goto('https://www.google.com/', wait_until='domcontentloaded', timeout=15000)
await page.wait_for_timeout(random.randint(1000, 3000))
except Exception:
logger.warning("Google navigation failed during page recreation")
await page.goto('https://www.facebook.com/', wait_until='domcontentloaded', timeout=30000)
await page.wait_for_timeout(random.randint(3000, 8000))
return page
async def search_facebook(page, context, query: str):
page = await _ensure_page(page, context)
url = f'https://www.facebook.com/search/posts/?q={urllib.parse.quote(query)}'
try:
await page.goto(url, wait_until='domcontentloaded', timeout=30000)
@@ -400,9 +417,9 @@ async def search_facebook(page, query: str) -> list[dict]:
raw = await page.evaluate('document.body.innerText')
posts = _extract_posts_from_text(raw, url)
except Exception as e:
logger.warning("Facebook search failed: %s", e)
return []
return posts
logger.warning("Facebook search '%s' failed: %s", query, e)
return page, []
return page, posts
def cleanup_chrome():
import subprocess, signal
@@ -425,7 +442,7 @@ async def scrape_facebook(profile_path: str | None = None, force: bool = False)
)
viewport = random.choice(VIEWPORTS)
context = await browser.new_context(
user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:150.0) Gecko/20100101 Firefox/150.0',
user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36',
viewport=viewport,
)
@@ -445,8 +462,11 @@ async def scrape_facebook(profile_path: str | None = None, force: bool = False)
page = await context.new_page()
# Navigate through google first for a legitimate Referer header
await page.goto('https://www.google.com/', wait_until='domcontentloaded', timeout=15000)
await page.wait_for_timeout(random.randint(1000, 3000))
try:
await page.goto('https://www.google.com/', wait_until='domcontentloaded', timeout=15000)
await page.wait_for_timeout(random.randint(1000, 3000))
except Exception:
logger.warning("Google navigation failed (IP may be blocked), trying Facebook directly")
await page.goto('https://www.facebook.com/', wait_until='domcontentloaded', timeout=30000)
await page.wait_for_timeout(random.randint(3000, 8000))
@@ -472,23 +492,22 @@ async def scrape_facebook(profile_path: str | None = None, force: bool = False)
all_posts = []
searches = random.sample(FB_SEARCHES, k=random.randint(5, 8))
for i, query in enumerate(searches):
try:
posts = await search_facebook(page, query)
all_posts.extend(posts)
# Between searches: random break with possible small scroll
if random.random() < 0.4:
await page.evaluate(f"window.scrollBy(0, {random.randint(-300, 300)})")
delay = random.uniform(8, 25)
await page.wait_for_timeout(int(delay * 1000))
# Tab switch: 15% chance after 2nd-3rd search
if i == random.randint(1, 2) and random.random() < 0.15:
new_page = await context.new_page()
await new_page.goto('https://www.messenger.com/', wait_until='domcontentloaded', timeout=15000)
await new_page.wait_for_timeout(random.randint(3000, 8000))
await new_page.close()
await page.wait_for_timeout(random.randint(1000, 3000))
except Exception as e:
logger.warning("Facebook search '%s' failed: %s", query, e)
page, posts = await search_facebook(page, context, query)
all_posts.extend(posts)
if not posts:
continue
# Between searches: random break with possible small scroll
if random.random() < 0.4:
await page.evaluate(f"window.scrollBy(0, {random.randint(-300, 300)})")
delay = random.uniform(8, 25)
await page.wait_for_timeout(int(delay * 1000))
# Tab switch: 15% chance after 2nd-3rd search
if i == random.randint(1, 2) and random.random() < 0.15:
new_page = await context.new_page()
await new_page.goto('https://www.messenger.com/', wait_until='domcontentloaded', timeout=15000)
await new_page.wait_for_timeout(random.randint(3000, 8000))
await new_page.close()
page = await _ensure_page(page, context)
# Idle before closing — human would pause
if random.random() < 0.5:
browser-use-service/test_scrape.py
+21
View File
@@ -0,0 +1,21 @@
import asyncio, sys, logging
logging.basicConfig(level=logging.DEBUG, format='%(asctime)s [%(levelname)s] %(message)s')
sys.path.insert(0, '.')
from main import get_fb_cookies, scrape_facebook
async def test():
print('Step 1: get cookies')
cookies = await get_fb_cookies()
print(f'Got {len(cookies)} cookies')
if not cookies:
print('No cookies - will still try scrape')
print('Step 2: run scrape_facebook')
result = await scrape_facebook(force=True)
print(f'Result success={result["success"]} leads={len(result["leads"])} error={result.get("error")}')
if result.get('leads'):
for l in result['leads'][:3]:
print(f' - {l["title"][:100]}')
if result.get('error'):
print(f' Error: {result["error"]}')
asyncio.run(test())