Merge branch 'main' of https://git.coastit.co.za/caitlin/CRM_ENVR
This commit is contained in:
+430
-47
@@ -1,9 +1,21 @@
|
||||
import os, json, asyncio, re, shutil, sqlite3, urllib.parse, random, logging
|
||||
import os, json, asyncio, re, shutil, sqlite3, urllib.parse, random, logging, tempfile
|
||||
from datetime import datetime, timedelta
|
||||
from fastapi import FastAPI, Query
|
||||
from fastapi import FastAPI, Query, Body
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
import uvicorn
|
||||
from playwright.async_api import async_playwright
|
||||
from browser_use import Agent, Browser
|
||||
from langchain_ollama import ChatOllama
|
||||
|
||||
|
||||
def make_ollama(model: str | None = None, **kwargs) -> ChatOllama:
|
||||
"""Create ChatOllama with required attrs for browser-use Agent compatibility."""
|
||||
llm = ChatOllama(model=model or CLASSIFY_MODEL, **kwargs)
|
||||
object.__setattr__(llm, 'provider', 'ollama')
|
||||
object.__setattr__(llm, 'name', 'ChatOllama')
|
||||
object.__setattr__(llm, 'model_name', llm.model)
|
||||
return llm
|
||||
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format='%(asctime)s [%(levelname)s] %(message)s')
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -20,7 +32,137 @@ PORT = int(os.getenv("PORT", "3008"))
|
||||
OLLAMA_URL = os.getenv("OLLAMA_URL", "http://localhost:11434")
|
||||
CLASSIFY_MODEL = os.getenv("CLASSIFY_MODEL", "dolphin-llama3:8b")
|
||||
|
||||
|
||||
FX_PROFILE = os.getenv('FX_PROFILE', '')
|
||||
CHROME_PROFILE = os.getenv('CHROME_PROFILE', '')
|
||||
|
||||
CHROME_LAUNCH_ARGS = [
|
||||
'--headless=new',
|
||||
'--disable-blink-features=AutomationControlled',
|
||||
'--disable-background-networking',
|
||||
'--no-first-run',
|
||||
'--disable-sync',
|
||||
'--disable-field-trial-config',
|
||||
'--disable-background-timer-throttling',
|
||||
'--disable-backgrounding-occluded-windows',
|
||||
'--disable-breakpad',
|
||||
'--disable-component-update',
|
||||
'--disable-default-apps',
|
||||
'--disable-dev-shm-usage',
|
||||
'--disable-popup-blocking',
|
||||
'--disable-prompt-on-repost',
|
||||
'--disable-renderer-backgrounding',
|
||||
'--metrics-recording-only',
|
||||
'--no-default-browser-check',
|
||||
'--disable-extensions-http-throttling',
|
||||
'--disable-hang-monitor',
|
||||
'--disable-ipc-flooding-protection',
|
||||
]
|
||||
|
||||
|
||||
def detect_browser_from_profile(profile_path: str) -> str | None:
|
||||
"""Detect browser type from profile path. Returns 'firefox', 'chromium', or None."""
|
||||
if not profile_path:
|
||||
return None
|
||||
p = profile_path.lower().replace('\\', '/')
|
||||
if 'firefox' in p or '.default' in p or '.dev-edition' in p:
|
||||
return 'firefox'
|
||||
if 'chrome' in p or 'chromium' in p or 'edge' in p:
|
||||
return 'chromium'
|
||||
return None
|
||||
|
||||
|
||||
def copy_firefox_profile(src_path: str) -> str:
|
||||
"""Copy essential Firefox profile files to a temp dir for Playwright."""
|
||||
essential = ['cookies.sqlite', 'webappsstore.sqlite', 'permissions.sqlite']
|
||||
dst = tempfile.mkdtemp(prefix='fb_fx_profile_')
|
||||
for f_name in essential:
|
||||
s = os.path.join(src_path, f_name)
|
||||
if os.path.exists(s):
|
||||
shutil.copy2(s, os.path.join(dst, f_name))
|
||||
with open(os.path.join(dst, 'profiles.ini'), 'w') as fh:
|
||||
fh.write("[Profile0]\nName=default\nIsRelative=0\nPath=.\nDefault=yes\n")
|
||||
logger.info("Copied Firefox profile to %s", dst)
|
||||
return dst
|
||||
|
||||
|
||||
def copy_chrome_profile(user_data_dir: str, profile_dir: str = 'Default') -> str:
|
||||
"""Copy Chrome profile to temp dir for launch_persistent_context."""
|
||||
essential = ['Cookies', 'Login Data', 'Bookmarks', 'Web Data']
|
||||
dst = tempfile.mkdtemp(prefix='fb_chrome_udir_')
|
||||
src_profile = os.path.join(user_data_dir, profile_dir)
|
||||
os.makedirs(dst, exist_ok=True)
|
||||
os.makedirs(os.path.join(dst, profile_dir), exist_ok=True)
|
||||
dst_profile = os.path.join(dst, profile_dir)
|
||||
for f_name in essential:
|
||||
s = os.path.join(src_profile, f_name)
|
||||
if os.path.exists(s):
|
||||
shutil.copy2(s, os.path.join(dst_profile, f_name))
|
||||
for sub in ['Local Storage', 'Session Storage']:
|
||||
s = os.path.join(src_profile, sub)
|
||||
if os.path.isdir(s):
|
||||
shutil.copytree(s, os.path.join(dst_profile, sub), dirs_exist_ok=True)
|
||||
local_state_src = os.path.join(user_data_dir, 'Local State')
|
||||
local_state_dst = os.path.join(dst, 'Local State')
|
||||
if os.path.exists(local_state_src):
|
||||
shutil.copy2(local_state_src, local_state_dst)
|
||||
logger.info("Copied Chrome profile from %s to %s", src_profile, dst)
|
||||
return dst
|
||||
|
||||
|
||||
def ensure_ublock_extension() -> str | None:
|
||||
"""Download and extract uBlock Origin extension for ad blocking."""
|
||||
ext_dir = os.path.join(tempfile.gettempdir(), 'fb_ublock_origin')
|
||||
manifest = os.path.join(ext_dir, 'manifest.json')
|
||||
if os.path.exists(manifest):
|
||||
return ext_dir
|
||||
import urllib.request, zipfile
|
||||
crx_url = ('https://clients2.google.com/service/update2/crx'
|
||||
'?response=redirect&prodversion=133&acceptformat=crx3'
|
||||
'&x=id%3Dddkjiahejlhfcafbddmgiahcphecmpfh%26uc')
|
||||
crx_path = os.path.join(tempfile.gettempdir(), 'ublock_origin.crx')
|
||||
try:
|
||||
if not os.path.exists(crx_path):
|
||||
logger.info("Downloading uBlock Origin extension...")
|
||||
urllib.request.urlretrieve(crx_url, crx_path)
|
||||
with zipfile.ZipFile(crx_path, 'r') as z:
|
||||
z.extractall(ext_dir)
|
||||
logger.info("uBlock Origin extension ready at %s", ext_dir)
|
||||
return ext_dir
|
||||
except Exception as e:
|
||||
logger.warning("Failed to load uBlock Origin extension: %s", e)
|
||||
return None
|
||||
|
||||
|
||||
def extract_agent_posts(agent_result: str, page_text: str) -> list[dict]:
|
||||
"""Parse posts from agent output or page text fallback."""
|
||||
posts = []
|
||||
# Try JSON from agent output
|
||||
json_match = re.search(r'\[.*?\]', agent_result, re.DOTALL)
|
||||
if json_match:
|
||||
try:
|
||||
parsed = json.loads(json_match.group())
|
||||
if isinstance(parsed, list):
|
||||
for item in parsed:
|
||||
if isinstance(item, dict) and item.get('content'):
|
||||
posts.append({
|
||||
'content': item.get('content', ''),
|
||||
'author': item.get('author', ''),
|
||||
'url': item.get('url', ''),
|
||||
'date': item.get('date', ''),
|
||||
})
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
|
||||
if not posts:
|
||||
lines = [l.strip() for l in page_text.split('\n') if l.strip()]
|
||||
for line in lines:
|
||||
if len(line) > 30 and not any(skip in line.lower() for skip in
|
||||
['facebook', 'messenger', 'notification', 'comment', 'like', 'share']):
|
||||
posts.append({'content': line[:500], 'author': '', 'url': '', 'date': ''})
|
||||
|
||||
return posts
|
||||
|
||||
|
||||
BROAD_KEYWORDS = [
|
||||
"website", "web design", "web develop", "web dev",
|
||||
@@ -91,6 +233,100 @@ REQUEST_PATTERNS = [
|
||||
r"\bquote\s+(please|for|to)\b",
|
||||
]
|
||||
|
||||
def firefox_init_script() -> str:
|
||||
return r"""// Firefox Anti-Detection
|
||||
Object.defineProperty(navigator, 'webdriver', { get: () => undefined });
|
||||
Object.defineProperty(navigator, 'languages', { get: () => ['en-US', 'en'] });
|
||||
Object.defineProperty(navigator, 'platform', { get: () => 'Win32' });
|
||||
Object.defineProperty(navigator, 'hardwareConcurrency', { get: () => 8 });
|
||||
Object.defineProperty(navigator, 'maxTouchPoints', { get: () => 0 });
|
||||
Object.defineProperty(navigator, 'oscpu', { get: () => 'Windows NT 10.0; Win64; x64' });
|
||||
Object.defineProperty(navigator, 'vendor', { get: () => '' });
|
||||
Object.defineProperty(navigator, 'vendorSub', { get: () => '' });
|
||||
Object.defineProperty(navigator, 'productSub', { get: () => '20100101' });
|
||||
Object.defineProperty(navigator, 'product', { get: () => 'Gecko' });
|
||||
Object.defineProperty(navigator, 'appCodeName', { get: () => 'Mozilla' });
|
||||
Object.defineProperty(navigator, 'appName', { get: () => 'Netscape' });
|
||||
Object.defineProperty(navigator, 'appVersion', { get: () => '5.0 (Windows NT 10.0; Win64; x64; rv:130.0) Gecko/20100101 Firefox/130.0' });
|
||||
Object.defineProperty(navigator, 'userAgent', { get: () => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:130.0) Gecko/20100101 Firefox/130.0' });
|
||||
Object.defineProperty(navigator, 'buildID', { get: () => '20250101000000' });
|
||||
if (window.chrome !== undefined) { window.chrome = undefined; }
|
||||
if (navigator.permissions) {
|
||||
const origQ = navigator.permissions.query.bind(navigator.permissions);
|
||||
navigator.permissions.query = (p) => origQ(p).then(r => (['camera','microphone','geolocation','notifications'].includes(p.name) ? Object.assign(r, { state: 'prompt' }) : r));
|
||||
}
|
||||
const _gEC = HTMLCanvasElement.prototype.getContext;
|
||||
HTMLCanvasElement.prototype.getContext = function(t, ...a) {
|
||||
const ctx = _gEC.call(this, t, ...a);
|
||||
if (ctx && (t === 'webgl' || t === 'webgl2')) {
|
||||
const _gP = ctx.getParameter.bind(ctx);
|
||||
ctx.getParameter = function(p) { if (p===37445) return 'Intel Inc.'; if (p===37446) return 'Intel Iris OpenGL Engine'; if (p===7936||p===7937||p===35724) return ''; return _gP(p); };
|
||||
const _gE = ctx.getExtension.bind(ctx);
|
||||
ctx.getExtension = function(n) { if (n==='WEBGL_debug_renderer_info') return null; return _gE(n); };
|
||||
}
|
||||
return ctx;
|
||||
};
|
||||
const _tDU = HTMLCanvasElement.prototype.toDataURL;
|
||||
HTMLCanvasElement.prototype.toDataURL = function(...a) {
|
||||
const img = _tDU.apply(this, a);
|
||||
if (img.includes('image/png') && img.length > 5000) { const n = btoa(String.fromCharCode(Math.floor(Math.random()*256))); return img.slice(0,-100)+n+img.slice(-100); }
|
||||
return img;
|
||||
};
|
||||
const _gFD = AnalyserNode.prototype.getFloatFrequencyData;
|
||||
if (_gFD) { AnalyserNode.prototype.getFloatFrequencyData = function(a) { _gFD.call(this,a); for(let i=0;i<a.length;i++) a[i]+=(Math.random()-0.5)*0.02; }; }
|
||||
Object.defineProperty(screen, 'colorDepth', { get: () => 24 });
|
||||
Object.defineProperty(screen, 'pixelDepth', { get: () => 24 });
|
||||
Object.defineProperty(navigator, 'plugins', { get: () => [{name:'Widevine Content Decryption Module',filename:'widevinecdm.dll',length:1,description:'Enables Widevine licenses'},{name:'OpenH264 Video Codec',filename:'openh264.dll',length:1,description:'H.264 video codec'}] });
|
||||
Object.defineProperty(navigator, 'mimeTypes', { get: () => [{type:'application/pdf',suffixes:'pdf',description:'Portable Document Format',enabledPlugin:navigator.plugins[0]}] });"""
|
||||
|
||||
|
||||
def chromium_init_script() -> str:
|
||||
return r"""// Enhanced Chromium Anti-Detection
|
||||
Object.defineProperty(navigator, 'webdriver', { get: () => false });
|
||||
Object.defineProperty(navigator, 'languages', { get: () => ['en-US', 'en'] });
|
||||
Object.defineProperty(navigator, 'platform', { get: () => 'Win32' });
|
||||
Object.defineProperty(navigator, 'hardwareConcurrency', { get: () => 8 });
|
||||
Object.defineProperty(navigator, 'deviceMemory', { get: () => 8 });
|
||||
Object.defineProperty(navigator, 'maxTouchPoints', { get: () => 0 });
|
||||
Object.defineProperty(navigator, 'oscpu', { get: () => 'Windows NT 10.0; Win64; x64' });
|
||||
Object.defineProperty(navigator, 'vendor', { get: () => 'Google Inc.' });
|
||||
Object.defineProperty(navigator, 'vendorSub', { get: () => '' });
|
||||
Object.defineProperty(navigator, 'productSub', { get: () => '20030107' });
|
||||
Object.defineProperty(navigator, 'product', { get: () => 'Gecko' });
|
||||
Object.defineProperty(navigator, 'appCodeName', { get: () => 'Mozilla' });
|
||||
Object.defineProperty(navigator, 'appName', { get: () => 'Netscape' });
|
||||
Object.defineProperty(navigator, 'appVersion', { get: () => '5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36' });
|
||||
Object.defineProperty(navigator, 'userAgent', { get: () => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36' });
|
||||
window.chrome = { runtime: { onConnect: { addListener: () => {} }, onMessage: { addListener: () => {} }, sendMessage: () => {} }, app: { isInstalled: false, InstallState: { DISABLED: 'disabled', INSTALLED: 'installed', NOT_INSTALLED: 'not_installed' }, getDetails: () => null, getIsInstalled: () => false }, csi: () => {}, loadTimes: () => {} };
|
||||
if (navigator.permissions) {
|
||||
const origQ = navigator.permissions.query.bind(navigator.permissions);
|
||||
navigator.permissions.query = (p) => origQ(p).then(r => (['camera','microphone','geolocation','notifications'].includes(p.name) ? Object.assign(r, { state: 'prompt' }) : r));
|
||||
}
|
||||
const _gEC = HTMLCanvasElement.prototype.getContext;
|
||||
HTMLCanvasElement.prototype.getContext = function(t, ...a) {
|
||||
const ctx = _gEC.call(this, t, ...a);
|
||||
if (ctx && (t === 'webgl' || t === 'webgl2')) {
|
||||
const _gP = ctx.getParameter.bind(ctx);
|
||||
ctx.getParameter = function(p) { if (p===37445) return 'Intel Inc.'; if (p===37446) return 'Intel Iris OpenGL Engine'; if (p===7936||p===7937||p===35724) return ''; return _gP(p); };
|
||||
const _gE = ctx.getExtension.bind(ctx);
|
||||
ctx.getExtension = function(n) { if (n==='WEBGL_debug_renderer_info') return null; return _gE(n); };
|
||||
}
|
||||
return ctx;
|
||||
};
|
||||
const _tDU = HTMLCanvasElement.prototype.toDataURL;
|
||||
HTMLCanvasElement.prototype.toDataURL = function(...a) {
|
||||
const img = _tDU.apply(this, a);
|
||||
if (img.includes('image/png') && img.length > 5000) { const n = btoa(String.fromCharCode(Math.floor(Math.random()*256))); return img.slice(0,-100)+n+img.slice(-100); }
|
||||
return img;
|
||||
};
|
||||
const _gFD = AnalyserNode.prototype.getFloatFrequencyData;
|
||||
if (_gFD) { AnalyserNode.prototype.getFloatFrequencyData = function(a) { _gFD.call(this,a); for(let i=0;i<a.length;i++) a[i]+=(Math.random()-0.5)*0.02; }; }
|
||||
Object.defineProperty(screen, 'colorDepth', { get: () => 24 });
|
||||
Object.defineProperty(screen, 'pixelDepth', { get: () => 24 });
|
||||
Object.defineProperty(navigator, 'plugins', { get: () => [1,2,3,4,5].map(i => ({name:['Chrome PDF Plugin','Chrome PDF Viewer','Native Client','Widevine Content Decryption Module','Chromium PDF Viewer'][i-1],filename:'internal-pdf-viewer',length:1,description:['Portable Document Format','','Native Client Executable','Enables Widevine licenses',''][i-1]})) });
|
||||
Object.defineProperty(navigator, 'mimeTypes', { get: () => [{type:'application/pdf',suffixes:'pdf',description:'Portable Document Format',enabledPlugin:navigator.plugins[0]},{type:'text/pdf',suffixes:'pdf',description:'Portable Document Format',enabledPlugin:navigator.plugins[0]}] });"""
|
||||
|
||||
|
||||
def kw_match(text: str) -> bool:
|
||||
t = text.lower()
|
||||
return any(kw in t for kw in BROAD_KEYWORDS)
|
||||
@@ -306,6 +542,13 @@ async def human_scroll(page, steps: int = None, total_delay: float = None):
|
||||
for i in range(steps):
|
||||
scroll_dist = random.randint(200, 600)
|
||||
await page.evaluate(f"window.scrollBy(0, {scroll_dist})")
|
||||
try:
|
||||
vp = await page.evaluate('({w: window.innerWidth, h: window.innerHeight})')
|
||||
x = random.randint(100, vp['w'] - 100)
|
||||
y = random.randint(100, vp['h'] - 100)
|
||||
await page.mouse.move(x, y, steps=random.randint(15, 35))
|
||||
except Exception:
|
||||
pass
|
||||
await page.wait_for_timeout(int(step_delay * 1000))
|
||||
if random.random() < 0.3:
|
||||
await page.evaluate(f"window.scrollBy(0, -{random.randint(100, 300)})")
|
||||
@@ -421,6 +664,25 @@ async def search_facebook(page, context, query: str):
|
||||
return page, []
|
||||
return page, posts
|
||||
|
||||
DETECTION_SIGNALS = [
|
||||
'/checkpoint/', '/login.php?', 'action=security_check',
|
||||
'unusual activity', 'suspicious login', 'suspicious activity',
|
||||
'temporarily blocked', 'temporarily restricted',
|
||||
'confirm your identity', 'enter the code we sent',
|
||||
'we noticed unusual', 'help us confirm',
|
||||
'security check', 'challenge', 'please verify',
|
||||
]
|
||||
|
||||
|
||||
def check_detection_signals(page_url: str, page_text: str = '') -> str | None:
|
||||
"""Check page for Facebook detection signals. Returns signal text or None."""
|
||||
combined = (page_url + ' ' + page_text).lower()
|
||||
for signal in DETECTION_SIGNALS:
|
||||
if signal in combined:
|
||||
return signal
|
||||
return None
|
||||
|
||||
|
||||
def cleanup_chrome():
|
||||
import subprocess, signal
|
||||
try:
|
||||
@@ -429,91 +691,110 @@ def cleanup_chrome():
|
||||
pass
|
||||
|
||||
async def scrape_facebook(profile_path: str | None = None, force: bool = False) -> dict:
|
||||
cleanup_chrome()
|
||||
fb_cookies = await get_fb_cookies(profile_path)
|
||||
if not fb_cookies:
|
||||
logger.warning("No Facebook cookies available")
|
||||
return {"success": False, "leads": [], "flagged": False, "flag_reason": None, "error": "No cookies available"}
|
||||
"""Dispatcher — Firefox primary, browser-use Agent fallback."""
|
||||
effective_path = profile_path or FX_PROFILE
|
||||
browser_type = detect_browser_from_profile(effective_path)
|
||||
|
||||
if not browser_type and CHROME_PROFILE:
|
||||
browser_type = detect_browser_from_profile(CHROME_PROFILE)
|
||||
effective_path = CHROME_PROFILE
|
||||
|
||||
logger.info("Detected browser: %s (profile: %s)", browser_type or "none", effective_path)
|
||||
|
||||
# Firefox primary (raw Playwright, stealth)
|
||||
if browser_type == "firefox":
|
||||
result = await _scrape_with_firefox(effective_path, force)
|
||||
if result.get("success") or not result.get("flagged"):
|
||||
return result
|
||||
logger.warning("Firefox path failed (%s), falling back to Agent", result.get("flag_reason", "unknown"))
|
||||
return await _scrape_with_agent(force)
|
||||
|
||||
# CHROME_PROFILE set or no profile → Agent
|
||||
if browser_type == "chromium":
|
||||
return await _scrape_with_agent(force)
|
||||
|
||||
# No profile at all → try Agent (fresh Chromium)
|
||||
return await _scrape_with_agent(force)
|
||||
|
||||
|
||||
async def _scrape_with_firefox(profile_path: str, force: bool) -> dict:
|
||||
"""Scrape Facebook using Firefox + persistent real profile (no cookie injection)."""
|
||||
if not profile_path:
|
||||
return {"success": False, "leads": [], "flagged": False, "flag_reason": None, "error": "No profile path"}
|
||||
|
||||
profile_dir = None
|
||||
try:
|
||||
profile_dir = copy_firefox_profile(profile_path)
|
||||
async with async_playwright() as pw:
|
||||
browser = await pw.chromium.launch(
|
||||
context = await pw.firefox.launch_persistent_context(
|
||||
user_data_dir=profile_dir,
|
||||
headless=True,
|
||||
args=['--disable-blink-features=AutomationControlled']
|
||||
)
|
||||
viewport = random.choice(VIEWPORTS)
|
||||
context = await browser.new_context(
|
||||
user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36',
|
||||
viewport=viewport,
|
||||
firefox_user_prefs={
|
||||
"dom.webdriver.enabled": False,
|
||||
"dom.webdriver.timeout": 0,
|
||||
"useAutomationExtension": False,
|
||||
"privacy.trackingprotection.enabled": False,
|
||||
"privacy.trackingprotection.fingerprinting.enabled": False,
|
||||
"privacy.trackingprotection.cryptomining.enabled": False,
|
||||
},
|
||||
)
|
||||
pages = context.pages
|
||||
page = pages[0] if pages else await context.new_page()
|
||||
await context.add_init_script(firefox_init_script())
|
||||
|
||||
await context.add_init_script("""
|
||||
Object.defineProperty(navigator, 'webdriver', { get: () => false });
|
||||
Object.defineProperty(navigator, 'languages', { get: () => ['en-US', 'en'] });
|
||||
Object.defineProperty(navigator, 'platform', { get: () => 'Win32' });
|
||||
window.chrome = { runtime: {} };
|
||||
Object.defineProperty(navigator, 'hardwareConcurrency', { get: () => 8 });
|
||||
Object.defineProperty(navigator, 'deviceMemory', { get: () => 8 });
|
||||
""")
|
||||
for c in fb_cookies:
|
||||
try:
|
||||
await context.add_cookies([c])
|
||||
except Exception as e:
|
||||
logger.warning("Failed to inject cookie %s: %s", c.get('name'), e)
|
||||
|
||||
page = await context.new_page()
|
||||
# Navigate through google first for a legitimate Referer header
|
||||
try:
|
||||
await page.goto('https://www.google.com/', wait_until='domcontentloaded', timeout=15000)
|
||||
await page.wait_for_timeout(random.randint(1000, 3000))
|
||||
except Exception:
|
||||
logger.warning("Google navigation failed (IP may be blocked), trying Facebook directly")
|
||||
logger.warning("Google navigation failed, trying Facebook directly")
|
||||
|
||||
await page.goto('https://www.facebook.com/', wait_until='domcontentloaded', timeout=30000)
|
||||
await page.wait_for_timeout(random.randint(3000, 8000))
|
||||
|
||||
url = page.url
|
||||
if '/login' in url.lower():
|
||||
logger.warning("Facebook login page detected — account flagged")
|
||||
await browser.close()
|
||||
return {"success": False, "leads": [], "flagged": True, "flag_reason": "login_page", "error": "Facebook login page detected"}
|
||||
page_text = await page.evaluate('document.body.innerText') if '/login' in url.lower() else ''
|
||||
det = check_detection_signals(url, page_text)
|
||||
if det or '/login' in url.lower():
|
||||
logger.warning("Facebook login page detected — flag: %s", det or "login_page")
|
||||
await context.close()
|
||||
return {"success": False, "leads": [], "flagged": True, "flag_reason": det or "login_page", "error": "Facebook login page detected"}
|
||||
|
||||
# Browse feed like a human before searching
|
||||
await human_scroll(page, steps=random.randint(2, 4), total_delay=random.uniform(8, 20))
|
||||
if random.random() < 0.25:
|
||||
await page.evaluate("window.scrollTo(0, 0)")
|
||||
await page.wait_for_timeout(random.randint(2000, 5000))
|
||||
await human_scroll(page, steps=random.randint(1, 2))
|
||||
|
||||
# False start: 30% chance to idle and leave without scraping (skipped when force=true)
|
||||
if not force and random.random() < 0.3:
|
||||
await page.wait_for_timeout(random.randint(8000, 20000))
|
||||
await browser.close()
|
||||
await context.close()
|
||||
return {"success": True, "leads": [], "flagged": False, "flag_reason": None, "error": None}
|
||||
|
||||
all_posts = []
|
||||
searches = random.sample(FB_SEARCHES, k=random.randint(5, 8))
|
||||
searches = random.sample(FB_SEARCHES, k=random.randint(2, 4))
|
||||
for i, query in enumerate(searches):
|
||||
page, posts = await search_facebook(page, context, query)
|
||||
all_posts.extend(posts)
|
||||
if not posts:
|
||||
continue
|
||||
# Between searches: random break with possible small scroll
|
||||
if random.random() < 0.4:
|
||||
await page.evaluate(f"window.scrollBy(0, {random.randint(-300, 300)})")
|
||||
delay = random.uniform(8, 25)
|
||||
await page.wait_for_timeout(int(delay * 1000))
|
||||
# Tab switch: 15% chance after 2nd-3rd search
|
||||
if i == random.randint(1, 2) and random.random() < 0.15:
|
||||
if i == random.randint(0, 1) and random.random() < 0.15:
|
||||
new_page = await context.new_page()
|
||||
await new_page.goto('https://www.messenger.com/', wait_until='domcontentloaded', timeout=15000)
|
||||
await new_page.wait_for_timeout(random.randint(3000, 8000))
|
||||
try:
|
||||
await new_page.goto('https://www.facebook.com/groups/', wait_until='domcontentloaded', timeout=15000)
|
||||
await new_page.wait_for_timeout(random.randint(3000, 8000))
|
||||
except Exception:
|
||||
pass
|
||||
await new_page.close()
|
||||
page = await _ensure_page(page, context)
|
||||
|
||||
# Idle before closing — human would pause
|
||||
if random.random() < 0.5:
|
||||
await page.wait_for_timeout(random.randint(3000, 10000))
|
||||
|
||||
await browser.close()
|
||||
await context.close()
|
||||
|
||||
seen = set()
|
||||
deduped = []
|
||||
@@ -528,9 +809,89 @@ async def scrape_facebook(profile_path: str | None = None, force: bool = False)
|
||||
leads = await classify_leads(leads)
|
||||
|
||||
return {"success": True, "leads": leads[:15], "flagged": False, "flag_reason": None, "error": None}
|
||||
|
||||
except Exception as e:
|
||||
logger.error("Facebook scrape failed: %s", e)
|
||||
logger.error("Firefox scrape failed: %s", e)
|
||||
return {"success": False, "leads": [], "flagged": False, "flag_reason": None, "error": str(e)}
|
||||
finally:
|
||||
if profile_dir and os.path.exists(profile_dir):
|
||||
try:
|
||||
shutil.rmtree(profile_dir, ignore_errors=True)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
async def _scrape_with_agent(force: bool = False) -> dict:
|
||||
"""Fallback scraper — browser-use Agent + ChatOllama (free/local, Chromium)."""
|
||||
cleanup_chrome()
|
||||
profile_dir = None
|
||||
try:
|
||||
user_data_dir = None
|
||||
if CHROME_PROFILE:
|
||||
profile_dir = copy_chrome_profile(CHROME_PROFILE)
|
||||
user_data_dir = profile_dir
|
||||
|
||||
browser = Browser(
|
||||
headless=True,
|
||||
args=CHROME_LAUNCH_ARGS,
|
||||
user_data_dir=user_data_dir,
|
||||
allowed_domains=["*.facebook.com", "*.messenger.com", "www.google.com"],
|
||||
)
|
||||
await browser.start()
|
||||
|
||||
all_posts = []
|
||||
for query in random.sample(FB_SEARCHES, k=random.randint(2, 4)):
|
||||
agent = Agent(
|
||||
task=f"""You are logged into Facebook. Do the following:
|
||||
1. Navigate to facebook.com and make sure you are on the homepage
|
||||
2. Use the Facebook search to find: {query}
|
||||
3. Scroll through the results
|
||||
4. For each relevant post, extract and list:
|
||||
- The post text content
|
||||
- The author name
|
||||
- The post URL (if visible)
|
||||
5. Collect as many posts as you can (aim for 5-10 per search)
|
||||
|
||||
When done, return the data as a JSON list with keys: content, author, url, date.""",
|
||||
llm=make_ollama(num_ctx=32000, temperature=0.3),
|
||||
browser=browser,
|
||||
max_actions_per_step=3,
|
||||
use_vision=False,
|
||||
max_failures=6,
|
||||
step_timeout=180,
|
||||
)
|
||||
history = await agent.run()
|
||||
result = history.final_result() or ""
|
||||
posts = extract_agent_posts(result, "")
|
||||
all_posts.extend(posts)
|
||||
|
||||
if random.random() < 0.3:
|
||||
await asyncio.sleep(random.randint(5, 15))
|
||||
|
||||
await browser.close()
|
||||
|
||||
seen = set()
|
||||
deduped = []
|
||||
for p in all_posts:
|
||||
key = p.get('content', '')[:100]
|
||||
if key not in seen:
|
||||
seen.add(key)
|
||||
deduped.append(p)
|
||||
|
||||
leads = deduped[:20]
|
||||
if leads:
|
||||
leads = await classify_leads(leads)
|
||||
|
||||
return {"success": True, "leads": leads[:15], "flagged": False, "flag_reason": None, "error": None}
|
||||
except Exception as e:
|
||||
logger.error("Agent scrape failed: %s", e)
|
||||
return {"success": False, "leads": [], "flagged": False, "flag_reason": None, "error": str(e)}
|
||||
finally:
|
||||
if profile_dir and os.path.isdir(profile_dir):
|
||||
try:
|
||||
shutil.rmtree(profile_dir, ignore_errors=True)
|
||||
except Exception as e:
|
||||
logger.warning("Failed to clean up Chrome profile %s: %s", profile_dir, e)
|
||||
|
||||
async def ask_ollama(prompt: str) -> str:
|
||||
import httpx
|
||||
@@ -621,6 +982,28 @@ Return a JSON array like ["yes","no","yes"] matching the order above."""
|
||||
async def health():
|
||||
return {"status": "ok"}
|
||||
|
||||
@app.post("/agent/run")
|
||||
async def agent_run(task: str = Body(..., embed=True)):
|
||||
"""Run a browser-use Agent with ChatOllama (free/local)."""
|
||||
try:
|
||||
browser = Browser(headless=True, args=CHROME_LAUNCH_ARGS)
|
||||
await browser.start()
|
||||
agent = Agent(
|
||||
task=task,
|
||||
llm=make_ollama(num_ctx=32000, temperature=0.3),
|
||||
browser=browser,
|
||||
use_vision=False,
|
||||
max_actions_per_step=5,
|
||||
max_failures=8,
|
||||
step_timeout=180,
|
||||
)
|
||||
history = await agent.run()
|
||||
await browser.close()
|
||||
return {"success": True, "result": history.final_result()}
|
||||
except Exception as e:
|
||||
logger.error("Agent run failed: %s", e)
|
||||
return {"success": False, "error": str(e)}
|
||||
|
||||
@app.post("/scrape/facebook")
|
||||
async def scrape_facebook_endpoint(profile_path: str | None = Query(None), force: bool = Query(False)):
|
||||
result = await scrape_facebook(profile_path, force)
|
||||
|
||||
Reference in New Issue
Block a user