diff --git a/ai-server/index.mjs b/ai-server/index.mjs index 9aded49..cd605d7 100644 --- a/ai-server/index.mjs +++ b/ai-server/index.mjs @@ -29,7 +29,7 @@ try { const PORT = parseInt(process.env.AI_PORT || "3001", 10) const HOST = process.env.AI_HOST || "0.0.0.0" const OLLAMA_URL = process.env.OLLAMA_BASE_URL || "http://localhost:11434" -const MODEL = process.env.AI_MODEL || "sam860/dolphin3-llama3.2:3b" +const MODEL = process.env.AI_MODEL || "llama3.2:3b" const DATABASE_URL = process.env.DATABASE_URL const JOBS_PATH = process.env.JOBS_PATH || path.join(ROOT, "data", "ai", "jobs.jsonl") const AI_MD_PATH = process.env.AI_MD_PATH || path.join(ROOT, "data", "ai", "ai.md") @@ -96,7 +96,55 @@ function appendToImprovementLog(entry) { } // ── Chat handler ──────────────────────────────────────────────── +async function scrapeFacebook() { + const profilePath = process.env.FX_PROFILE || "" + const urlPath = `/scrape/facebook?force=true${profilePath ? `&profile_path=${encodeURIComponent(profilePath)}` : ""}` + const logPath = "C:\\Users\\USER-PC\\AppData\\Local\\Temp\\opencode\\ai-scrape-debug.log" + try { + const body = await new Promise((resolve, reject) => { + const req = http.request({ hostname: "127.0.0.1", port: 3008, path: urlPath, method: "POST", timeout: 360000 }, (res) => { + let data = "" + res.on("data", (c) => data += c) + res.on("end", () => resolve(data)) + res.on("error", reject) + }) + req.on("timeout", () => { req.destroy(); reject(new Error("timeout")) }) + req.on("error", reject) + req.end() + }) + const data = JSON.parse(body) + return data + } catch (e) { + return null + } +} + +function formatLeads(leads) { + if (!leads || leads.length === 0) return "No leads found from the latest scrape." + let output = `**${leads.length} leads found:**\n\n` + for (let i = 0; i < leads.length; i++) { + const l = leads[i] + output += `${i + 1}. ${l.title || "No title"}\n` + if (l.author) output += ` Author: ${l.author}\n` + if (l.date) output += ` Date: ${l.date}\n` + if (l.url) output += ` URL: ${l.url}\n` + output += "\n" + } + return output.trim() +} + async function handleChat(userMessage, userId, userRole) { + const lowerMsg = userMessage.toLowerCase() + const triggerWords = ["lists", "listings", "leads", "recent leads", "pull leads", "show me leads", "show listings"] + + if (triggerWords.some(w => lowerMsg.includes(w))) { + const result = await scrapeFacebook() + if (result && result.success) { + return formatLeads(result.leads) + } + return "Scraper returned no results or encountered an error. Try again later." + } + const jobs = loadedJobs const instructions = readInstructions() diff --git a/browser-use-service/main.py b/browser-use-service/main.py index d088037..af098c5 100644 --- a/browser-use-service/main.py +++ b/browser-use-service/main.py @@ -1,9 +1,21 @@ -import os, json, asyncio, re, shutil, sqlite3, urllib.parse, random, logging +import os, json, asyncio, re, shutil, sqlite3, urllib.parse, random, logging, tempfile from datetime import datetime, timedelta -from fastapi import FastAPI, Query +from fastapi import FastAPI, Query, Body from fastapi.middleware.cors import CORSMiddleware import uvicorn from playwright.async_api import async_playwright +from browser_use import Agent, Browser +from langchain_ollama import ChatOllama + + +def make_ollama(model: str | None = None, **kwargs) -> ChatOllama: + """Create ChatOllama with required attrs for browser-use Agent compatibility.""" + llm = ChatOllama(model=model or CLASSIFY_MODEL, **kwargs) + object.__setattr__(llm, 'provider', 'ollama') + object.__setattr__(llm, 'name', 'ChatOllama') + object.__setattr__(llm, 'model_name', llm.model) + return llm + logging.basicConfig(level=logging.INFO, format='%(asctime)s [%(levelname)s] %(message)s') logger = logging.getLogger(__name__) @@ -20,7 +32,146 @@ PORT = int(os.getenv("PORT", "3008")) OLLAMA_URL = os.getenv("OLLAMA_URL", "http://localhost:11434") CLASSIFY_MODEL = os.getenv("CLASSIFY_MODEL", "dolphin-llama3:8b") + FX_PROFILE = os.getenv('FX_PROFILE', '') +CHROME_PROFILE = os.getenv('CHROME_PROFILE', '') + +CHROME_LAUNCH_ARGS = [ + '--headless=new', + '--disable-blink-features=AutomationControlled', + '--disable-background-networking', + '--no-first-run', + '--disable-sync', + '--disable-field-trial-config', + '--disable-background-timer-throttling', + '--disable-backgrounding-occluded-windows', + '--disable-breakpad', + '--disable-component-update', + '--disable-default-apps', + '--disable-dev-shm-usage', + '--disable-popup-blocking', + '--disable-prompt-on-repost', + '--disable-renderer-backgrounding', + '--metrics-recording-only', + '--no-default-browser-check', + '--disable-extensions-http-throttling', + '--disable-hang-monitor', + '--disable-ipc-flooding-protection', +] + + +def detect_browser_from_profile(profile_path: str) -> str | None: + """Detect browser type from profile path. Returns 'firefox', 'chromium', or None.""" + if not profile_path: + return None + p = profile_path.lower().replace('\\', '/') + if 'firefox' in p or '.default' in p or '.dev-edition' in p: + return 'firefox' + if 'chrome' in p or 'chromium' in p or 'edge' in p: + return 'chromium' + return None + + +def copy_firefox_profile(src_path: str) -> str: + """Copy essential Firefox profile files to a temp dir for Playwright.""" + essential = ['cookies.sqlite', 'webappsstore.sqlite', 'permissions.sqlite'] + dst = tempfile.mkdtemp(prefix='fb_fx_profile_') + for f_name in essential: + s = os.path.join(src_path, f_name) + if os.path.exists(s): + shutil.copy2(s, os.path.join(dst, f_name)) + with open(os.path.join(dst, 'profiles.ini'), 'w') as fh: + fh.write("[Profile0]\nName=default\nIsRelative=0\nPath=.\nDefault=yes\n") + logger.info("Copied Firefox profile to %s", dst) + return dst + + +def copy_chrome_profile(user_data_dir: str, profile_dir: str = 'Default') -> str: + """Copy Chrome profile to temp dir for launch_persistent_context.""" + essential = ['Cookies', 'Login Data', 'Bookmarks', 'Web Data'] + dst = tempfile.mkdtemp(prefix='fb_chrome_udir_') + src_profile = os.path.join(user_data_dir, profile_dir) + os.makedirs(dst, exist_ok=True) + os.makedirs(os.path.join(dst, profile_dir), exist_ok=True) + dst_profile = os.path.join(dst, profile_dir) + for f_name in essential: + s = os.path.join(src_profile, f_name) + if os.path.exists(s): + shutil.copy2(s, os.path.join(dst_profile, f_name)) + for sub in ['Local Storage', 'Session Storage']: + s = os.path.join(src_profile, sub) + if os.path.isdir(s): + shutil.copytree(s, os.path.join(dst_profile, sub), dirs_exist_ok=True) + local_state_src = os.path.join(user_data_dir, 'Local State') + local_state_dst = os.path.join(dst, 'Local State') + if os.path.exists(local_state_src): + shutil.copy2(local_state_src, local_state_dst) + logger.info("Copied Chrome profile from %s to %s", src_profile, dst) + return dst + + +def ensure_ublock_extension() -> str | None: + """Download and extract uBlock Origin extension for ad blocking.""" + ext_dir = os.path.join(tempfile.gettempdir(), 'fb_ublock_origin') + manifest = os.path.join(ext_dir, 'manifest.json') + if os.path.exists(manifest): + return ext_dir + import urllib.request, zipfile + crx_url = ('https://clients2.google.com/service/update2/crx' + '?response=redirect&prodversion=133&acceptformat=crx3' + '&x=id%3Dddkjiahejlhfcafbddmgiahcphecmpfh%26uc') + crx_path = os.path.join(tempfile.gettempdir(), 'ublock_origin.crx') + try: + if not os.path.exists(crx_path): + logger.info("Downloading uBlock Origin extension...") + urllib.request.urlretrieve(crx_url, crx_path) + with zipfile.ZipFile(crx_path, 'r') as z: + z.extractall(ext_dir) + logger.info("uBlock Origin extension ready at %s", ext_dir) + return ext_dir + except Exception as e: + logger.warning("Failed to load uBlock Origin extension: %s", e) + return None + + +def extract_agent_posts(agent_result: str, page_text: str) -> list[dict]: + """Parse posts from agent output or page text fallback.""" + posts = [] + # Try JSON from agent output + json_match = re.search(r'\[.*?\]', agent_result, re.DOTALL) + if json_match: + try: + parsed = json.loads(json_match.group()) + if isinstance(parsed, list): + for item in parsed: + if isinstance(item, dict) and item.get('content'): + posts.append({ + 'content': item.get('content', ''), + 'author': item.get('author', '') or item.get('publisher', '') or '', + 'url': item.get('url', ''), + 'date': item.get('date', ''), + }) + except json.JSONDecodeError: + pass + + if not posts: + lines = [l.strip() for l in agent_result.split('\n') if l.strip()] + cur = [] + for l in lines: + if l.startswith(('1.', '2.', '3.', '4.', '5.', '6.', '7.', '8.', '9.', '- ', '* ')): + if cur: + combined = ' '.join(cur) + if len(combined) > 30: + posts.append({'content': combined[:500], 'author': '', 'url': '', 'date': ''}) + cur = [] + cur.append(l) + if cur: + combined = ' '.join(cur) + if len(combined) > 30: + posts.append({'content': combined[:500], 'author': '', 'url': '', 'date': ''}) + + return posts + BROAD_KEYWORDS = [ "website", "web design", "web develop", "web dev", @@ -91,6 +242,100 @@ REQUEST_PATTERNS = [ r"\bquote\s+(please|for|to)\b", ] +def firefox_init_script() -> str: + return r"""// Firefox Anti-Detection +Object.defineProperty(navigator, 'webdriver', { get: () => undefined }); +Object.defineProperty(navigator, 'languages', { get: () => ['en-US', 'en'] }); +Object.defineProperty(navigator, 'platform', { get: () => 'Win32' }); +Object.defineProperty(navigator, 'hardwareConcurrency', { get: () => 8 }); +Object.defineProperty(navigator, 'maxTouchPoints', { get: () => 0 }); +Object.defineProperty(navigator, 'oscpu', { get: () => 'Windows NT 10.0; Win64; x64' }); +Object.defineProperty(navigator, 'vendor', { get: () => '' }); +Object.defineProperty(navigator, 'vendorSub', { get: () => '' }); +Object.defineProperty(navigator, 'productSub', { get: () => '20100101' }); +Object.defineProperty(navigator, 'product', { get: () => 'Gecko' }); +Object.defineProperty(navigator, 'appCodeName', { get: () => 'Mozilla' }); +Object.defineProperty(navigator, 'appName', { get: () => 'Netscape' }); +Object.defineProperty(navigator, 'appVersion', { get: () => '5.0 (Windows NT 10.0; Win64; x64; rv:130.0) Gecko/20100101 Firefox/130.0' }); +Object.defineProperty(navigator, 'userAgent', { get: () => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:130.0) Gecko/20100101 Firefox/130.0' }); +Object.defineProperty(navigator, 'buildID', { get: () => '20250101000000' }); +if (window.chrome !== undefined) { window.chrome = undefined; } +if (navigator.permissions) { + const origQ = navigator.permissions.query.bind(navigator.permissions); + navigator.permissions.query = (p) => origQ(p).then(r => (['camera','microphone','geolocation','notifications'].includes(p.name) ? Object.assign(r, { state: 'prompt' }) : r)); +} +const _gEC = HTMLCanvasElement.prototype.getContext; +HTMLCanvasElement.prototype.getContext = function(t, ...a) { + const ctx = _gEC.call(this, t, ...a); + if (ctx && (t === 'webgl' || t === 'webgl2')) { + const _gP = ctx.getParameter.bind(ctx); + ctx.getParameter = function(p) { if (p===37445) return 'Intel Inc.'; if (p===37446) return 'Intel Iris OpenGL Engine'; if (p===7936||p===7937||p===35724) return ''; return _gP(p); }; + const _gE = ctx.getExtension.bind(ctx); + ctx.getExtension = function(n) { if (n==='WEBGL_debug_renderer_info') return null; return _gE(n); }; + } + return ctx; +}; +const _tDU = HTMLCanvasElement.prototype.toDataURL; +HTMLCanvasElement.prototype.toDataURL = function(...a) { + const img = _tDU.apply(this, a); + if (img.includes('image/png') && img.length > 5000) { const n = btoa(String.fromCharCode(Math.floor(Math.random()*256))); return img.slice(0,-100)+n+img.slice(-100); } + return img; +}; +const _gFD = AnalyserNode.prototype.getFloatFrequencyData; +if (_gFD) { AnalyserNode.prototype.getFloatFrequencyData = function(a) { _gFD.call(this,a); for(let i=0;i 24 }); +Object.defineProperty(screen, 'pixelDepth', { get: () => 24 }); +Object.defineProperty(navigator, 'plugins', { get: () => [{name:'Widevine Content Decryption Module',filename:'widevinecdm.dll',length:1,description:'Enables Widevine licenses'},{name:'OpenH264 Video Codec',filename:'openh264.dll',length:1,description:'H.264 video codec'}] }); +Object.defineProperty(navigator, 'mimeTypes', { get: () => [{type:'application/pdf',suffixes:'pdf',description:'Portable Document Format',enabledPlugin:navigator.plugins[0]}] });""" + + +def chromium_init_script() -> str: + return r"""// Enhanced Chromium Anti-Detection +Object.defineProperty(navigator, 'webdriver', { get: () => false }); +Object.defineProperty(navigator, 'languages', { get: () => ['en-US', 'en'] }); +Object.defineProperty(navigator, 'platform', { get: () => 'Win32' }); +Object.defineProperty(navigator, 'hardwareConcurrency', { get: () => 8 }); +Object.defineProperty(navigator, 'deviceMemory', { get: () => 8 }); +Object.defineProperty(navigator, 'maxTouchPoints', { get: () => 0 }); +Object.defineProperty(navigator, 'oscpu', { get: () => 'Windows NT 10.0; Win64; x64' }); +Object.defineProperty(navigator, 'vendor', { get: () => 'Google Inc.' }); +Object.defineProperty(navigator, 'vendorSub', { get: () => '' }); +Object.defineProperty(navigator, 'productSub', { get: () => '20030107' }); +Object.defineProperty(navigator, 'product', { get: () => 'Gecko' }); +Object.defineProperty(navigator, 'appCodeName', { get: () => 'Mozilla' }); +Object.defineProperty(navigator, 'appName', { get: () => 'Netscape' }); +Object.defineProperty(navigator, 'appVersion', { get: () => '5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36' }); +Object.defineProperty(navigator, 'userAgent', { get: () => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36' }); +window.chrome = { runtime: { onConnect: { addListener: () => {} }, onMessage: { addListener: () => {} }, sendMessage: () => {} }, app: { isInstalled: false, InstallState: { DISABLED: 'disabled', INSTALLED: 'installed', NOT_INSTALLED: 'not_installed' }, getDetails: () => null, getIsInstalled: () => false }, csi: () => {}, loadTimes: () => {} }; +if (navigator.permissions) { + const origQ = navigator.permissions.query.bind(navigator.permissions); + navigator.permissions.query = (p) => origQ(p).then(r => (['camera','microphone','geolocation','notifications'].includes(p.name) ? Object.assign(r, { state: 'prompt' }) : r)); +} +const _gEC = HTMLCanvasElement.prototype.getContext; +HTMLCanvasElement.prototype.getContext = function(t, ...a) { + const ctx = _gEC.call(this, t, ...a); + if (ctx && (t === 'webgl' || t === 'webgl2')) { + const _gP = ctx.getParameter.bind(ctx); + ctx.getParameter = function(p) { if (p===37445) return 'Intel Inc.'; if (p===37446) return 'Intel Iris OpenGL Engine'; if (p===7936||p===7937||p===35724) return ''; return _gP(p); }; + const _gE = ctx.getExtension.bind(ctx); + ctx.getExtension = function(n) { if (n==='WEBGL_debug_renderer_info') return null; return _gE(n); }; + } + return ctx; +}; +const _tDU = HTMLCanvasElement.prototype.toDataURL; +HTMLCanvasElement.prototype.toDataURL = function(...a) { + const img = _tDU.apply(this, a); + if (img.includes('image/png') && img.length > 5000) { const n = btoa(String.fromCharCode(Math.floor(Math.random()*256))); return img.slice(0,-100)+n+img.slice(-100); } + return img; +}; +const _gFD = AnalyserNode.prototype.getFloatFrequencyData; +if (_gFD) { AnalyserNode.prototype.getFloatFrequencyData = function(a) { _gFD.call(this,a); for(let i=0;i 24 }); +Object.defineProperty(screen, 'pixelDepth', { get: () => 24 }); +Object.defineProperty(navigator, 'plugins', { get: () => [1,2,3,4,5].map(i => ({name:['Chrome PDF Plugin','Chrome PDF Viewer','Native Client','Widevine Content Decryption Module','Chromium PDF Viewer'][i-1],filename:'internal-pdf-viewer',length:1,description:['Portable Document Format','','Native Client Executable','Enables Widevine licenses',''][i-1]})) }); +Object.defineProperty(navigator, 'mimeTypes', { get: () => [{type:'application/pdf',suffixes:'pdf',description:'Portable Document Format',enabledPlugin:navigator.plugins[0]},{type:'text/pdf',suffixes:'pdf',description:'Portable Document Format',enabledPlugin:navigator.plugins[0]}] });""" + + def kw_match(text: str) -> bool: t = text.lower() return any(kw in t for kw in BROAD_KEYWORDS) @@ -207,6 +452,18 @@ def _parse_fb_date(block: list[str]) -> str: pass return datetime.now().strftime('%Y-%m-%d') + +def _is_within_days(date_str: str, max_days: int = 3) -> bool: + """Check if date is within max_days from now. Empty/unparseable = keep.""" + if not date_str: + return True + try: + dt = datetime.strptime(date_str.strip(), '%Y-%m-%d') + return (datetime.now() - dt).days <= max_days + except ValueError: + return True + + def _clean_fb_text(text: str) -> str: cleaned_lines = [] for l in text.split('\n'): @@ -239,13 +496,26 @@ def _extract_posts_from_elements(elements: list[dict], base_url: str) -> list[di seen_texts.add(dekey) request_score = 2 if is_request(text) else 0 post_url = el.get('url') or base_url + + # Prefer JS-extracted date, fall back to text parsing + js_date = (el.get('date') or '').strip() + if js_date: + # Try ISO datetime from