diff --git a/ai-server/index.mjs b/ai-server/index.mjs index 14813e8..4ffb1f5 100644 --- a/ai-server/index.mjs +++ b/ai-server/index.mjs @@ -1,21 +1,62 @@ +// ── CRM AI Server ────────────────────────────────────────────────── +// Provides: +// - Chat API (POST /ai/chat) — routes user messages to Ollama for sales coaching +// - Setup wizard endpoints (GET /setup/status, POST /setup/profile, etc.) +// - Combined /status endpoint for splash page health polling +// - Configuration routes (GET/POST /ai/instructions, GET /ai/jobs) +// - Model pull support (POST /setup/ollama/pull) +// +// This is a zero-dependency Node.js HTTP server (no Express needed). + import http from "node:http" import fs from "node:fs" import path from "node:path" +import { spawn } from "node:child_process" import { fileURLToPath } from "node:url" const __dirname = path.dirname(fileURLToPath(import.meta.url)) const ROOT = path.resolve(__dirname, "..") +// ── Load .env.local ────────────────────────────────────────────── +// Reads key=value pairs and sets them as process.env so they're +// available throughout the server. Ignores comments and blank lines. +// Values with matching quotes are unquoted. +try { + const envPath = path.join(ROOT, ".env.local") + const envContent = fs.readFileSync(envPath, "utf-8") + for (const line of envContent.split("\n")) { + const trimmed = line.trim() + if (!trimmed || trimmed.startsWith("#")) continue + const eqIdx = trimmed.indexOf("=") + if (eqIdx === -1) continue + const k = trimmed.substring(0, eqIdx).trim() + let v = trimmed.substring(eqIdx + 1).trim() + if ((v.startsWith('"') && v.endsWith('"')) || (v.startsWith("'") && v.endsWith("'"))) v = v.slice(1, -1) + if (!process.env[k]) process.env[k] = v + } + console.log("Loaded .env.local") +} catch { + // .env.local may not exist (first run), which is fine +} + // ── Config from env ───────────────────────────────────────────── const PORT = parseInt(process.env.AI_PORT || "3001", 10) const HOST = process.env.AI_HOST || "0.0.0.0" const OLLAMA_URL = process.env.OLLAMA_BASE_URL || "http://localhost:11434" -const MODEL = process.env.AI_MODEL || "sam860/dolphin3-llama3.2:3b" +const MODEL = process.env.AI_MODEL || "llama3.2:3b" const DATABASE_URL = process.env.DATABASE_URL const JOBS_PATH = process.env.JOBS_PATH || path.join(ROOT, "data", "ai", "jobs.jsonl") const AI_MD_PATH = process.env.AI_MD_PATH || path.join(ROOT, "data", "ai", "ai.md") +// ── Setup state ────────────────────────────────────────────────── +// Tracks the Ollama model pull process so the setup wizard can +// poll for download progress. +let pullProcess = null +let pullProgress = { status: "idle", progress: 0, message: "" } + // ── Job loading ───────────────────────────────────────────────── +// Loads job categories from a JSONL file (one JSON object per line). +// Used as context for the AI sales coach chat responses. function loadJobs() { try { const content = fs.readFileSync(JOBS_PATH, "utf-8") @@ -40,6 +81,8 @@ function loadJobs() { } // ── ai.md management ──────────────────────────────────────────── +// ai.md is a Markdown file containing system instructions for the AI. +// It can be read, written, or appended to via the API. function readInstructions() { try { return fs.readFileSync(AI_MD_PATH, "utf-8") @@ -54,6 +97,7 @@ function writeInstructions(content) { } function appendToImprovementLog(entry) { + // Adds a timestamped entry to the ## Improvement Log section of ai.md const current = readInstructions() const timestamp = new Date().toISOString().replace("T", " ").substring(0, 16) const logEntry = `\n- ${timestamp} — ${entry}` @@ -77,7 +121,60 @@ function appendToImprovementLog(entry) { } // ── Chat handler ──────────────────────────────────────────────── +// scrapeFacebook() calls the scraper service (port 3008) to get leads. +// handleChat() processes user messages — triggers lead scraping when +// the user asks for "leads" or "listings", otherwise routes to Ollama +// for AI-powered sales coaching. +async function scrapeFacebook() { + const profilePath = process.env.FX_PROFILE || "" + const urlPath = `/scrape/facebook?force=true${profilePath ? `&profile_path=${encodeURIComponent(profilePath)}` : ""}` + try { + const body = await new Promise((resolve, reject) => { + const req = http.request({ hostname: "127.0.0.1", port: 3008, path: urlPath, method: "POST", timeout: 360000 }, (res) => { + let data = "" + res.on("data", (c) => data += c) + res.on("end", () => resolve(data)) + res.on("error", reject) + }) + req.on("timeout", () => { req.destroy(); reject(new Error("timeout")) }) + req.on("error", reject) + req.end() + }) + const data = JSON.parse(body) + return data + } catch (e) { + return null + } +} + +function formatLeads(leads) { + if (!leads || leads.length === 0) return "No leads found from the latest scrape." + let output = `**${leads.length} leads found:**\n\n` + for (let i = 0; i < leads.length; i++) { + const l = leads[i] + output += `${i + 1}. ${l.title || "No title"}\n` + if (l.author) output += ` Author: ${l.author}\n` + if (l.date) output += ` Date: ${l.date}\n` + if (l.url) output += ` URL: ${l.url}\n` + output += "\n" + } + return output.trim() +} + async function handleChat(userMessage, userId, userRole) { + // If the user asks for leads, trigger the scraper + const lowerMsg = userMessage.toLowerCase() + const triggerWords = ["lists", "listings", "leads", "recent leads", "pull leads", "show me leads", "show listings"] + + if (triggerWords.some(w => lowerMsg.includes(w))) { + const result = await scrapeFacebook() + if (result && result.success) { + return formatLeads(result.leads) + } + return "Scraper returned no results or encountered an error. Try again later." + } + + // Otherwise, build a system prompt with job context and send to Ollama const jobs = loadedJobs const instructions = readInstructions() @@ -117,7 +214,7 @@ Provide concise, actionable sales advice. When asked about a specific job catego const data = await ollamaRes.json() const responseText = data.message?.content || "" - // Try to persist to DB (best-effort) + // Persist conversation to PostgreSQL (best-effort — table may not exist yet) try { if (pgPool && userId) { await pgPool.query( @@ -133,6 +230,8 @@ Provide concise, actionable sales advice. When asked about a specific job catego } // ── PG pool (lazy init) ──────────────────────────────────────── +// PostgreSQL connection pool for storing conversation history. +// Lazy-initialized so the server starts even without a DB. let pgPool = null async function initPg() { if (!DATABASE_URL) return @@ -174,8 +273,9 @@ function parseURL(req) { return { pathname: url.pathname, searchParams: url.searchParams } } +// ── HTTP Server ───────────────────────────────────────────────── const server = http.createServer(async (req, res) => { - // CORS + // CORS headers — allow the Next.js frontend (port 3006) to call us res.setHeader("Access-Control-Allow-Origin", "*") res.setHeader("Access-Control-Allow-Methods", "GET, POST, OPTIONS") res.setHeader("Access-Control-Allow-Headers", "Content-Type") @@ -189,23 +289,236 @@ const server = http.createServer(async (req, res) => { const { pathname } = parseURL(req) try { + // GET /splash — loading screen + if (req.method === "GET" && pathname === "/splash") { + const splashPath = path.join(ROOT, "splash.html") + if (fs.existsSync(splashPath)) { + const html = fs.readFileSync(splashPath, "utf-8") + res.writeHead(200, { "Content-Type": "text/html" }) + res.end(html) + return + } + sendJSON(res, 200, { status: "ok" }) + return + } + // GET /health if (req.method === "GET" && pathname === "/health") { return sendJSON(res, 200, { status: "ok", model: MODEL }) } - // GET /ai/jobs + // GET /status — combined health of all services + // Used by the splash page to check if AI, Scraper, and Frontend are ready. + // Polls each service internally to avoid cross-origin CORS issues. + if (req.method === "GET" && pathname === "/status") { + const { default: http } = await import("http") + const results = { ai: true } + // Check scraper (port 3008) + try { + await new Promise((resolve, reject) => { + const r = http.get("http://127.0.0.1:3008/health", { timeout: 3000 }, (res) => { res.resume(); resolve() }) + r.on("error", reject) + }) + results.scraper = true + } catch { results.scraper = false } + // Check frontend (port 3006) + try { + await new Promise((resolve, reject) => { + const r = http.get("http://127.0.0.1:3006", { timeout: 3000 }, (res) => { res.resume(); resolve() }) + r.on("error", reject) + }) + results.frontend = true + } catch { results.frontend = false } + return sendJSON(res, 200, results) + } + + // ── Setup endpoints ───────────────────────────────────────── + + // GET /setup/status — check environment + // Called by the splash page on boot. Returns info about: + // - Ollama availability + // - Model presence + // - Detected browsers with login status + // - Whether this is a first run (wizard needed) + if (req.method === "GET" && pathname === "/setup/status") { + const envExists = fs.existsSync(path.join(ROOT, ".env.local")) + + // Ollama check + let ollamaRunning = false + try { + await fetch(`${OLLAMA_URL}/api/tags`, { signal: AbortSignal.timeout(3000) }) + ollamaRunning = true + } catch {} + + // Model check + let modelAvailable = false + if (ollamaRunning) { + try { + const r = await fetch(`${OLLAMA_URL}/api/show`, { + method: "POST", body: JSON.stringify({ name: MODEL }), + signal: AbortSignal.timeout(5000), + }) + modelAvailable = r.ok + } catch {} + } + + // Detect all browsers via scraper + let browsers = { firefox: { path: null }, opera: { path: null }, chrome: { path: null }, edge: { path: null } } + let facebookLoggedIn = false + let selectedBrowser = process.env.SELECTED_BROWSER || "" + + try { + await fetch("http://127.0.0.1:3008/health", { signal: AbortSignal.timeout(2000) }) + const profiles = await (await fetch("http://127.0.0.1:3008/setup/profile", { signal: AbortSignal.timeout(5000) })).json() + for (const [b, p] of Object.entries(profiles)) { + if (p) browsers[b] = { path: p } + } + // Check login for the selected browser first, then try all + const detectedList = Object.entries(browsers).filter(([, v]) => v.path) + for (const [b, v] of detectedList) { + try { + const r = await fetch("http://127.0.0.1:3008/setup/check-login", { + method: "POST", headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ browser: b, profile_path: v.path }), + signal: AbortSignal.timeout(20000), + }) + if (r.ok) { + const d = await r.json() + browsers[b].logged_in = d.logged_in === true + if (d.logged_in && !facebookLoggedIn) { + facebookLoggedIn = true + if (!selectedBrowser) selectedBrowser = b + } + } + } catch {} + } + } catch {} + + const anyDetected = Object.values(browsers).some(v => v.path) + // first_run = any setup step is incomplete + const firstRun = !envExists || !ollamaRunning || !anyDetected || !facebookLoggedIn || !modelAvailable + + return sendJSON(res, 200, { + first_run: firstRun, + env_exists: envExists, + ollama_running: ollamaRunning, + model_available: modelAvailable, + model_name: MODEL, + selected_browser: selectedBrowser, + browsers, + facebook_logged_in: facebookLoggedIn, + }) + } + + // POST /setup/profile — save selected browser + path to .env.local + // Called by the setup wizard when the user confirms their browser choice. + // Writes SELECTED_BROWSER and the matching profile env var to .env.local. + if (req.method === "POST" && pathname === "/setup/profile") { + const body = await parseBody(req) + const browserName = (body.browser || "").trim().toLowerCase() + const profilePath = (body.path || "").trim() + if (!browserName || !["firefox", "opera", "chrome", "edge"].includes(browserName)) + return sendJSON(res, 400, { error: "Valid browser required (firefox/opera/chrome/edge)" }) + if (!profilePath) + return sendJSON(res, 400, { error: "Path required" }) + + const envKey = browserName === "firefox" ? "FX_PROFILE" + : browserName === "opera" ? "OPERA_PROFILE" + : browserName === "edge" ? "EDGE_PROFILE" + : "CHROME_PROFILE" + + const envPath = path.join(ROOT, ".env.local") + let content = "" + try { content = fs.readFileSync(envPath, "utf-8") } catch {} + let lines = content.split("\n") + // Update or add SELECTED_BROWSER + let foundSel = false + for (let i = 0; i < lines.length; i++) { + if (lines[i].trim().startsWith("SELECTED_BROWSER=")) { + lines[i] = `SELECTED_BROWSER=${browserName}` + foundSel = true + break + } + } + if (!foundSel) lines.push(`SELECTED_BROWSER=${browserName}`) + // Update or add browser profile + let foundProf = false + for (let i = 0; i < lines.length; i++) { + if (lines[i].trim().startsWith(`${envKey}=`)) { + lines[i] = `${envKey}=${profilePath}` + foundProf = true + break + } + } + if (!foundProf) lines.push(`${envKey}=${profilePath}`) + fs.writeFileSync(envPath, lines.join("\n"), "utf-8") + process.env.SELECTED_BROWSER = browserName + process.env[envKey] = profilePath + return sendJSON(res, 200, { success: true, browser: browserName, path: profilePath }) + } + + // POST /setup/check-login — proxy to scraper, accepts browser + profile_path + // The splash page calls this (via the AI server) to verify Facebook login status. + if (req.method === "POST" && pathname === "/setup/check-login") { + const body = await parseBody(req) + const browserName = (body.browser || "").trim().toLowerCase() || process.env.SELECTED_BROWSER || "" + const profilePath = (body.profile_path || "").trim() + + if (!profilePath) return sendJSON(res, 200, { logged_in: false, reason: "no_profile" }) + try { + const r = await fetch("http://127.0.0.1:3008/setup/check-login", { + method: "POST", headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ browser: browserName, profile_path: profilePath }), + signal: AbortSignal.timeout(20000), + }) + if (r.ok) { const d = await r.json(); return sendJSON(res, 200, d) } + } catch {} + return sendJSON(res, 200, { logged_in: false, reason: "scraper_unavailable" }) + } + + // POST /setup/ollama/pull — start pulling the model + // Spawns "ollama pull" as a child process. The setup wizard polls + // the progress endpoint to show a download progress bar. + if (req.method === "POST" && pathname === "/setup/ollama/pull") { + if (pullProcess) return sendJSON(res, 200, { status: "already_running" }) + pullProgress = { status: "downloading", progress: 0, message: "Starting..." } + const isWin = process.platform === "win32" + const cmd = isWin ? "ollama.exe" : "ollama" + pullProcess = spawn(cmd, ["pull", MODEL], { stdio: ["ignore", "pipe", "pipe"] }) + + pullProcess.stdout.on("data", (data) => { + const text = data.toString() + pullProgress.message = text.trim() + // Extract percentage from patterns like "pulling xxxx... 45%" + const m = text.match(/(\d+)%/) + if (m) pullProgress.progress = parseInt(m[1], 10) + }) + pullProcess.on("close", (code) => { + pullProcess = null + pullProgress.status = code === 0 ? "done" : "failed" + if (code === 0) pullProgress.progress = 100 + }) + return sendJSON(res, 200, { status: "started" }) + } + + // GET /setup/ollama/pull/progress + // Returns current download progress for the setup wizard. + if (req.method === "GET" && pathname === "/setup/ollama/pull/progress") { + return sendJSON(res, 200, pullProgress) + } + + // GET /ai/jobs — return loaded job categories if (req.method === "GET" && pathname === "/ai/jobs") { return sendJSON(res, 200, { jobs: loadedJobs }) } - // GET /ai/instructions + // GET /ai/instructions — return current ai.md content if (req.method === "GET" && pathname === "/ai/instructions") { const instructions = readInstructions() return sendJSON(res, 200, { success: true, instructions }) } - // POST /ai/instructions + // POST /ai/instructions — update ai.md or append improvement log entry if (req.method === "POST" && pathname === "/ai/instructions") { const body = await parseBody(req) if (body.content) { @@ -219,9 +532,27 @@ const server = http.createServer(async (req, res) => { }) } - // POST /ai/chat + // POST /ai/chat — main AI chat endpoint + // Accepts { message, user_id?, user_role? } and returns AI response. + // user_role must be "sales", "admin", or "super_admin" if provided. if (req.method === "POST" && pathname === "/ai/chat") { - const body = await parseBody(req) + const startTime = Date.now() + const chunks = [] + req.on("data", c => chunks.push(c)) + req.on("end", () => { + const rawBody = Buffer.concat(chunks).toString() + try { + const body = JSON.parse(rawBody) + processRequest(req, res, body, startTime) + } catch { + sendJSON(res, 400, { error: "Invalid JSON" }) + } + }) + return + } + +// Separate handler for /ai/chat (defined here due to hoisting within the IIFE) +async function processRequest(req, res, body, startTime) { const { message, user_id, user_role } = body if (!message) { @@ -237,7 +568,7 @@ const server = http.createServer(async (req, res) => { return sendJSON(res, 200, { response }) } - // 404 + // 404 fallback sendJSON(res, 404, { error: "Not found" }) } catch (err) { console.error("Request error:", err) diff --git a/browser-use-service/main.py b/browser-use-service/main.py index 12bc69a..abac579 100644 --- a/browser-use-service/main.py +++ b/browser-use-service/main.py @@ -1,10 +1,28 @@ -import os, json, asyncio, re, shutil, sqlite3, urllib.parse, random, logging +# ── Imports ────────────────────────────────────────────────────────── +import os, json, asyncio, re, shutil, sqlite3, urllib.parse, random, logging, tempfile from datetime import datetime, timedelta -from fastapi import FastAPI, Query +from fastapi import FastAPI, Query, Body from fastapi.middleware.cors import CORSMiddleware import uvicorn from playwright.async_api import async_playwright +from browser_use import Agent, Browser +from langchain_ollama import ChatOllama + +# ── Helpers ────────────────────────────────────────────────────────── +def make_ollama(model: str | None = None, **kwargs) -> ChatOllama: + """ + Create ChatOllama with required attrs for browser-use Agent compatibility. + The browser-use Agent expects llm.name, llm.model_name, and llm.provider attrs. + """ + llm = ChatOllama(model=model or CLASSIFY_MODEL, **kwargs) + object.__setattr__(llm, 'provider', 'ollama') + object.__setattr__(llm, 'name', 'ChatOllama') + object.__setattr__(llm, 'model_name', llm.model) + return llm + + +# ── Logging & App Setup ────────────────────────────────────────────── logging.basicConfig(level=logging.INFO, format='%(asctime)s [%(levelname)s] %(message)s') logger = logging.getLogger(__name__) @@ -17,10 +35,287 @@ app.add_middleware( ) PORT = int(os.getenv("PORT", "3008")) +# ── AI / Ollama Config ─────────────────────────────────────────────── OLLAMA_URL = os.getenv("OLLAMA_URL", "http://localhost:11434") CLASSIFY_MODEL = os.getenv("CLASSIFY_MODEL", "dolphin-llama3:8b") + +# ── Env Vars (set via .env.local) ────────────────────────────────────── +# These store detected browser profile paths from the setup wizard. FX_PROFILE = os.getenv('FX_PROFILE', '') +CHROME_PROFILE = os.getenv('CHROME_PROFILE', '') +OPERA_PROFILE = os.getenv('OPERA_PROFILE', '') +EDGE_PROFILE = os.getenv('EDGE_PROFILE', '') +# SELECTED_BROWSER determines which browser the scrape dispatcher tries first. +# Values: "firefox", "chrome", "opera", "edge", or "" (auto-detect all). +SELECTED_BROWSER = os.getenv('SELECTED_BROWSER', '') + +# ── Chromium Launch Args ────────────────────────────────────────────── +# These flags reduce detectable automation signals in Chrome/Edge/Opera. +# They disable background services, sync, updates, throttling, popups, etc. +CHROME_LAUNCH_ARGS = [ + '--headless=new', + '--disable-blink-features=AutomationControlled', + '--disable-background-networking', + '--no-first-run', + '--disable-sync', + '--disable-field-trial-config', + '--disable-background-timer-throttling', + '--disable-backgrounding-occluded-windows', + '--disable-breakpad', + '--disable-component-update', + '--disable-default-apps', + '--disable-dev-shm-usage', + '--disable-popup-blocking', + '--disable-prompt-on-repost', + '--disable-renderer-backgrounding', + '--metrics-recording-only', + '--no-default-browser-check', + '--disable-extensions-http-throttling', + '--disable-hang-monitor', + '--disable-ipc-flooding-protection', +] + + +# ── Profile Detection ──────────────────────────────────────────────── +# All find_*_profile() functions return the parent directory (User Data or equivalent), +# NOT the "Default" subdirectory. This is important because copy_chrome_profile() +# appends "Default" internally, so returning the full Default path would cause +# duplicated path segments (e.g. ".../User Data/Default/Default"). + +def detect_browser_from_profile(profile_path: str) -> str | None: + """ + Infer browser type from the profile path string. + Checks for keywords like "firefox", "opera", "edge", "chrome" in the path. + Returns "firefox", "chrome", "opera", "edge", or None. + """ + if not profile_path: + return None + p = profile_path.lower().replace('\\', '/') + if 'firefox' in p or '.default' in p or '.dev-edition' in p: + return 'firefox' + if 'opera' in p: + return 'opera' + if 'edge' in p: + return 'edge' + if 'chrome' in p or 'chromium' in p: + return 'chrome' + return None + + +def find_firefox_profile() -> str | None: + """Auto-detect Firefox profile directory cross-platform. + Scans the Profiles dir and returns the first .default or .dev-edition folder found, + preferring "default-release" profiles.""" + import sys + home = os.path.expanduser("~") + candidates = [] + + if sys.platform == "win32": + appdata = os.environ.get("APPDATA", "") + if appdata: + profiles_dir = os.path.join(appdata, "Mozilla", "Firefox", "Profiles") + else: + profiles_dir = os.path.join(home, "AppData", "Roaming", "Mozilla", "Firefox", "Profiles") + elif sys.platform == "darwin": + profiles_dir = os.path.join(home, "Library", "Application Support", "Firefox", "Profiles") + else: + profiles_dir = os.path.join(home, ".mozilla", "firefox") + + if os.path.isdir(profiles_dir): + for entry in os.listdir(profiles_dir): + full = os.path.join(profiles_dir, entry) + if os.path.isdir(full) and (".default" in entry.lower() or ".dev-edition" in entry.lower()): + candidates.append(full) + candidates.sort(key=lambda p: "default-release" in p.lower(), reverse=True) + if candidates: + logger.info("Auto-detected Firefox profile: %s", candidates[0]) + return candidates[0] + return None + + +def find_chrome_profile() -> str | None: + """Auto-detect Chrome profile (User Data dir) cross-platform. + Returns the User Data parent dir (NOT the Default subdirectory).""" + import sys + home = os.path.expanduser("~") + + if sys.platform == "win32": + base = os.path.join(os.environ.get("LOCALAPPDATA", ""), "Google", "Chrome", "User Data") + elif sys.platform == "darwin": + base = os.path.join(home, "Library", "Application Support", "Google", "Chrome") + else: + base = os.path.join(home, ".config", "google-chrome") + + if os.path.isdir(os.path.join(base, "Default")): + logger.info("Auto-detected Chrome profile: %s", base) + return base + return None + + +def find_opera_profile() -> str | None: + """Auto-detect Opera profile directory cross-platform. + Opera stores its profile at the "Opera Stable" level (no "User Data/Default" + nesting like Chrome/Edge), so this returns the profile root directly.""" + import sys + home = os.path.expanduser("~") + + if sys.platform == "win32": + base = os.path.join(os.environ.get("APPDATA", ""), "Opera Software", "Opera Stable") + elif sys.platform == "darwin": + base = os.path.join(home, "Library", "Application Support", "com.operasoftware.Opera") + else: + base = os.path.join(home, ".config", "opera") + + if os.path.isdir(base) and os.listdir(base): + logger.info("Auto-detected Opera profile: %s", base) + return base + return None + + +def find_edge_profile() -> str | None: + """Auto-detect Edge profile (User Data dir) cross-platform. + Returns the User Data parent dir (NOT the Default subdirectory).""" + import sys + home = os.path.expanduser("~") + + if sys.platform == "win32": + base = os.path.join(os.environ.get("LOCALAPPDATA", ""), "Microsoft", "Edge", "User Data") + elif sys.platform == "darwin": + base = os.path.join(home, "Library", "Application Support", "Microsoft Edge") + else: + base = os.path.join(home, ".config", "microsoft-edge") + + if os.path.isdir(os.path.join(base, "Default")): + logger.info("Auto-detected Edge profile: %s", base) + return base + return None + + +# ── Profile Copying ───────────────────────────────────────────────── +# Both functions copy essential cookies/login/storage files from the user's +# real browser profile to a temp directory. Playwright then uses these temp +# profiles via launch_persistent_context(), preserving Facebook login state +# without modifying the user's actual profile. + +def copy_firefox_profile(src_path: str) -> str: + """Copy essential Firefox profile files (cookies, localStorage, permissions) to a temp dir for Playwright. + Also creates a minimal profiles.ini so Firefox treats it as a valid profile.""" + essential = ['cookies.sqlite', 'webappsstore.sqlite', 'permissions.sqlite'] + dst = tempfile.mkdtemp(prefix='fb_fx_profile_') + for f_name in essential: + s = os.path.join(src_path, f_name) + if os.path.exists(s): + shutil.copy2(s, os.path.join(dst, f_name)) + with open(os.path.join(dst, 'profiles.ini'), 'w') as fh: + fh.write("[Profile0]\nName=default\nIsRelative=0\nPath=.\nDefault=yes\n") + logger.info("Copied Firefox profile to %s", dst) + return dst + + +def copy_chrome_profile(user_data_dir: str, profile_dir: str = 'Default') -> str: + """Copy Chromium-based profile (Chrome/Edge/Opera) to a temp dir. + Copies Cookies, Login Data, Bookmarks, Web Data, Local Storage, Session Storage, and Local State. + Args: + user_data_dir: Parent directory (e.g. ".../Chrome/User Data") + profile_dir: Subdirectory name, typically "Default" (default) + """ + essential = ['Cookies', 'Login Data', 'Bookmarks', 'Web Data'] + dst = tempfile.mkdtemp(prefix='fb_chrome_udir_') + src_profile = os.path.join(user_data_dir, profile_dir) + os.makedirs(dst, exist_ok=True) + os.makedirs(os.path.join(dst, profile_dir), exist_ok=True) + dst_profile = os.path.join(dst, profile_dir) + for f_name in essential: + s = os.path.join(src_profile, f_name) + if os.path.exists(s): + shutil.copy2(s, os.path.join(dst_profile, f_name)) + for sub in ['Local Storage', 'Session Storage']: + s = os.path.join(src_profile, sub) + if os.path.isdir(s): + shutil.copytree(s, os.path.join(dst_profile, sub), dirs_exist_ok=True) + local_state_src = os.path.join(user_data_dir, 'Local State') + local_state_dst = os.path.join(dst, 'Local State') + if os.path.exists(local_state_src): + shutil.copy2(local_state_src, local_state_dst) + logger.info("Copied Chrome profile from %s to %s", src_profile, dst) + return dst + + +def ensure_ublock_extension() -> str | None: + """Download and extract uBlock Origin extension for ad blocking via Chromium. + Cached after first download to avoid re-downloading on every scrape.""" + ext_dir = os.path.join(tempfile.gettempdir(), 'fb_ublock_origin') + manifest = os.path.join(ext_dir, 'manifest.json') + if os.path.exists(manifest): + return ext_dir + import urllib.request, zipfile + crx_url = ('https://clients2.google.com/service/update2/crx' + '?response=redirect&prodversion=133&acceptformat=crx3' + '&x=id%3Dddkjiahejlhfcafbddmgiahcphecmpfh%26uc') + crx_path = os.path.join(tempfile.gettempdir(), 'ublock_origin.crx') + try: + if not os.path.exists(crx_path): + logger.info("Downloading uBlock Origin extension...") + urllib.request.urlretrieve(crx_url, crx_path) + with zipfile.ZipFile(crx_path, 'r') as z: + z.extractall(ext_dir) + logger.info("uBlock Origin extension ready at %s", ext_dir) + return ext_dir + except Exception as e: + logger.warning("Failed to load uBlock Origin extension: %s", e) + return None + + +# ── Agent Output Parsing ────────────────────────────────────────────── +# When the browser-use Agent fallback is used, it returns natural language +# output containing post listings. This function tries to extract structured +# data from that output, first attempting JSON parsing, then falling back to +# a line-by-line heuristic (numbered or bulleted items). + +def extract_agent_posts(agent_result: str, page_text: str) -> list[dict]: + """Parse posts from agent output or page text fallback.""" + posts = [] + json_match = re.search(r'\[.*?\]', agent_result, re.DOTALL) + if json_match: + try: + parsed = json.loads(json_match.group()) + if isinstance(parsed, list): + for item in parsed: + if isinstance(item, dict) and item.get('content'): + posts.append({ + 'content': item.get('content', ''), + 'author': item.get('author', '') or item.get('publisher', '') or '', + 'url': item.get('url', ''), + 'date': item.get('date', ''), + }) + except json.JSONDecodeError: + pass + + if not posts: + lines = [l.strip() for l in agent_result.split('\n') if l.strip()] + cur = [] + for l in lines: + if l.startswith(('1.', '2.', '3.', '4.', '5.', '6.', '7.', '8.', '9.', '- ', '* ')): + if cur: + combined = ' '.join(cur) + if len(combined) > 30: + posts.append({'content': combined[:500], 'author': '', 'url': '', 'date': ''}) + cur = [] + cur.append(l) + if cur: + combined = ' '.join(cur) + if len(combined) > 30: + posts.append({'content': combined[:500], 'author': '', 'url': '', 'date': ''}) + + return posts + + +# ── Keyword Filters for Lead Detection ────────────────────────────── +# BROAD_KEYWORDS — catch any post that might relate to web development needs. +# OFFER_PATTERNS — regex patterns matching service offers (which we ignore). +# REQUEST_PATTERNS — regex patterns matching people asking for help/services. +# Together these form a two-pass filter + AI classification pipeline. BROAD_KEYWORDS = [ "website", "web design", "web develop", "web dev", @@ -37,10 +332,180 @@ BROAD_KEYWORDS = [ "want someone", "need help with", ] +OFFER_PATTERNS = [ + r"\bfree\s+domain\b", + r"\bweb\s+hosting\b", + r"\bfree\s+website\b", + r"\bprofessional\s+website", + r"\baffordable\s+web\b", + r"\bget\s+a\s+free\b", + r"\bsee\s+more\b", + r"\bbook\s+your\b", + r"\bhire\s+a\s+freelancer\b", + r"\bcontent\s+strategy\b", + r"\bdigital\s+marketing\b", + r"\bseo\s+service", + r"\bsocial\s+media\b", + r"\blimited\s+time\s+offer\b", + r"\bspecial\s+offer\b", + r"\bsign\s+up\s+now\b", + r"\br\d[\d,.]", + r"\bstarting\s+at\b", + r"\bper\s+month\b", + r"\bmonthly\b", + r"\bfreelancer\s+marketplace\b", + r"\bfirst\s+order\b", + r"\bdomain\s+name\b", + r"\bregister\s+your\s+domain\b", + r"\bsponsored\b", + r"\bpromoted\b", + r"\badvertisement\b", +] + +REQUEST_PATTERNS = [ + r"\bi\s+need\b", + r"\bi\s+need\s+someone\b", + r"\bi\s+need\s+a\b", + r"\bi\s+need\s+an\b", + r"\bi\s+want\b", + r"\bi\s+would\s+like\b", + r"\bi'm\s+looking\b", + r"\bi\s+am\s+looking\b", + r"\blooking\s+for\b", + r"\blooking\s+to\b", + r"\bwho\s+can\b", + r"\bcan\s+someone\b", + r"\bhelp\s+me\b", + r"\bneed\s+help\b", + r"\bwould\s+like\s+to\b", + r"\bwant\s+to\b", + r"\banyone\s+know\b", + r"\bdoes\s+anyone\b", + r"\brecommend\b", + r"\bsuggestion\b", + r"\bquote\s+(please|for|to)\b", +] + +# ── Anti-Detection Scripts ──────────────────────────────────────────── +# These JS scripts are injected into every page via add_init_script(). +# They override navigator properties (webdriver, userAgent, plugins, etc.) +# and WebGL/canvas/audio fingerprints to make Playwright harder to detect. +# Firefox and Chromium variants differ slightly (e.g. Chrome needs the +# "window.chrome" object; Firefox must NOT have it). + +def firefox_init_script() -> str: + return r"""// Firefox Anti-Detection +Object.defineProperty(navigator, 'webdriver', { get: () => undefined }); +Object.defineProperty(navigator, 'languages', { get: () => ['en-US', 'en'] }); +Object.defineProperty(navigator, 'platform', { get: () => 'Win32' }); +Object.defineProperty(navigator, 'hardwareConcurrency', { get: () => 8 }); +Object.defineProperty(navigator, 'maxTouchPoints', { get: () => 0 }); +Object.defineProperty(navigator, 'oscpu', { get: () => 'Windows NT 10.0; Win64; x64' }); +Object.defineProperty(navigator, 'vendor', { get: () => '' }); +Object.defineProperty(navigator, 'vendorSub', { get: () => '' }); +Object.defineProperty(navigator, 'productSub', { get: () => '20100101' }); +Object.defineProperty(navigator, 'product', { get: () => 'Gecko' }); +Object.defineProperty(navigator, 'appCodeName', { get: () => 'Mozilla' }); +Object.defineProperty(navigator, 'appName', { get: () => 'Netscape' }); +Object.defineProperty(navigator, 'appVersion', { get: () => '5.0 (Windows NT 10.0; Win64; x64; rv:130.0) Gecko/20100101 Firefox/130.0' }); +Object.defineProperty(navigator, 'userAgent', { get: () => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:130.0) Gecko/20100101 Firefox/130.0' }); +Object.defineProperty(navigator, 'buildID', { get: () => '20250101000000' }); +if (window.chrome !== undefined) { window.chrome = undefined; } +if (navigator.permissions) { + const origQ = navigator.permissions.query.bind(navigator.permissions); + navigator.permissions.query = (p) => origQ(p).then(r => (['camera','microphone','geolocation','notifications'].includes(p.name) ? Object.assign(r, { state: 'prompt' }) : r)); +} +const _gEC = HTMLCanvasElement.prototype.getContext; +HTMLCanvasElement.prototype.getContext = function(t, ...a) { + const ctx = _gEC.call(this, t, ...a); + if (ctx && (t === 'webgl' || t === 'webgl2')) { + const _gP = ctx.getParameter.bind(ctx); + ctx.getParameter = function(p) { if (p===37445) return 'Intel Inc.'; if (p===37446) return 'Intel Iris OpenGL Engine'; if (p===7936||p===7937||p===35724) return ''; return _gP(p); }; + const _gE = ctx.getExtension.bind(ctx); + ctx.getExtension = function(n) { if (n==='WEBGL_debug_renderer_info') return null; return _gE(n); }; + } + return ctx; +}; +const _tDU = HTMLCanvasElement.prototype.toDataURL; +HTMLCanvasElement.prototype.toDataURL = function(...a) { + const img = _tDU.apply(this, a); + if (img.includes('image/png') && img.length > 5000) { const n = btoa(String.fromCharCode(Math.floor(Math.random()*256))); return img.slice(0,-100)+n+img.slice(-100); } + return img; +}; +const _gFD = AnalyserNode.prototype.getFloatFrequencyData; +if (_gFD) { AnalyserNode.prototype.getFloatFrequencyData = function(a) { _gFD.call(this,a); for(let i=0;i 24 }); +Object.defineProperty(screen, 'pixelDepth', { get: () => 24 }); +Object.defineProperty(navigator, 'plugins', { get: () => [{name:'Widevine Content Decryption Module',filename:'widevinecdm.dll',length:1,description:'Enables Widevine licenses'},{name:'OpenH264 Video Codec',filename:'openh264.dll',length:1,description:'H.264 video codec'}] }); +Object.defineProperty(navigator, 'mimeTypes', { get: () => [{type:'application/pdf',suffixes:'pdf',description:'Portable Document Format',enabledPlugin:navigator.plugins[0]}] });""" + + +def chromium_init_script() -> str: + return r"""// Enhanced Chromium Anti-Detection +Object.defineProperty(navigator, 'webdriver', { get: () => false }); +Object.defineProperty(navigator, 'languages', { get: () => ['en-US', 'en'] }); +Object.defineProperty(navigator, 'platform', { get: () => 'Win32' }); +Object.defineProperty(navigator, 'hardwareConcurrency', { get: () => 8 }); +Object.defineProperty(navigator, 'deviceMemory', { get: () => 8 }); +Object.defineProperty(navigator, 'maxTouchPoints', { get: () => 0 }); +Object.defineProperty(navigator, 'oscpu', { get: () => 'Windows NT 10.0; Win64; x64' }); +Object.defineProperty(navigator, 'vendor', { get: () => 'Google Inc.' }); +Object.defineProperty(navigator, 'vendorSub', { get: () => '' }); +Object.defineProperty(navigator, 'productSub', { get: () => '20030107' }); +Object.defineProperty(navigator, 'product', { get: () => 'Gecko' }); +Object.defineProperty(navigator, 'appCodeName', { get: () => 'Mozilla' }); +Object.defineProperty(navigator, 'appName', { get: () => 'Netscape' }); +Object.defineProperty(navigator, 'appVersion', { get: () => '5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36' }); +Object.defineProperty(navigator, 'userAgent', { get: () => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36' }); +window.chrome = { runtime: { onConnect: { addListener: () => {} }, onMessage: { addListener: () => {} }, sendMessage: () => {} }, app: { isInstalled: false, InstallState: { DISABLED: 'disabled', INSTALLED: 'installed', NOT_INSTALLED: 'not_installed' }, getDetails: () => null, getIsInstalled: () => false }, csi: () => {}, loadTimes: () => {} }; +if (navigator.permissions) { + const origQ = navigator.permissions.query.bind(navigator.permissions); + navigator.permissions.query = (p) => origQ(p).then(r => (['camera','microphone','geolocation','notifications'].includes(p.name) ? Object.assign(r, { state: 'prompt' }) : r)); +} +const _gEC = HTMLCanvasElement.prototype.getContext; +HTMLCanvasElement.prototype.getContext = function(t, ...a) { + const ctx = _gEC.call(this, t, ...a); + if (ctx && (t === 'webgl' || t === 'webgl2')) { + const _gP = ctx.getParameter.bind(ctx); + ctx.getParameter = function(p) { if (p===37445) return 'Intel Inc.'; if (p===37446) return 'Intel Iris OpenGL Engine'; if (p===7936||p===7937||p===35724) return ''; return _gP(p); }; + const _gE = ctx.getExtension.bind(ctx); + ctx.getExtension = function(n) { if (n==='WEBGL_debug_renderer_info') return null; return _gE(n); }; + } + return ctx; +}; +const _tDU = HTMLCanvasElement.prototype.toDataURL; +HTMLCanvasElement.prototype.toDataURL = function(...a) { + const img = _tDU.apply(this, a); + if (img.includes('image/png') && img.length > 5000) { const n = btoa(String.fromCharCode(Math.floor(Math.random()*256))); return img.slice(0,-100)+n+img.slice(-100); } + return img; +}; +const _gFD = AnalyserNode.prototype.getFloatFrequencyData; +if (_gFD) { AnalyserNode.prototype.getFloatFrequencyData = function(a) { _gFD.call(this,a); for(let i=0;i 24 }); +Object.defineProperty(screen, 'pixelDepth', { get: () => 24 }); +Object.defineProperty(navigator, 'plugins', { get: () => [1,2,3,4,5].map(i => ({name:['Chrome PDF Plugin','Chrome PDF Viewer','Native Client','Widevine Content Decryption Module','Chromium PDF Viewer'][i-1],filename:'internal-pdf-viewer',length:1,description:['Portable Document Format','','Native Client Executable','Enables Widevine licenses',''][i-1]})) }); +Object.defineProperty(navigator, 'mimeTypes', { get: () => [{type:'application/pdf',suffixes:'pdf',description:'Portable Document Format',enabledPlugin:navigator.plugins[0]},{type:'text/pdf',suffixes:'pdf',description:'Portable Document Format',enabledPlugin:navigator.plugins[0]}] });""" + + +# ── Text Classification Utilities ──────────────────────────────────── def kw_match(text: str) -> bool: t = text.lower() return any(kw in t for kw in BROAD_KEYWORDS) +def is_request(text: str) -> bool: + """Detect if text contains a request for services (e.g. "I need", "looking for").""" + t = text.lower() + return any(re.search(p, t) for p in REQUEST_PATTERNS) + +def is_offer(text: str) -> bool: + """Detect if text is an offer/advertisement (we skip these, not leads).""" + t = text.lower() + return any(re.search(p, t) for p in OFFER_PATTERNS) + +# ── Facebook Search Queries ─────────────────────────────────────────── +# These are the search terms the scraper will use on Facebook. +# They target people looking for web development / website services. +# Each run picks 2-4 random queries to vary behavior and reduce detection. FB_SEARCHES = [ "looking for web developer", "need a website designed", @@ -50,10 +515,19 @@ FB_SEARCHES = [ "looking for someone to create website", "need ecommerce website built", "want to hire web developer", - "website quote please", "need wordpress website", "I need a website for my business", "need a site for my business", + "looking for website designer South Africa", + "need website for my small business", + "who can build me a website", + "want to create a website for my business", + "looking for affordable web design", + "need a web developer South Africa", + "help me build a website", + "need someone to design my website", + "looking for web designer near me", + "need a website for my startup", ] VIEWPORTS = [ @@ -64,6 +538,12 @@ VIEWPORTS = [ {'width': 1920, 'height': 1080}, ] +# ── Cookie Extraction ──────────────────────────────────────────────── +# Reads Facebook cookies from the browser profile's SQLite database. +# Firefox stores cookies in cookies.sqlite; Chromium stores them in Cookies +# (a SQLite DB as well). This function handles Firefox's format. +# The cookies are copied to a temp file first to avoid locking issues. + async def get_fb_cookies(profile_path: str | None = None): cookie_db_path = profile_path or FX_PROFILE if not cookie_db_path: @@ -108,6 +588,9 @@ async def get_fb_cookies(profile_path: str | None = None): pass return [] +# ── Date Parsing ───────────────────────────────────────────────────── +# Facebook displays dates in relative format ("2h ago", "Yesterday", "Monday") +# or various absolute formats. These functions normalize them to YYYY-MM-DD. WEEKDAY_ORDER = ['monday', 'tuesday', 'wednesday', 'thursday', 'friday', 'saturday', 'sunday'] def _parse_fb_date(block: list[str]) -> str: @@ -136,54 +619,134 @@ def _parse_fb_date(block: list[str]) -> str: pass return datetime.now().strftime('%Y-%m-%d') -def _extract_posts_from_text(raw: str, url: str) -> list[dict]: - lines = [l.strip() for l in raw.split('\n')] - blocks = [] - cur = [] - fb_run = 0 - for l in lines: - if 'facebook' in l.lower(): - fb_run += 1 - if cur and fb_run >= 2: - blocks.append(cur) - cur = [] - else: - fb_run = 0 - if l and len(l) >= 15: - words = re.findall(r'[A-Za-z]{2,}', l) - if len(words) >= 2: - cur.append(l) - if cur: - blocks.append(cur) +def _is_within_days(date_str: str, max_days: int = 3) -> bool: + """Check if date is within max_days from now. Empty/unparseable = keep.""" + if not date_str: + return True + try: + dt = datetime.strptime(date_str.strip(), '%Y-%m-%d') + return (datetime.now() - dt).days <= max_days + except ValueError: + return True + + +def _clean_fb_text(text: str) -> str: + cleaned_lines = [] + for l in text.split('\n'): + stripped = l.strip() + if not stripped: + continue + if len(stripped) < 3: + continue + if all(not c.isalpha() for c in stripped): + continue + cleaned_lines.append(stripped) + return '\n'.join(cleaned_lines) + +# ── Post Extraction ────────────────────────────────────────────────── +# Two strategies for extracting posts from page content: +# 1. _extract_posts_from_elements — uses structured DOM data from _get_article_elements() +# 2. _extract_posts_from_text — fallback that parses raw page text line by line +# Both apply dedup (seen_texts), offer filtering, and request scoring. + +def _extract_posts_from_elements(elements: list[dict], base_url: str) -> list[dict]: posts = [] seen_texts = set() - for block in blocks: - if not block: + for el in elements: + raw_text = el.get('text', '') + if len(raw_text) < 40: continue - kw_indices = [i for i, l in enumerate(block) if kw_match(l)] - if not kw_indices: + text = _clean_fb_text(raw_text) + if len(text) < 40: continue - i = kw_indices[0] - start = max(0, i - 2) - end = min(len(block), i + 5) - snippet = ' '.join(block[start:end]) - if len(snippet) < 40: + if is_offer(text): continue - dekey = snippet[:80] + lines = text.split('\n') + dekey = text[:80] if dekey in seen_texts: continue seen_texts.add(dekey) + request_score = 2 if is_request(text) else 0 + post_url = el.get('url') or base_url + + # Prefer JS-extracted date, fall back to text parsing + js_date = (el.get('date') or '').strip() + if js_date: + # Try ISO datetime from