diff --git a/ai-server/index.mjs b/ai-server/index.mjs index e8a5a7a..4ffb1f5 100644 --- a/ai-server/index.mjs +++ b/ai-server/index.mjs @@ -1,3 +1,13 @@ +// ── CRM AI Server ────────────────────────────────────────────────── +// Provides: +// - Chat API (POST /ai/chat) — routes user messages to Ollama for sales coaching +// - Setup wizard endpoints (GET /setup/status, POST /setup/profile, etc.) +// - Combined /status endpoint for splash page health polling +// - Configuration routes (GET/POST /ai/instructions, GET /ai/jobs) +// - Model pull support (POST /setup/ollama/pull) +// +// This is a zero-dependency Node.js HTTP server (no Express needed). + import http from "node:http" import fs from "node:fs" import path from "node:path" @@ -8,6 +18,9 @@ const __dirname = path.dirname(fileURLToPath(import.meta.url)) const ROOT = path.resolve(__dirname, "..") // ── Load .env.local ────────────────────────────────────────────── +// Reads key=value pairs and sets them as process.env so they're +// available throughout the server. Ignores comments and blank lines. +// Values with matching quotes are unquoted. try { const envPath = path.join(ROOT, ".env.local") const envContent = fs.readFileSync(envPath, "utf-8") @@ -23,7 +36,7 @@ try { } console.log("Loaded .env.local") } catch { - // .env.local may not exist, ignore + // .env.local may not exist (first run), which is fine } // ── Config from env ───────────────────────────────────────────── @@ -36,10 +49,14 @@ const JOBS_PATH = process.env.JOBS_PATH || path.join(ROOT, "data", "ai", "jobs.j const AI_MD_PATH = process.env.AI_MD_PATH || path.join(ROOT, "data", "ai", "ai.md") // ── Setup state ────────────────────────────────────────────────── +// Tracks the Ollama model pull process so the setup wizard can +// poll for download progress. let pullProcess = null let pullProgress = { status: "idle", progress: 0, message: "" } // ── Job loading ───────────────────────────────────────────────── +// Loads job categories from a JSONL file (one JSON object per line). +// Used as context for the AI sales coach chat responses. function loadJobs() { try { const content = fs.readFileSync(JOBS_PATH, "utf-8") @@ -64,6 +81,8 @@ function loadJobs() { } // ── ai.md management ──────────────────────────────────────────── +// ai.md is a Markdown file containing system instructions for the AI. +// It can be read, written, or appended to via the API. function readInstructions() { try { return fs.readFileSync(AI_MD_PATH, "utf-8") @@ -78,6 +97,7 @@ function writeInstructions(content) { } function appendToImprovementLog(entry) { + // Adds a timestamped entry to the ## Improvement Log section of ai.md const current = readInstructions() const timestamp = new Date().toISOString().replace("T", " ").substring(0, 16) const logEntry = `\n- ${timestamp} — ${entry}` @@ -101,10 +121,13 @@ function appendToImprovementLog(entry) { } // ── Chat handler ──────────────────────────────────────────────── +// scrapeFacebook() calls the scraper service (port 3008) to get leads. +// handleChat() processes user messages — triggers lead scraping when +// the user asks for "leads" or "listings", otherwise routes to Ollama +// for AI-powered sales coaching. async function scrapeFacebook() { const profilePath = process.env.FX_PROFILE || "" const urlPath = `/scrape/facebook?force=true${profilePath ? `&profile_path=${encodeURIComponent(profilePath)}` : ""}` - const logPath = "C:\\Users\\USER-PC\\AppData\\Local\\Temp\\opencode\\ai-scrape-debug.log" try { const body = await new Promise((resolve, reject) => { const req = http.request({ hostname: "127.0.0.1", port: 3008, path: urlPath, method: "POST", timeout: 360000 }, (res) => { @@ -139,6 +162,7 @@ function formatLeads(leads) { } async function handleChat(userMessage, userId, userRole) { + // If the user asks for leads, trigger the scraper const lowerMsg = userMessage.toLowerCase() const triggerWords = ["lists", "listings", "leads", "recent leads", "pull leads", "show me leads", "show listings"] @@ -150,6 +174,7 @@ async function handleChat(userMessage, userId, userRole) { return "Scraper returned no results or encountered an error. Try again later." } + // Otherwise, build a system prompt with job context and send to Ollama const jobs = loadedJobs const instructions = readInstructions() @@ -189,7 +214,7 @@ Provide concise, actionable sales advice. When asked about a specific job catego const data = await ollamaRes.json() const responseText = data.message?.content || "" - // Try to persist to DB (best-effort) + // Persist conversation to PostgreSQL (best-effort — table may not exist yet) try { if (pgPool && userId) { await pgPool.query( @@ -205,6 +230,8 @@ Provide concise, actionable sales advice. When asked about a specific job catego } // ── PG pool (lazy init) ──────────────────────────────────────── +// PostgreSQL connection pool for storing conversation history. +// Lazy-initialized so the server starts even without a DB. let pgPool = null async function initPg() { if (!DATABASE_URL) return @@ -246,8 +273,9 @@ function parseURL(req) { return { pathname: url.pathname, searchParams: url.searchParams } } +// ── HTTP Server ───────────────────────────────────────────────── const server = http.createServer(async (req, res) => { - // CORS + // CORS headers — allow the Next.js frontend (port 3006) to call us res.setHeader("Access-Control-Allow-Origin", "*") res.setHeader("Access-Control-Allow-Methods", "GET, POST, OPTIONS") res.setHeader("Access-Control-Allow-Headers", "Content-Type") @@ -280,10 +308,12 @@ const server = http.createServer(async (req, res) => { } // GET /status — combined health of all services + // Used by the splash page to check if AI, Scraper, and Frontend are ready. + // Polls each service internally to avoid cross-origin CORS issues. if (req.method === "GET" && pathname === "/status") { const { default: http } = await import("http") const results = { ai: true } - // Check scraper + // Check scraper (port 3008) try { await new Promise((resolve, reject) => { const r = http.get("http://127.0.0.1:3008/health", { timeout: 3000 }, (res) => { res.resume(); resolve() }) @@ -291,7 +321,7 @@ const server = http.createServer(async (req, res) => { }) results.scraper = true } catch { results.scraper = false } - // Check frontend + // Check frontend (port 3006) try { await new Promise((resolve, reject) => { const r = http.get("http://127.0.0.1:3006", { timeout: 3000 }, (res) => { res.resume(); resolve() }) @@ -305,6 +335,11 @@ const server = http.createServer(async (req, res) => { // ── Setup endpoints ───────────────────────────────────────── // GET /setup/status — check environment + // Called by the splash page on boot. Returns info about: + // - Ollama availability + // - Model presence + // - Detected browsers with login status + // - Whether this is a first run (wizard needed) if (req.method === "GET" && pathname === "/setup/status") { const envExists = fs.existsSync(path.join(ROOT, ".env.local")) @@ -327,33 +362,41 @@ const server = http.createServer(async (req, res) => { } catch {} } - // Profile auto-detect - let profilePath = process.env.FX_PROFILE || "" - let profileDetected = !!profilePath - if (!profileDetected) { - try { - const r = await fetch("http://127.0.0.1:3008/health", { signal: AbortSignal.timeout(2000) }) - if (r.ok) { - const diag = await (await fetch("http://127.0.0.1:3008/setup/profile", { signal: AbortSignal.timeout(5000) })).json() - if (diag.path) { profilePath = diag.path; profileDetected = true } - } - } catch {} - } - - // Login check + // Detect all browsers via scraper + let browsers = { firefox: { path: null }, opera: { path: null }, chrome: { path: null }, edge: { path: null } } let facebookLoggedIn = false - if (profileDetected) { - try { - const r = await fetch("http://127.0.0.1:3008/setup/check-login", { - method: "POST", headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ profile_path: profilePath }), - signal: AbortSignal.timeout(15000), - }) - if (r.ok) { const d = await r.json(); facebookLoggedIn = d.logged_in === true } - } catch {} - } + let selectedBrowser = process.env.SELECTED_BROWSER || "" - const firstRun = !envExists || !ollamaRunning || !profileDetected || !modelAvailable + try { + await fetch("http://127.0.0.1:3008/health", { signal: AbortSignal.timeout(2000) }) + const profiles = await (await fetch("http://127.0.0.1:3008/setup/profile", { signal: AbortSignal.timeout(5000) })).json() + for (const [b, p] of Object.entries(profiles)) { + if (p) browsers[b] = { path: p } + } + // Check login for the selected browser first, then try all + const detectedList = Object.entries(browsers).filter(([, v]) => v.path) + for (const [b, v] of detectedList) { + try { + const r = await fetch("http://127.0.0.1:3008/setup/check-login", { + method: "POST", headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ browser: b, profile_path: v.path }), + signal: AbortSignal.timeout(20000), + }) + if (r.ok) { + const d = await r.json() + browsers[b].logged_in = d.logged_in === true + if (d.logged_in && !facebookLoggedIn) { + facebookLoggedIn = true + if (!selectedBrowser) selectedBrowser = b + } + } + } catch {} + } + } catch {} + + const anyDetected = Object.values(browsers).some(v => v.path) + // first_run = any setup step is incomplete + const firstRun = !envExists || !ollamaRunning || !anyDetected || !facebookLoggedIn || !modelAvailable return sendJSON(res, 200, { first_run: firstRun, @@ -361,46 +404,71 @@ const server = http.createServer(async (req, res) => { ollama_running: ollamaRunning, model_available: modelAvailable, model_name: MODEL, - profile_detected: profileDetected, - profile_path: profilePath || null, + selected_browser: selectedBrowser, + browsers, facebook_logged_in: facebookLoggedIn, }) } - // POST /setup/profile — save profile path to .env.local + // POST /setup/profile — save selected browser + path to .env.local + // Called by the setup wizard when the user confirms their browser choice. + // Writes SELECTED_BROWSER and the matching profile env var to .env.local. if (req.method === "POST" && pathname === "/setup/profile") { const body = await parseBody(req) + const browserName = (body.browser || "").trim().toLowerCase() const profilePath = (body.path || "").trim() - if (!profilePath) return sendJSON(res, 400, { error: "Path required" }) - if (!fs.existsSync(profilePath)) return sendJSON(res, 400, { error: "Path does not exist" }) + if (!browserName || !["firefox", "opera", "chrome", "edge"].includes(browserName)) + return sendJSON(res, 400, { error: "Valid browser required (firefox/opera/chrome/edge)" }) + if (!profilePath) + return sendJSON(res, 400, { error: "Path required" }) + + const envKey = browserName === "firefox" ? "FX_PROFILE" + : browserName === "opera" ? "OPERA_PROFILE" + : browserName === "edge" ? "EDGE_PROFILE" + : "CHROME_PROFILE" const envPath = path.join(ROOT, ".env.local") let content = "" try { content = fs.readFileSync(envPath, "utf-8") } catch {} - const lines = content.split("\n") - let found = false + let lines = content.split("\n") + // Update or add SELECTED_BROWSER + let foundSel = false for (let i = 0; i < lines.length; i++) { - if (lines[i].trim().startsWith("FX_PROFILE=")) { - lines[i] = `FX_PROFILE=${profilePath}` - found = true + if (lines[i].trim().startsWith("SELECTED_BROWSER=")) { + lines[i] = `SELECTED_BROWSER=${browserName}` + foundSel = true break } } - if (!found) lines.push(`FX_PROFILE=${profilePath}`) + if (!foundSel) lines.push(`SELECTED_BROWSER=${browserName}`) + // Update or add browser profile + let foundProf = false + for (let i = 0; i < lines.length; i++) { + if (lines[i].trim().startsWith(`${envKey}=`)) { + lines[i] = `${envKey}=${profilePath}` + foundProf = true + break + } + } + if (!foundProf) lines.push(`${envKey}=${profilePath}`) fs.writeFileSync(envPath, lines.join("\n"), "utf-8") - process.env.FX_PROFILE = profilePath - return sendJSON(res, 200, { success: true, path: profilePath }) + process.env.SELECTED_BROWSER = browserName + process.env[envKey] = profilePath + return sendJSON(res, 200, { success: true, browser: browserName, path: profilePath }) } - // POST /setup/check-login — verify Facebook login in the given profile + // POST /setup/check-login — proxy to scraper, accepts browser + profile_path + // The splash page calls this (via the AI server) to verify Facebook login status. if (req.method === "POST" && pathname === "/setup/check-login") { const body = await parseBody(req) - const profilePath = body.profile_path || process.env.FX_PROFILE || "" + const browserName = (body.browser || "").trim().toLowerCase() || process.env.SELECTED_BROWSER || "" + const profilePath = (body.profile_path || "").trim() + if (!profilePath) return sendJSON(res, 200, { logged_in: false, reason: "no_profile" }) try { const r = await fetch("http://127.0.0.1:3008/setup/check-login", { method: "POST", headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ profile_path: profilePath }), + body: JSON.stringify({ browser: browserName, profile_path: profilePath }), signal: AbortSignal.timeout(20000), }) if (r.ok) { const d = await r.json(); return sendJSON(res, 200, d) } @@ -409,6 +477,8 @@ const server = http.createServer(async (req, res) => { } // POST /setup/ollama/pull — start pulling the model + // Spawns "ollama pull" as a child process. The setup wizard polls + // the progress endpoint to show a download progress bar. if (req.method === "POST" && pathname === "/setup/ollama/pull") { if (pullProcess) return sendJSON(res, 200, { status: "already_running" }) pullProgress = { status: "downloading", progress: 0, message: "Starting..." } @@ -432,22 +502,23 @@ const server = http.createServer(async (req, res) => { } // GET /setup/ollama/pull/progress + // Returns current download progress for the setup wizard. if (req.method === "GET" && pathname === "/setup/ollama/pull/progress") { return sendJSON(res, 200, pullProgress) } - // GET /ai/jobs + // GET /ai/jobs — return loaded job categories if (req.method === "GET" && pathname === "/ai/jobs") { return sendJSON(res, 200, { jobs: loadedJobs }) } - // GET /ai/instructions + // GET /ai/instructions — return current ai.md content if (req.method === "GET" && pathname === "/ai/instructions") { const instructions = readInstructions() return sendJSON(res, 200, { success: true, instructions }) } - // POST /ai/instructions + // POST /ai/instructions — update ai.md or append improvement log entry if (req.method === "POST" && pathname === "/ai/instructions") { const body = await parseBody(req) if (body.content) { @@ -461,18 +532,17 @@ const server = http.createServer(async (req, res) => { }) } - // POST /ai/chat + // POST /ai/chat — main AI chat endpoint + // Accepts { message, user_id?, user_role? } and returns AI response. + // user_role must be "sales", "admin", or "super_admin" if provided. if (req.method === "POST" && pathname === "/ai/chat") { const startTime = Date.now() const chunks = [] req.on("data", c => chunks.push(c)) req.on("end", () => { const rawBody = Buffer.concat(chunks).toString() - try { fs.appendFileSync("C:\\Users\\USER-PC\\AppData\\Local\\Temp\\opencode\\ai-req-log.txt", - `${new Date().toISOString()} headers=${JSON.stringify(req.headers)} body=${rawBody}\n`) } catch {} try { const body = JSON.parse(rawBody) - // Continue processing processRequest(req, res, body, startTime) } catch { sendJSON(res, 400, { error: "Invalid JSON" }) @@ -481,7 +551,7 @@ const server = http.createServer(async (req, res) => { return } -// Separate handler +// Separate handler for /ai/chat (defined here due to hoisting within the IIFE) async function processRequest(req, res, body, startTime) { const { message, user_id, user_role } = body @@ -498,7 +568,7 @@ async function processRequest(req, res, body, startTime) { return sendJSON(res, 200, { response }) } - // 404 + // 404 fallback sendJSON(res, 404, { error: "Not found" }) } catch (err) { console.error("Request error:", err) diff --git a/browser-use-service/main.py b/browser-use-service/main.py index cddb226..abac579 100644 --- a/browser-use-service/main.py +++ b/browser-use-service/main.py @@ -1,3 +1,4 @@ +# ── Imports ────────────────────────────────────────────────────────── import os, json, asyncio, re, shutil, sqlite3, urllib.parse, random, logging, tempfile from datetime import datetime, timedelta from fastapi import FastAPI, Query, Body @@ -8,8 +9,12 @@ from browser_use import Agent, Browser from langchain_ollama import ChatOllama +# ── Helpers ────────────────────────────────────────────────────────── def make_ollama(model: str | None = None, **kwargs) -> ChatOllama: - """Create ChatOllama with required attrs for browser-use Agent compatibility.""" + """ + Create ChatOllama with required attrs for browser-use Agent compatibility. + The browser-use Agent expects llm.name, llm.model_name, and llm.provider attrs. + """ llm = ChatOllama(model=model or CLASSIFY_MODEL, **kwargs) object.__setattr__(llm, 'provider', 'ollama') object.__setattr__(llm, 'name', 'ChatOllama') @@ -17,6 +22,7 @@ def make_ollama(model: str | None = None, **kwargs) -> ChatOllama: return llm +# ── Logging & App Setup ────────────────────────────────────────────── logging.basicConfig(level=logging.INFO, format='%(asctime)s [%(levelname)s] %(message)s') logger = logging.getLogger(__name__) @@ -29,13 +35,24 @@ app.add_middleware( ) PORT = int(os.getenv("PORT", "3008")) +# ── AI / Ollama Config ─────────────────────────────────────────────── OLLAMA_URL = os.getenv("OLLAMA_URL", "http://localhost:11434") CLASSIFY_MODEL = os.getenv("CLASSIFY_MODEL", "dolphin-llama3:8b") +# ── Env Vars (set via .env.local) ────────────────────────────────────── +# These store detected browser profile paths from the setup wizard. FX_PROFILE = os.getenv('FX_PROFILE', '') CHROME_PROFILE = os.getenv('CHROME_PROFILE', '') +OPERA_PROFILE = os.getenv('OPERA_PROFILE', '') +EDGE_PROFILE = os.getenv('EDGE_PROFILE', '') +# SELECTED_BROWSER determines which browser the scrape dispatcher tries first. +# Values: "firefox", "chrome", "opera", "edge", or "" (auto-detect all). +SELECTED_BROWSER = os.getenv('SELECTED_BROWSER', '') +# ── Chromium Launch Args ────────────────────────────────────────────── +# These flags reduce detectable automation signals in Chrome/Edge/Opera. +# They disable background services, sync, updates, throttling, popups, etc. CHROME_LAUNCH_ARGS = [ '--headless=new', '--disable-blink-features=AutomationControlled', @@ -60,20 +77,36 @@ CHROME_LAUNCH_ARGS = [ ] +# ── Profile Detection ──────────────────────────────────────────────── +# All find_*_profile() functions return the parent directory (User Data or equivalent), +# NOT the "Default" subdirectory. This is important because copy_chrome_profile() +# appends "Default" internally, so returning the full Default path would cause +# duplicated path segments (e.g. ".../User Data/Default/Default"). + def detect_browser_from_profile(profile_path: str) -> str | None: - """Detect browser type from profile path. Returns 'firefox', 'chromium', or None.""" + """ + Infer browser type from the profile path string. + Checks for keywords like "firefox", "opera", "edge", "chrome" in the path. + Returns "firefox", "chrome", "opera", "edge", or None. + """ if not profile_path: return None p = profile_path.lower().replace('\\', '/') if 'firefox' in p or '.default' in p or '.dev-edition' in p: return 'firefox' - if 'chrome' in p or 'chromium' in p or 'edge' in p: - return 'chromium' + if 'opera' in p: + return 'opera' + if 'edge' in p: + return 'edge' + if 'chrome' in p or 'chromium' in p: + return 'chrome' return None def find_firefox_profile() -> str | None: - """Auto-detect Firefox profile directory cross-platform.""" + """Auto-detect Firefox profile directory cross-platform. + Scans the Profiles dir and returns the first .default or .dev-edition folder found, + preferring "default-release" profiles.""" import sys home = os.path.expanduser("~") candidates = [] @@ -102,7 +135,8 @@ def find_firefox_profile() -> str | None: def find_chrome_profile() -> str | None: - """Auto-detect Chrome/Chromium profile directory cross-platform.""" + """Auto-detect Chrome profile (User Data dir) cross-platform. + Returns the User Data parent dir (NOT the Default subdirectory).""" import sys home = os.path.expanduser("~") @@ -113,15 +147,60 @@ def find_chrome_profile() -> str | None: else: base = os.path.join(home, ".config", "google-chrome") - default_profile = os.path.join(base, "Default") - if os.path.isdir(default_profile): - logger.info("Auto-detected Chrome profile: %s", default_profile) - return default_profile + if os.path.isdir(os.path.join(base, "Default")): + logger.info("Auto-detected Chrome profile: %s", base) + return base return None +def find_opera_profile() -> str | None: + """Auto-detect Opera profile directory cross-platform. + Opera stores its profile at the "Opera Stable" level (no "User Data/Default" + nesting like Chrome/Edge), so this returns the profile root directly.""" + import sys + home = os.path.expanduser("~") + + if sys.platform == "win32": + base = os.path.join(os.environ.get("APPDATA", ""), "Opera Software", "Opera Stable") + elif sys.platform == "darwin": + base = os.path.join(home, "Library", "Application Support", "com.operasoftware.Opera") + else: + base = os.path.join(home, ".config", "opera") + + if os.path.isdir(base) and os.listdir(base): + logger.info("Auto-detected Opera profile: %s", base) + return base + return None + + +def find_edge_profile() -> str | None: + """Auto-detect Edge profile (User Data dir) cross-platform. + Returns the User Data parent dir (NOT the Default subdirectory).""" + import sys + home = os.path.expanduser("~") + + if sys.platform == "win32": + base = os.path.join(os.environ.get("LOCALAPPDATA", ""), "Microsoft", "Edge", "User Data") + elif sys.platform == "darwin": + base = os.path.join(home, "Library", "Application Support", "Microsoft Edge") + else: + base = os.path.join(home, ".config", "microsoft-edge") + + if os.path.isdir(os.path.join(base, "Default")): + logger.info("Auto-detected Edge profile: %s", base) + return base + return None + + +# ── Profile Copying ───────────────────────────────────────────────── +# Both functions copy essential cookies/login/storage files from the user's +# real browser profile to a temp directory. Playwright then uses these temp +# profiles via launch_persistent_context(), preserving Facebook login state +# without modifying the user's actual profile. + def copy_firefox_profile(src_path: str) -> str: - """Copy essential Firefox profile files to a temp dir for Playwright.""" + """Copy essential Firefox profile files (cookies, localStorage, permissions) to a temp dir for Playwright. + Also creates a minimal profiles.ini so Firefox treats it as a valid profile.""" essential = ['cookies.sqlite', 'webappsstore.sqlite', 'permissions.sqlite'] dst = tempfile.mkdtemp(prefix='fb_fx_profile_') for f_name in essential: @@ -135,7 +214,12 @@ def copy_firefox_profile(src_path: str) -> str: def copy_chrome_profile(user_data_dir: str, profile_dir: str = 'Default') -> str: - """Copy Chrome profile to temp dir for launch_persistent_context.""" + """Copy Chromium-based profile (Chrome/Edge/Opera) to a temp dir. + Copies Cookies, Login Data, Bookmarks, Web Data, Local Storage, Session Storage, and Local State. + Args: + user_data_dir: Parent directory (e.g. ".../Chrome/User Data") + profile_dir: Subdirectory name, typically "Default" (default) + """ essential = ['Cookies', 'Login Data', 'Bookmarks', 'Web Data'] dst = tempfile.mkdtemp(prefix='fb_chrome_udir_') src_profile = os.path.join(user_data_dir, profile_dir) @@ -159,7 +243,8 @@ def copy_chrome_profile(user_data_dir: str, profile_dir: str = 'Default') -> str def ensure_ublock_extension() -> str | None: - """Download and extract uBlock Origin extension for ad blocking.""" + """Download and extract uBlock Origin extension for ad blocking via Chromium. + Cached after first download to avoid re-downloading on every scrape.""" ext_dir = os.path.join(tempfile.gettempdir(), 'fb_ublock_origin') manifest = os.path.join(ext_dir, 'manifest.json') if os.path.exists(manifest): @@ -182,10 +267,15 @@ def ensure_ublock_extension() -> str | None: return None +# ── Agent Output Parsing ────────────────────────────────────────────── +# When the browser-use Agent fallback is used, it returns natural language +# output containing post listings. This function tries to extract structured +# data from that output, first attempting JSON parsing, then falling back to +# a line-by-line heuristic (numbered or bulleted items). + def extract_agent_posts(agent_result: str, page_text: str) -> list[dict]: """Parse posts from agent output or page text fallback.""" posts = [] - # Try JSON from agent output json_match = re.search(r'\[.*?\]', agent_result, re.DOTALL) if json_match: try: @@ -221,6 +311,12 @@ def extract_agent_posts(agent_result: str, page_text: str) -> list[dict]: return posts +# ── Keyword Filters for Lead Detection ────────────────────────────── +# BROAD_KEYWORDS — catch any post that might relate to web development needs. +# OFFER_PATTERNS — regex patterns matching service offers (which we ignore). +# REQUEST_PATTERNS — regex patterns matching people asking for help/services. +# Together these form a two-pass filter + AI classification pipeline. + BROAD_KEYWORDS = [ "website", "web design", "web develop", "web dev", "build my website", "build a website", "create a website", @@ -290,6 +386,13 @@ REQUEST_PATTERNS = [ r"\bquote\s+(please|for|to)\b", ] +# ── Anti-Detection Scripts ──────────────────────────────────────────── +# These JS scripts are injected into every page via add_init_script(). +# They override navigator properties (webdriver, userAgent, plugins, etc.) +# and WebGL/canvas/audio fingerprints to make Playwright harder to detect. +# Firefox and Chromium variants differ slightly (e.g. Chrome needs the +# "window.chrome" object; Firefox must NOT have it). + def firefox_init_script() -> str: return r"""// Firefox Anti-Detection Object.defineProperty(navigator, 'webdriver', { get: () => undefined }); @@ -384,18 +487,25 @@ Object.defineProperty(navigator, 'plugins', { get: () => [1,2,3,4,5].map(i => ({ Object.defineProperty(navigator, 'mimeTypes', { get: () => [{type:'application/pdf',suffixes:'pdf',description:'Portable Document Format',enabledPlugin:navigator.plugins[0]},{type:'text/pdf',suffixes:'pdf',description:'Portable Document Format',enabledPlugin:navigator.plugins[0]}] });""" +# ── Text Classification Utilities ──────────────────────────────────── def kw_match(text: str) -> bool: t = text.lower() return any(kw in t for kw in BROAD_KEYWORDS) def is_request(text: str) -> bool: + """Detect if text contains a request for services (e.g. "I need", "looking for").""" t = text.lower() return any(re.search(p, t) for p in REQUEST_PATTERNS) def is_offer(text: str) -> bool: + """Detect if text is an offer/advertisement (we skip these, not leads).""" t = text.lower() return any(re.search(p, t) for p in OFFER_PATTERNS) +# ── Facebook Search Queries ─────────────────────────────────────────── +# These are the search terms the scraper will use on Facebook. +# They target people looking for web development / website services. +# Each run picks 2-4 random queries to vary behavior and reduce detection. FB_SEARCHES = [ "looking for web developer", "need a website designed", @@ -428,6 +538,12 @@ VIEWPORTS = [ {'width': 1920, 'height': 1080}, ] +# ── Cookie Extraction ──────────────────────────────────────────────── +# Reads Facebook cookies from the browser profile's SQLite database. +# Firefox stores cookies in cookies.sqlite; Chromium stores them in Cookies +# (a SQLite DB as well). This function handles Firefox's format. +# The cookies are copied to a temp file first to avoid locking issues. + async def get_fb_cookies(profile_path: str | None = None): cookie_db_path = profile_path or FX_PROFILE if not cookie_db_path: @@ -472,6 +588,9 @@ async def get_fb_cookies(profile_path: str | None = None): pass return [] +# ── Date Parsing ───────────────────────────────────────────────────── +# Facebook displays dates in relative format ("2h ago", "Yesterday", "Monday") +# or various absolute formats. These functions normalize them to YYYY-MM-DD. WEEKDAY_ORDER = ['monday', 'tuesday', 'wednesday', 'thursday', 'friday', 'saturday', 'sunday'] def _parse_fb_date(block: list[str]) -> str: @@ -525,6 +644,12 @@ def _clean_fb_text(text: str) -> str: cleaned_lines.append(stripped) return '\n'.join(cleaned_lines) +# ── Post Extraction ────────────────────────────────────────────────── +# Two strategies for extracting posts from page content: +# 1. _extract_posts_from_elements — uses structured DOM data from _get_article_elements() +# 2. _extract_posts_from_text — fallback that parses raw page text line by line +# Both apply dedup (seen_texts), offer filtering, and request scoring. + def _extract_posts_from_elements(elements: list[dict], base_url: str) -> list[dict]: posts = [] seen_texts = set() @@ -617,6 +742,11 @@ def _extract_posts_from_text(raw: str, url: str) -> list[dict]: cur.pop(0) return posts +# ── Human-like Behavior Simulation ────────────────────────────────── +# These functions add random delays, mouse movements, and scroll patterns +# to make automated browsing look more like a real human user. +# This is critical for avoiding Facebook's bot detection. + async def human_scroll(page, steps: int = None, total_delay: float = None): steps = steps or random.randint(2, 5) total_delay = total_delay or random.uniform(6, 18) @@ -652,6 +782,12 @@ async def random_idle(page): except Exception: pass +# ── Facebook DOM Parsing ───────────────────────────────────────────── +# This JS function runs inside the page context to extract structured post +# data (text, author, URL, date) from Facebook's complex DOM structure. +# It tries multiple selectors (article, feed, etc.) and uses the first +# one that returns results, since Facebook's DOM changes frequently. + async def _get_article_elements(page) -> list[dict]: return await page.evaluate('''() => { const results = []; @@ -713,6 +849,8 @@ async def _get_article_elements(page) -> list[dict]: }''') async def _ensure_page(page, context): + """Check if the current page is still alive. If closed (e.g. by a popup), + create a fresh page and navigate to Facebook.""" try: await page.evaluate('1') return page @@ -728,6 +866,12 @@ async def _ensure_page(page, context): await page.wait_for_timeout(random.randint(3000, 8000)) return page + +# ── Facebook Search & Scrape ───────────────────────────────────────── +# Performs a Facebook search for a given query, scrolls through results, +# extracts posts, and returns them. Includes randomization of scroll behavior, +# mouse movements, and idle actions to mimic human browsing patterns. + async def search_facebook(page, context, query: str): page = await _ensure_page(page, context) url = f'https://www.facebook.com/search/posts/?q={urllib.parse.quote(query)}' @@ -764,6 +908,10 @@ async def search_facebook(page, context, query: str): return page, [] return page, posts +# ── Detection Signals ──────────────────────────────────────────────── +# Keywords/phrases in page URL or body text that indicate Facebook's +# security/bot detection systems have flagged the session. + DETECTION_SIGNALS = [ '/checkpoint/', '/login.php?', 'action=security_check', 'unusual activity', 'suspicious login', 'suspicious activity', @@ -784,54 +932,90 @@ def check_detection_signals(page_url: str, page_text: str = '') -> str | None: def cleanup_chrome(): + """Kill orphaned Chrome headless shell processes. These sometimes linger + after failed scrapes and interfere with subsequent runs.""" import subprocess, signal try: subprocess.run(["taskkill", "/F", "/IM", "chrome-headless-shell.exe"], capture_output=True, timeout=5) except Exception: pass + +# ── Main Scrape Dispatcher ──────────────────────────────────────────── +# scrape_facebook() is the main entry point. It: +# 1. Resolves the browser profile path (from SELECTED_BROWSER env var or auto-detect) +# 2. Dispatches to the correct browser-specific scraper +# 3. If the browser scrape is flagged by Facebook, falls through to the Agent fallback +# +# Priority order for browser selection: +# - SELECTED_BROWSER env var (set by setup wizard) → that browser's scraper +# - Auto-detect: Firefox → Opera → Chrome → Edge (first found wins) +# - If no profile found → Agent (browser-use + ChatOllama) fallback + async def scrape_facebook(profile_path: str | None = None, force: bool = False) -> dict: - """Dispatcher — Firefox primary, browser-use Agent fallback.""" - effective_path = profile_path or FX_PROFILE + """Dispatcher — respect SELECTED_BROWSER, fall through on flag, then Agent.""" + effective_path = profile_path or "" + browser_type = "" - # Auto-detect Firefox profile if none provided - if not effective_path: - detected = find_firefox_profile() - if detected: - effective_path = detected - os.environ["FX_PROFILE"] = detected + # Resolve profile and browser type + if effective_path: + browser_type = detect_browser_from_profile(effective_path) or "" + elif SELECTED_BROWSER == "firefox": + effective_path = FX_PROFILE or find_firefox_profile() or "" + browser_type = "firefox" + elif SELECTED_BROWSER == "opera": + effective_path = OPERA_PROFILE or find_opera_profile() or "" + browser_type = "opera" + elif SELECTED_BROWSER == "edge": + effective_path = EDGE_PROFILE or find_edge_profile() or "" + browser_type = "edge" + elif SELECTED_BROWSER == "chrome": + effective_path = CHROME_PROFILE or find_chrome_profile() or "" + browser_type = "chrome" + else: + # Auto-detect — try all in priority order + for name, env_var, find_fn in [ + ("firefox", "FX_PROFILE", find_firefox_profile), + ("opera", "OPERA_PROFILE", find_opera_profile), + ("chrome", "CHROME_PROFILE", find_chrome_profile), + ("edge", "EDGE_PROFILE", find_edge_profile), + ]: + p = os.getenv(env_var, "") or find_fn() or "" + if p: + effective_path = p + browser_type = name + break - browser_type = detect_browser_from_profile(effective_path) + if not browser_type or not effective_path: + logger.info("No profile found — falling back to Agent") + return await _scrape_with_agent(force) - if not browser_type and CHROME_PROFILE: - browser_type = detect_browser_from_profile(CHROME_PROFILE) - effective_path = CHROME_PROFILE + logger.info("Selected browser: %s (profile: %s)", browser_type, effective_path) - # Auto-detect Chrome profile if still nothing - if not browser_type: - detected_chrome = find_chrome_profile() - if detected_chrome: - effective_path = detected_chrome - browser_type = 'chromium' - - logger.info("Detected browser: %s (profile: %s)", browser_type or "none", effective_path) - - # Firefox primary (raw Playwright, stealth) + # Firefox path if browser_type == "firefox": result = await _scrape_with_firefox(effective_path, force) if result.get("success") or not result.get("flagged"): return result - logger.warning("Firefox path failed (%s), falling back to Agent", result.get("flag_reason", "unknown")) + logger.warning("Firefox flagged (%s), trying Agent", result.get("flag_reason", "unknown")) return await _scrape_with_agent(force) - # CHROME_PROFILE set or no profile → Agent - if browser_type == "chromium": - return await _scrape_with_agent(force) - - # No profile at all → try Agent (fresh Chromium) + # Chromium-based (chrome / opera / edge) + result = await _scrape_with_chromium(effective_path, browser_type, force) + if result.get("success") or not result.get("flagged"): + return result + logger.warning("%s flagged (%s), trying Agent", browser_type, result.get("flag_reason", "unknown")) return await _scrape_with_agent(force) +# ── Firefox Scraper ────────────────────────────────────────────────── +# Launches a headless Firefox with the user's real profile cookies (copied +# to a temp dir). Uses firefox_user_prefs to disable automation flags and +# tracking protection. Injects anti-detection script via add_init_script(). +# If force=False, randomly skips 30% of runs as a decoy (looks like random +# human browsing). If flagged by Facebook, returns flagged=True for the +# dispatcher to fall through to the Agent. + async def _scrape_with_firefox(profile_path: str, force: bool) -> dict: """Scrape Facebook using Firefox + persistent real profile (no cookie injection).""" if not profile_path: @@ -939,6 +1123,136 @@ async def _scrape_with_firefox(profile_path: str, force: bool) -> dict: pass +# ── Chromium Scraper ───────────────────────────────────────────────── +# Generic scraper for Chrome/Edge/Opera. Uses the same structure as the +# Firefox scraper but with Chromium-specific launch config: +# - Chrome: channel="chrome" +# - Edge: channel="msedge" +# - Opera: executable_path from shutil.which("opera") +# Cookies come from copy_chrome_profile (temp copy of user's profile data). +# Anti-detection script differs slightly from Firefox variant. +# Same decoy-skip and flagged-fallback behavior as Firefox. + +async def _scrape_with_chromium(profile_path: str, browser: str, force: bool = False) -> dict: + """Scrape Facebook using a Chromium-based browser profile (Chrome/Edge/Opera).""" + if not profile_path: + return {"success": False, "leads": [], "flagged": False, "flag_reason": None, "error": "No profile path"} + + channel = None + executable_path = None + if browser == "chrome": + channel = "chrome" + elif browser == "edge": + channel = "msedge" + elif browser == "opera": + executable_path = shutil.which("opera") or shutil.which("Opera") + + profile_dir = None + try: + profile_dir = copy_chrome_profile(profile_path) + async with async_playwright() as pw: + launch_kwargs = dict( + user_data_dir=profile_dir, + headless=True, + args=CHROME_LAUNCH_ARGS, + ) + if channel: + launch_kwargs["channel"] = channel + if executable_path: + launch_kwargs["executable_path"] = executable_path + + context = await pw.chromium.launch_persistent_context(**launch_kwargs) + pages = context.pages + page = pages[0] if pages else await context.new_page() + await context.add_init_script(chromium_init_script()) + + try: + await page.goto('https://www.google.com/', wait_until='domcontentloaded', timeout=15000) + await page.wait_for_timeout(random.randint(1000, 3000)) + except Exception: + logger.warning("Google navigation failed, trying Facebook directly") + + await page.goto('https://www.facebook.com/', wait_until='domcontentloaded', timeout=30000) + await page.wait_for_timeout(random.randint(3000, 8000)) + + url = page.url + page_text = await page.evaluate('document.body.innerText') if '/login' in url.lower() else '' + det = check_detection_signals(url, page_text) + if det or '/login' in url.lower(): + logger.warning("Facebook login page detected — flag: %s", det or "login_page") + await context.close() + return {"success": False, "leads": [], "flagged": True, "flag_reason": det or "login_page", "error": "Facebook login page detected"} + + await human_scroll(page, steps=random.randint(2, 4), total_delay=random.uniform(8, 20)) + if random.random() < 0.25: + await page.evaluate("window.scrollTo(0, 0)") + await page.wait_for_timeout(random.randint(2000, 5000)) + await human_scroll(page, steps=random.randint(1, 2)) + + if not force and random.random() < 0.3: + await page.wait_for_timeout(random.randint(8000, 20000)) + await context.close() + return {"success": True, "leads": [], "flagged": False, "flag_reason": None, "error": None} + + all_posts = [] + searches = random.sample(FB_SEARCHES, k=random.randint(2, 4)) + for i, query in enumerate(searches): + page, posts = await search_facebook(page, context, query) + all_posts.extend(posts) + if not posts: + continue + if random.random() < 0.4: + await page.evaluate(f"window.scrollBy(0, {random.randint(-300, 300)})") + delay = random.uniform(8, 25) + await page.wait_for_timeout(int(delay * 1000)) + if i == random.randint(0, 1) and random.random() < 0.15: + new_page = await context.new_page() + try: + await new_page.goto('https://www.facebook.com/groups/', wait_until='domcontentloaded', timeout=15000) + await new_page.wait_for_timeout(random.randint(3000, 8000)) + except Exception: + pass + await new_page.close() + page = await _ensure_page(page, context) + + if random.random() < 0.5: + await page.wait_for_timeout(random.randint(3000, 10000)) + + await context.close() + + seen = set() + deduped = [] + for p in all_posts: + key = p.get('content', '')[:100] + if key not in seen: + seen.add(key) + deduped.append(p) + + deduped = [p for p in deduped if _is_within_days(p.get('date', ''), 3)] + leads = deduped[:20] + if leads: + leads = await classify_leads(leads) + + return {"success": True, "leads": leads[:15], "flagged": False, "flag_reason": None, "error": None} + + except Exception as e: + logger.error("%s scrape failed: %s", browser, e) + return {"success": False, "leads": [], "flagged": False, "flag_reason": None, "error": str(e)} + finally: + if profile_dir and os.path.exists(profile_dir): + try: + shutil.rmtree(profile_dir, ignore_errors=True) + except Exception: + pass + + +# ── Agent Fallback ────────────────────────────────────────────────── +# When all browser-based scrapers fail/are flagged, this fallback uses +# browser-use Agent with ChatOllama (local, free) to navigate Facebook +# autonomously via GPT-style prompting. No API keys needed. +# Uses Chromium headless with the same launch args as _scrape_with_chromium. +# The Agent is prompted to extract structured post data and return JSON. + async def _scrape_with_agent(force: bool = False) -> dict: """Fallback scraper — browser-use Agent + ChatOllama (free/local, Chromium).""" cleanup_chrome() @@ -1016,6 +1330,12 @@ When done, return the data as a JSON list with keys: content, author, url, date. except Exception as e: logger.warning("Failed to clean up Chrome profile %s: %s", profile_dir, e) +# ── AI Classification ──────────────────────────────────────────────── +# Uses the local Ollama model to classify scraped posts as LEAD or NOT LEAD. +# Falls back to keyword-based filtering if the AI is unavailable. +# ask_ollama() sends a raw prompt to Ollama's /api/chat endpoint. +# classify_leads() builds a classification prompt and parses the JSON response. + async def ask_ollama(prompt: str) -> str: import httpx async with httpx.AsyncClient(timeout=120) as c: @@ -1101,39 +1421,97 @@ Return a JSON array like ["yes","no","yes"] matching the order above.""" return filtered[:10] return [] +# ══════════════════════════════════════════════════════════════════════ +# FastAPI Endpoints +# ══════════════════════════════════════════════════════════════════════ + @app.get("/health") async def health(): + """Simple health check for the splash page status polling.""" return {"status": "ok"} +def _detect_all_profiles() -> dict: + """Return all detected browser profiles.""" + return { + "firefox": FX_PROFILE or find_firefox_profile() or "", + "opera": OPERA_PROFILE or find_opera_profile() or "", + "chrome": CHROME_PROFILE or find_chrome_profile() or "", + "edge": EDGE_PROFILE or find_edge_profile() or "", + } + @app.get("/setup/profile") async def setup_profile(): - """Auto-detect Firefox profile path.""" - path = FX_PROFILE or find_firefox_profile() - return {"path": path, "detected": bool(path)} + """Auto-detect all browser profiles.""" + return _detect_all_profiles() + +async def _check_browser_login(profile_path: str, browser: str) -> dict: + """Check Facebook login for a specific browser type.""" + profile_dir = None + try: + if browser == "firefox": + profile_dir = copy_firefox_profile(profile_path) + async with async_playwright() as pw: + context = await pw.firefox.launch_persistent_context( + user_data_dir=profile_dir, headless=True, + firefox_user_prefs={"dom.webdriver.enabled": False}, + ) + page = context.pages[0] if context.pages else await context.new_page() + await page.goto("https://www.facebook.com/", wait_until="domcontentloaded", timeout=30000) + await page.wait_for_timeout(3000) + logged_in = "/login" not in page.url.lower() + await context.close() + shutil.rmtree(profile_dir, ignore_errors=True) + return {"logged_in": logged_in, "browser": browser, "path": profile_path} + else: + channel = None + exe = None + if browser == "chrome": + channel = "chrome" + elif browser == "edge": + channel = "msedge" + elif browser == "opera": + exe = shutil.which("opera") or shutil.which("Opera") + profile_dir = copy_chrome_profile(profile_path) + async with async_playwright() as pw: + kw = dict(user_data_dir=profile_dir, headless=True, args=CHROME_LAUNCH_ARGS) + if channel: + kw["channel"] = channel + if exe: + kw["executable_path"] = exe + context = await pw.chromium.launch_persistent_context(**kw) + page = context.pages[0] if context.pages else await context.new_page() + await page.goto("https://www.facebook.com/", wait_until="domcontentloaded", timeout=30000) + await page.wait_for_timeout(3000) + logged_in = "/login" not in page.url.lower() + await context.close() + shutil.rmtree(profile_dir, ignore_errors=True) + return {"logged_in": logged_in, "browser": browser, "path": profile_path} + except Exception as e: + if profile_dir: + try: shutil.rmtree(profile_dir, ignore_errors=True) + except: pass + return {"logged_in": False, "browser": browser, "path": profile_path, "error": str(e)} @app.post("/setup/check-login") async def setup_check_login(body: dict): - """Check if Facebook is logged in using the given profile.""" - profile_path = (body.get("profile_path") or "").strip() or FX_PROFILE or find_firefox_profile() - if not profile_path: - return {"logged_in": False, "reason": "no_profile", "error": "No Firefox profile found"} - try: - profile_dir = copy_firefox_profile(profile_path) - async with async_playwright() as pw: - context = await pw.firefox.launch_persistent_context( - user_data_dir=profile_dir, headless=True, - firefox_user_prefs={"dom.webdriver.enabled": False}, - ) - page = context.pages[0] if context.pages else await context.new_page() - await page.goto("https://www.facebook.com/", wait_until="domcontentloaded", timeout=30000) - await page.wait_for_timeout(3000) - logged_in = "/login" not in page.url.lower() - await context.close() - shutil.rmtree(profile_dir, ignore_errors=True) - return {"logged_in": logged_in, "reason": None if logged_in else "login_page"} - except Exception as e: - logger.error("Setup check-login failed: %s", e) - return {"logged_in": False, "reason": "error", "error": str(e)} + """Check if Facebook is logged in. Accepts optional 'browser' param, tries all if not given.""" + browser = (body.get("browser") or "").strip().lower() + profile_path = (body.get("profile_path") or "").strip() + + if browser and profile_path: + return await _check_browser_login(profile_path, browser) + + # No specific browser — try all detected + profiles = _detect_all_profiles() + priority = ["firefox", "opera", "chrome", "edge"] + for b in priority: + p = profiles.get(b, "") + if p: + result = await _check_browser_login(p, b) + if result.get("logged_in"): + return result + logger.info("%s not logged in, trying next", b) + return {"logged_in": False, "reason": "none_logged_in", "message": "No browser with Facebook login found"} @app.post("/agent/run") async def agent_run(task: str = Body(..., embed=True)): diff --git a/browser-use-service/requirements.txt b/browser-use-service/requirements.txt index 1d64ec7..4a3c4e7 100644 --- a/browser-use-service/requirements.txt +++ b/browser-use-service/requirements.txt @@ -1,5 +1,18 @@ +# ── Python Dependencies for the Facebook Scraper (FastAPI) ─────── +# Install via: pip install -r requirements.txt + +# Web framework for the REST API (health, setup, scrape endpoints) fastapi>=0.115.0 + +# ASGI server for running the FastAPI app uvicorn>=0.34.0 + +# Browser automation (launches Firefox, Chrome, Edge, Opera via Playwright) playwright>=1.49.0 + +# AI-powered browser agent (fallback when direct browser scraping is flagged) +# Uses ChatOllama locally — no API keys needed browser-use>=0.1.0 + +# LangChain integration for ChatOllama (provides the LLM for browser-use Agent) langchain-ollama>=0.2.0 diff --git a/scripts/ensure-ollama.mjs b/scripts/ensure-ollama.mjs index 8743779..9a299be 100644 --- a/scripts/ensure-ollama.mjs +++ b/scripts/ensure-ollama.mjs @@ -1,3 +1,9 @@ +// ── Ollama Launcher ──────────────────────────────────────────────── +// Checks if Ollama is already running. If not, finds the binary and +// starts it as a detached background process. +// Cross-platform: uses tasklist (Windows) or pgrep (Linux/Mac) to +// detect running process; uses where/which to find the binary. + import { execSync, spawn } from "node:child_process" import { platform } from "node:os" @@ -16,6 +22,7 @@ function isRunning() { function findOllama() { if (platform() === "win32") { + // Try PATH first, then common install locations try { return execSync("where ollama", { encoding: "utf8", timeout: 3000 }).trim().split("\n")[0] } catch {} @@ -36,11 +43,13 @@ if (!isRunning()) { process.exit(1) } console.log("Starting Ollama...") + // Spawn detached so it outlives this script and the npm process if (platform() === "win32") { spawn(bin, ["serve"], { stdio: "ignore", detached: true, windowsHide: true }).unref() } else { spawn(bin, ["serve"], { stdio: "ignore", detached: true }).unref() } + // Give it a moment to start listening execSync("sleep 3", { stdio: "ignore", timeout: 5000 }) } else { console.log("Ollama already running") diff --git a/scripts/open-browser.mjs b/scripts/open-browser.mjs index af8622f..ae1c04d 100644 --- a/scripts/open-browser.mjs +++ b/scripts/open-browser.mjs @@ -1,3 +1,9 @@ +// ── Browser Opener ───────────────────────────────────────────────── +// Opens the splash page in the user's default browser after an 8-second +// delay. The delay gives all services time to start before the user +// sees the loading screen. +// Cross-platform: uses start (Windows), open (Mac), or xdg-open (Linux). + import { execSync } from "node:child_process" import { platform } from "node:os" diff --git a/scripts/precheck.mjs b/scripts/precheck.mjs index 30f471f..e434519 100644 --- a/scripts/precheck.mjs +++ b/scripts/precheck.mjs @@ -1,9 +1,16 @@ +// ── Port Precheck ────────────────────────────────────────────────── +// Kills any existing processes on ports 3001, 3006, 3007, 3008. +// These are the AI server, Next.js frontend, Signaling server, and +// Python scraper respectively. +// Runs before anything else starts to avoid EADDRINUSE errors. + import { execSync } from "node:child_process" import { platform } from "node:os" const PORTS = [3001, 3006, 3007, 3008] if (platform() === "win32") { + // Windows: use netstat + findstr to find listening PIDs, then taskkill for (const port of PORTS) { try { const out = execSync(`netstat -ano | findstr "LISTENING" | findstr ":${port} "`, { encoding: "utf8", timeout: 5000 }) @@ -18,6 +25,7 @@ if (platform() === "win32") { } catch {} } } else { + // Linux/Mac: use lsof -ti to find PIDs, then kill -9 for (const port of PORTS) { try { const pid = execSync(`lsof -ti:${port} 2>/dev/null`, { encoding: "utf8", timeout: 5000 }).trim() diff --git a/scripts/run-python.mjs b/scripts/run-python.mjs index d3a3134..ead08c7 100644 --- a/scripts/run-python.mjs +++ b/scripts/run-python.mjs @@ -1,3 +1,9 @@ +// ── Python Runner ────────────────────────────────────────────────── +// Detects the system's Python executable (python vs python3) and runs +// a given script with arguments. Used by the dev:browser-use npm script. +// Avoids shell:true — spawns Python directly with its full path. +// Cross-platform: uses where (Windows) or which (Linux/Mac). + import { execSync, spawn } from "node:child_process" import { platform } from "node:os" @@ -23,5 +29,6 @@ if (!script) { process.exit(1) } +// Spawn Python with inherited stdio so the script's output is visible const proc = spawn(PYTHON, [script, ...args], { stdio: "inherit" }) proc.on("exit", (code) => process.exit(code ?? 1)) diff --git a/scripts/setup.mjs b/scripts/setup.mjs index 6deceda..1cbdf0c 100644 --- a/scripts/setup.mjs +++ b/scripts/setup.mjs @@ -1,9 +1,21 @@ +// ── One-command Setup ────────────────────────────────────────────── +// Run via: npm run setup +// Does the following in order: +// 1. npm install (Node.js dependencies) +// 2. pip install -r requirements.txt (Python dependencies) +// 3. playwright install firefox chromium (Playwright browsers) +// 4. Copies .env.example to .env.local if not exists +// +// All steps are cross-platform (Windows, Mac, Linux). +// Uses execSync for simplicity since each step blocks the next. + import { execSync } from "node:child_process" import { existsSync, copyFileSync } from "node:fs" import { platform } from "node:os" const SEP = platform() === "win32" ? "&" : ";" +// Auto-detect Python executable (python vs python3) function detectPython() { const candidates = platform() === "win32" ? ["python", "python3"] : ["python3", "python"] for (const cmd of candidates) { @@ -16,6 +28,7 @@ function detectPython() { process.exit(1) } +// Auto-detect pip (pip vs pip3), fall back to python -m pip function detectPip(python) { const candidates = platform() === "win32" ? ["pip", "pip3"] : ["pip3", "pip"] for (const cmd of candidates) { @@ -45,13 +58,13 @@ console.log("=== CoastIT CRM Setup ===\n") // 1. Node dependencies run("npm install", "Installing Node.js dependencies") -// 2. Python dependencies +// 2. Python dependencies (run from browser-use-service directory) run(`cd browser-use-service ${SEP} ${PIP} install -r requirements.txt`, "Installing Python dependencies") -// 3. Playwright browsers +// 3. Playwright browsers (Firefox for primary scraping, Chromium for Chrome/Edge/Opera + Agent fallback) run(`${PY} -m playwright install firefox chromium`, "Installing Playwright browsers") -// 4. .env file +// 4. .env file — create from template if it doesn't exist if (!existsSync(".env.local")) { console.log("\n── Creating .env.local ──") copyFileSync(".env.example", ".env.local") @@ -60,7 +73,7 @@ if (!existsSync(".env.local")) { console.log("\n── .env.local already exists, skipping ──") } -// 5. Ollama model +// 5. Remaining manual steps console.log("\n── Next steps ──") console.log(" 1. Make sure PostgreSQL is running with database 'crm'") console.log(" 2. Pull the Ollama model: ollama pull dolphin-llama3:8b") diff --git a/splash.html b/splash.html index 83696eb..406ec71 100644 --- a/splash.html +++ b/splash.html @@ -5,6 +5,7 @@