Update on auto setup

This commit is contained in:
Ace
2026-06-26 13:07:40 +02:00
parent 5feb95187c
commit 9ce9506e8e
9 changed files with 769 additions and 265 deletions
+120 -50
View File
@@ -1,3 +1,13 @@
// ── CRM AI Server ──────────────────────────────────────────────────
// Provides:
// - Chat API (POST /ai/chat) — routes user messages to Ollama for sales coaching
// - Setup wizard endpoints (GET /setup/status, POST /setup/profile, etc.)
// - Combined /status endpoint for splash page health polling
// - Configuration routes (GET/POST /ai/instructions, GET /ai/jobs)
// - Model pull support (POST /setup/ollama/pull)
//
// This is a zero-dependency Node.js HTTP server (no Express needed).
import http from "node:http"
import fs from "node:fs"
import path from "node:path"
@@ -8,6 +18,9 @@ const __dirname = path.dirname(fileURLToPath(import.meta.url))
const ROOT = path.resolve(__dirname, "..")
// ── Load .env.local ──────────────────────────────────────────────
// Reads key=value pairs and sets them as process.env so they're
// available throughout the server. Ignores comments and blank lines.
// Values with matching quotes are unquoted.
try {
const envPath = path.join(ROOT, ".env.local")
const envContent = fs.readFileSync(envPath, "utf-8")
@@ -23,7 +36,7 @@ try {
}
console.log("Loaded .env.local")
} catch {
// .env.local may not exist, ignore
// .env.local may not exist (first run), which is fine
}
// ── Config from env ─────────────────────────────────────────────
@@ -36,10 +49,14 @@ const JOBS_PATH = process.env.JOBS_PATH || path.join(ROOT, "data", "ai", "jobs.j
const AI_MD_PATH = process.env.AI_MD_PATH || path.join(ROOT, "data", "ai", "ai.md")
// ── Setup state ──────────────────────────────────────────────────
// Tracks the Ollama model pull process so the setup wizard can
// poll for download progress.
let pullProcess = null
let pullProgress = { status: "idle", progress: 0, message: "" }
// ── Job loading ─────────────────────────────────────────────────
// Loads job categories from a JSONL file (one JSON object per line).
// Used as context for the AI sales coach chat responses.
function loadJobs() {
try {
const content = fs.readFileSync(JOBS_PATH, "utf-8")
@@ -64,6 +81,8 @@ function loadJobs() {
}
// ── ai.md management ────────────────────────────────────────────
// ai.md is a Markdown file containing system instructions for the AI.
// It can be read, written, or appended to via the API.
function readInstructions() {
try {
return fs.readFileSync(AI_MD_PATH, "utf-8")
@@ -78,6 +97,7 @@ function writeInstructions(content) {
}
function appendToImprovementLog(entry) {
// Adds a timestamped entry to the ## Improvement Log section of ai.md
const current = readInstructions()
const timestamp = new Date().toISOString().replace("T", " ").substring(0, 16)
const logEntry = `\n- ${timestamp}${entry}`
@@ -101,10 +121,13 @@ function appendToImprovementLog(entry) {
}
// ── Chat handler ────────────────────────────────────────────────
// scrapeFacebook() calls the scraper service (port 3008) to get leads.
// handleChat() processes user messages — triggers lead scraping when
// the user asks for "leads" or "listings", otherwise routes to Ollama
// for AI-powered sales coaching.
async function scrapeFacebook() {
const profilePath = process.env.FX_PROFILE || ""
const urlPath = `/scrape/facebook?force=true${profilePath ? `&profile_path=${encodeURIComponent(profilePath)}` : ""}`
const logPath = "C:\\Users\\USER-PC\\AppData\\Local\\Temp\\opencode\\ai-scrape-debug.log"
try {
const body = await new Promise((resolve, reject) => {
const req = http.request({ hostname: "127.0.0.1", port: 3008, path: urlPath, method: "POST", timeout: 360000 }, (res) => {
@@ -139,6 +162,7 @@ function formatLeads(leads) {
}
async function handleChat(userMessage, userId, userRole) {
// If the user asks for leads, trigger the scraper
const lowerMsg = userMessage.toLowerCase()
const triggerWords = ["lists", "listings", "leads", "recent leads", "pull leads", "show me leads", "show listings"]
@@ -150,6 +174,7 @@ async function handleChat(userMessage, userId, userRole) {
return "Scraper returned no results or encountered an error. Try again later."
}
// Otherwise, build a system prompt with job context and send to Ollama
const jobs = loadedJobs
const instructions = readInstructions()
@@ -189,7 +214,7 @@ Provide concise, actionable sales advice. When asked about a specific job catego
const data = await ollamaRes.json()
const responseText = data.message?.content || ""
// Try to persist to DB (best-effort)
// Persist conversation to PostgreSQL (best-effort — table may not exist yet)
try {
if (pgPool && userId) {
await pgPool.query(
@@ -205,6 +230,8 @@ Provide concise, actionable sales advice. When asked about a specific job catego
}
// ── PG pool (lazy init) ────────────────────────────────────────
// PostgreSQL connection pool for storing conversation history.
// Lazy-initialized so the server starts even without a DB.
let pgPool = null
async function initPg() {
if (!DATABASE_URL) return
@@ -246,8 +273,9 @@ function parseURL(req) {
return { pathname: url.pathname, searchParams: url.searchParams }
}
// ── HTTP Server ─────────────────────────────────────────────────
const server = http.createServer(async (req, res) => {
// CORS
// CORS headers — allow the Next.js frontend (port 3006) to call us
res.setHeader("Access-Control-Allow-Origin", "*")
res.setHeader("Access-Control-Allow-Methods", "GET, POST, OPTIONS")
res.setHeader("Access-Control-Allow-Headers", "Content-Type")
@@ -280,10 +308,12 @@ const server = http.createServer(async (req, res) => {
}
// GET /status — combined health of all services
// Used by the splash page to check if AI, Scraper, and Frontend are ready.
// Polls each service internally to avoid cross-origin CORS issues.
if (req.method === "GET" && pathname === "/status") {
const { default: http } = await import("http")
const results = { ai: true }
// Check scraper
// Check scraper (port 3008)
try {
await new Promise((resolve, reject) => {
const r = http.get("http://127.0.0.1:3008/health", { timeout: 3000 }, (res) => { res.resume(); resolve() })
@@ -291,7 +321,7 @@ const server = http.createServer(async (req, res) => {
})
results.scraper = true
} catch { results.scraper = false }
// Check frontend
// Check frontend (port 3006)
try {
await new Promise((resolve, reject) => {
const r = http.get("http://127.0.0.1:3006", { timeout: 3000 }, (res) => { res.resume(); resolve() })
@@ -305,6 +335,11 @@ const server = http.createServer(async (req, res) => {
// ── Setup endpoints ─────────────────────────────────────────
// GET /setup/status — check environment
// Called by the splash page on boot. Returns info about:
// - Ollama availability
// - Model presence
// - Detected browsers with login status
// - Whether this is a first run (wizard needed)
if (req.method === "GET" && pathname === "/setup/status") {
const envExists = fs.existsSync(path.join(ROOT, ".env.local"))
@@ -327,33 +362,41 @@ const server = http.createServer(async (req, res) => {
} catch {}
}
// Profile auto-detect
let profilePath = process.env.FX_PROFILE || ""
let profileDetected = !!profilePath
if (!profileDetected) {
try {
const r = await fetch("http://127.0.0.1:3008/health", { signal: AbortSignal.timeout(2000) })
if (r.ok) {
const diag = await (await fetch("http://127.0.0.1:3008/setup/profile", { signal: AbortSignal.timeout(5000) })).json()
if (diag.path) { profilePath = diag.path; profileDetected = true }
}
} catch {}
}
// Login check
// Detect all browsers via scraper
let browsers = { firefox: { path: null }, opera: { path: null }, chrome: { path: null }, edge: { path: null } }
let facebookLoggedIn = false
if (profileDetected) {
let selectedBrowser = process.env.SELECTED_BROWSER || ""
try {
await fetch("http://127.0.0.1:3008/health", { signal: AbortSignal.timeout(2000) })
const profiles = await (await fetch("http://127.0.0.1:3008/setup/profile", { signal: AbortSignal.timeout(5000) })).json()
for (const [b, p] of Object.entries(profiles)) {
if (p) browsers[b] = { path: p }
}
// Check login for the selected browser first, then try all
const detectedList = Object.entries(browsers).filter(([, v]) => v.path)
for (const [b, v] of detectedList) {
try {
const r = await fetch("http://127.0.0.1:3008/setup/check-login", {
method: "POST", headers: { "Content-Type": "application/json" },
body: JSON.stringify({ profile_path: profilePath }),
signal: AbortSignal.timeout(15000),
body: JSON.stringify({ browser: b, profile_path: v.path }),
signal: AbortSignal.timeout(20000),
})
if (r.ok) { const d = await r.json(); facebookLoggedIn = d.logged_in === true }
if (r.ok) {
const d = await r.json()
browsers[b].logged_in = d.logged_in === true
if (d.logged_in && !facebookLoggedIn) {
facebookLoggedIn = true
if (!selectedBrowser) selectedBrowser = b
}
}
} catch {}
}
} catch {}
const firstRun = !envExists || !ollamaRunning || !profileDetected || !modelAvailable
const anyDetected = Object.values(browsers).some(v => v.path)
// first_run = any setup step is incomplete
const firstRun = !envExists || !ollamaRunning || !anyDetected || !facebookLoggedIn || !modelAvailable
return sendJSON(res, 200, {
first_run: firstRun,
@@ -361,46 +404,71 @@ const server = http.createServer(async (req, res) => {
ollama_running: ollamaRunning,
model_available: modelAvailable,
model_name: MODEL,
profile_detected: profileDetected,
profile_path: profilePath || null,
selected_browser: selectedBrowser,
browsers,
facebook_logged_in: facebookLoggedIn,
})
}
// POST /setup/profile — save profile path to .env.local
// POST /setup/profile — save selected browser + path to .env.local
// Called by the setup wizard when the user confirms their browser choice.
// Writes SELECTED_BROWSER and the matching profile env var to .env.local.
if (req.method === "POST" && pathname === "/setup/profile") {
const body = await parseBody(req)
const browserName = (body.browser || "").trim().toLowerCase()
const profilePath = (body.path || "").trim()
if (!profilePath) return sendJSON(res, 400, { error: "Path required" })
if (!fs.existsSync(profilePath)) return sendJSON(res, 400, { error: "Path does not exist" })
if (!browserName || !["firefox", "opera", "chrome", "edge"].includes(browserName))
return sendJSON(res, 400, { error: "Valid browser required (firefox/opera/chrome/edge)" })
if (!profilePath)
return sendJSON(res, 400, { error: "Path required" })
const envKey = browserName === "firefox" ? "FX_PROFILE"
: browserName === "opera" ? "OPERA_PROFILE"
: browserName === "edge" ? "EDGE_PROFILE"
: "CHROME_PROFILE"
const envPath = path.join(ROOT, ".env.local")
let content = ""
try { content = fs.readFileSync(envPath, "utf-8") } catch {}
const lines = content.split("\n")
let found = false
let lines = content.split("\n")
// Update or add SELECTED_BROWSER
let foundSel = false
for (let i = 0; i < lines.length; i++) {
if (lines[i].trim().startsWith("FX_PROFILE=")) {
lines[i] = `FX_PROFILE=${profilePath}`
found = true
if (lines[i].trim().startsWith("SELECTED_BROWSER=")) {
lines[i] = `SELECTED_BROWSER=${browserName}`
foundSel = true
break
}
}
if (!found) lines.push(`FX_PROFILE=${profilePath}`)
if (!foundSel) lines.push(`SELECTED_BROWSER=${browserName}`)
// Update or add browser profile
let foundProf = false
for (let i = 0; i < lines.length; i++) {
if (lines[i].trim().startsWith(`${envKey}=`)) {
lines[i] = `${envKey}=${profilePath}`
foundProf = true
break
}
}
if (!foundProf) lines.push(`${envKey}=${profilePath}`)
fs.writeFileSync(envPath, lines.join("\n"), "utf-8")
process.env.FX_PROFILE = profilePath
return sendJSON(res, 200, { success: true, path: profilePath })
process.env.SELECTED_BROWSER = browserName
process.env[envKey] = profilePath
return sendJSON(res, 200, { success: true, browser: browserName, path: profilePath })
}
// POST /setup/check-login — verify Facebook login in the given profile
// POST /setup/check-login — proxy to scraper, accepts browser + profile_path
// The splash page calls this (via the AI server) to verify Facebook login status.
if (req.method === "POST" && pathname === "/setup/check-login") {
const body = await parseBody(req)
const profilePath = body.profile_path || process.env.FX_PROFILE || ""
const browserName = (body.browser || "").trim().toLowerCase() || process.env.SELECTED_BROWSER || ""
const profilePath = (body.profile_path || "").trim()
if (!profilePath) return sendJSON(res, 200, { logged_in: false, reason: "no_profile" })
try {
const r = await fetch("http://127.0.0.1:3008/setup/check-login", {
method: "POST", headers: { "Content-Type": "application/json" },
body: JSON.stringify({ profile_path: profilePath }),
body: JSON.stringify({ browser: browserName, profile_path: profilePath }),
signal: AbortSignal.timeout(20000),
})
if (r.ok) { const d = await r.json(); return sendJSON(res, 200, d) }
@@ -409,6 +477,8 @@ const server = http.createServer(async (req, res) => {
}
// POST /setup/ollama/pull — start pulling the model
// Spawns "ollama pull" as a child process. The setup wizard polls
// the progress endpoint to show a download progress bar.
if (req.method === "POST" && pathname === "/setup/ollama/pull") {
if (pullProcess) return sendJSON(res, 200, { status: "already_running" })
pullProgress = { status: "downloading", progress: 0, message: "Starting..." }
@@ -432,22 +502,23 @@ const server = http.createServer(async (req, res) => {
}
// GET /setup/ollama/pull/progress
// Returns current download progress for the setup wizard.
if (req.method === "GET" && pathname === "/setup/ollama/pull/progress") {
return sendJSON(res, 200, pullProgress)
}
// GET /ai/jobs
// GET /ai/jobs — return loaded job categories
if (req.method === "GET" && pathname === "/ai/jobs") {
return sendJSON(res, 200, { jobs: loadedJobs })
}
// GET /ai/instructions
// GET /ai/instructions — return current ai.md content
if (req.method === "GET" && pathname === "/ai/instructions") {
const instructions = readInstructions()
return sendJSON(res, 200, { success: true, instructions })
}
// POST /ai/instructions
// POST /ai/instructions — update ai.md or append improvement log entry
if (req.method === "POST" && pathname === "/ai/instructions") {
const body = await parseBody(req)
if (body.content) {
@@ -461,18 +532,17 @@ const server = http.createServer(async (req, res) => {
})
}
// POST /ai/chat
// POST /ai/chat — main AI chat endpoint
// Accepts { message, user_id?, user_role? } and returns AI response.
// user_role must be "sales", "admin", or "super_admin" if provided.
if (req.method === "POST" && pathname === "/ai/chat") {
const startTime = Date.now()
const chunks = []
req.on("data", c => chunks.push(c))
req.on("end", () => {
const rawBody = Buffer.concat(chunks).toString()
try { fs.appendFileSync("C:\\Users\\USER-PC\\AppData\\Local\\Temp\\opencode\\ai-req-log.txt",
`${new Date().toISOString()} headers=${JSON.stringify(req.headers)} body=${rawBody}\n`) } catch {}
try {
const body = JSON.parse(rawBody)
// Continue processing
processRequest(req, res, body, startTime)
} catch {
sendJSON(res, 400, { error: "Invalid JSON" })
@@ -481,7 +551,7 @@ const server = http.createServer(async (req, res) => {
return
}
// Separate handler
// Separate handler for /ai/chat (defined here due to hoisting within the IIFE)
async function processRequest(req, res, body, startTime) {
const { message, user_id, user_role } = body
@@ -498,7 +568,7 @@ async function processRequest(req, res, body, startTime) {
return sendJSON(res, 200, { response })
}
// 404
// 404 fallback
sendJSON(res, 404, { error: "Not found" })
} catch (err) {
console.error("Request error:", err)
+431 -53
View File
@@ -1,3 +1,4 @@
# ── Imports ──────────────────────────────────────────────────────────
import os, json, asyncio, re, shutil, sqlite3, urllib.parse, random, logging, tempfile
from datetime import datetime, timedelta
from fastapi import FastAPI, Query, Body
@@ -8,8 +9,12 @@ from browser_use import Agent, Browser
from langchain_ollama import ChatOllama
# ── Helpers ──────────────────────────────────────────────────────────
def make_ollama(model: str | None = None, **kwargs) -> ChatOllama:
"""Create ChatOllama with required attrs for browser-use Agent compatibility."""
"""
Create ChatOllama with required attrs for browser-use Agent compatibility.
The browser-use Agent expects llm.name, llm.model_name, and llm.provider attrs.
"""
llm = ChatOllama(model=model or CLASSIFY_MODEL, **kwargs)
object.__setattr__(llm, 'provider', 'ollama')
object.__setattr__(llm, 'name', 'ChatOllama')
@@ -17,6 +22,7 @@ def make_ollama(model: str | None = None, **kwargs) -> ChatOllama:
return llm
# ── Logging & App Setup ──────────────────────────────────────────────
logging.basicConfig(level=logging.INFO, format='%(asctime)s [%(levelname)s] %(message)s')
logger = logging.getLogger(__name__)
@@ -29,13 +35,24 @@ app.add_middleware(
)
PORT = int(os.getenv("PORT", "3008"))
# ── AI / Ollama Config ───────────────────────────────────────────────
OLLAMA_URL = os.getenv("OLLAMA_URL", "http://localhost:11434")
CLASSIFY_MODEL = os.getenv("CLASSIFY_MODEL", "dolphin-llama3:8b")
# ── Env Vars (set via .env.local) ──────────────────────────────────────
# These store detected browser profile paths from the setup wizard.
FX_PROFILE = os.getenv('FX_PROFILE', '')
CHROME_PROFILE = os.getenv('CHROME_PROFILE', '')
OPERA_PROFILE = os.getenv('OPERA_PROFILE', '')
EDGE_PROFILE = os.getenv('EDGE_PROFILE', '')
# SELECTED_BROWSER determines which browser the scrape dispatcher tries first.
# Values: "firefox", "chrome", "opera", "edge", or "" (auto-detect all).
SELECTED_BROWSER = os.getenv('SELECTED_BROWSER', '')
# ── Chromium Launch Args ──────────────────────────────────────────────
# These flags reduce detectable automation signals in Chrome/Edge/Opera.
# They disable background services, sync, updates, throttling, popups, etc.
CHROME_LAUNCH_ARGS = [
'--headless=new',
'--disable-blink-features=AutomationControlled',
@@ -60,20 +77,36 @@ CHROME_LAUNCH_ARGS = [
]
# ── Profile Detection ────────────────────────────────────────────────
# All find_*_profile() functions return the parent directory (User Data or equivalent),
# NOT the "Default" subdirectory. This is important because copy_chrome_profile()
# appends "Default" internally, so returning the full Default path would cause
# duplicated path segments (e.g. ".../User Data/Default/Default").
def detect_browser_from_profile(profile_path: str) -> str | None:
"""Detect browser type from profile path. Returns 'firefox', 'chromium', or None."""
"""
Infer browser type from the profile path string.
Checks for keywords like "firefox", "opera", "edge", "chrome" in the path.
Returns "firefox", "chrome", "opera", "edge", or None.
"""
if not profile_path:
return None
p = profile_path.lower().replace('\\', '/')
if 'firefox' in p or '.default' in p or '.dev-edition' in p:
return 'firefox'
if 'chrome' in p or 'chromium' in p or 'edge' in p:
return 'chromium'
if 'opera' in p:
return 'opera'
if 'edge' in p:
return 'edge'
if 'chrome' in p or 'chromium' in p:
return 'chrome'
return None
def find_firefox_profile() -> str | None:
"""Auto-detect Firefox profile directory cross-platform."""
"""Auto-detect Firefox profile directory cross-platform.
Scans the Profiles dir and returns the first .default or .dev-edition folder found,
preferring "default-release" profiles."""
import sys
home = os.path.expanduser("~")
candidates = []
@@ -102,7 +135,8 @@ def find_firefox_profile() -> str | None:
def find_chrome_profile() -> str | None:
"""Auto-detect Chrome/Chromium profile directory cross-platform."""
"""Auto-detect Chrome profile (User Data dir) cross-platform.
Returns the User Data parent dir (NOT the Default subdirectory)."""
import sys
home = os.path.expanduser("~")
@@ -113,15 +147,60 @@ def find_chrome_profile() -> str | None:
else:
base = os.path.join(home, ".config", "google-chrome")
default_profile = os.path.join(base, "Default")
if os.path.isdir(default_profile):
logger.info("Auto-detected Chrome profile: %s", default_profile)
return default_profile
if os.path.isdir(os.path.join(base, "Default")):
logger.info("Auto-detected Chrome profile: %s", base)
return base
return None
def find_opera_profile() -> str | None:
"""Auto-detect Opera profile directory cross-platform.
Opera stores its profile at the "Opera Stable" level (no "User Data/Default"
nesting like Chrome/Edge), so this returns the profile root directly."""
import sys
home = os.path.expanduser("~")
if sys.platform == "win32":
base = os.path.join(os.environ.get("APPDATA", ""), "Opera Software", "Opera Stable")
elif sys.platform == "darwin":
base = os.path.join(home, "Library", "Application Support", "com.operasoftware.Opera")
else:
base = os.path.join(home, ".config", "opera")
if os.path.isdir(base) and os.listdir(base):
logger.info("Auto-detected Opera profile: %s", base)
return base
return None
def find_edge_profile() -> str | None:
"""Auto-detect Edge profile (User Data dir) cross-platform.
Returns the User Data parent dir (NOT the Default subdirectory)."""
import sys
home = os.path.expanduser("~")
if sys.platform == "win32":
base = os.path.join(os.environ.get("LOCALAPPDATA", ""), "Microsoft", "Edge", "User Data")
elif sys.platform == "darwin":
base = os.path.join(home, "Library", "Application Support", "Microsoft Edge")
else:
base = os.path.join(home, ".config", "microsoft-edge")
if os.path.isdir(os.path.join(base, "Default")):
logger.info("Auto-detected Edge profile: %s", base)
return base
return None
# ── Profile Copying ─────────────────────────────────────────────────
# Both functions copy essential cookies/login/storage files from the user's
# real browser profile to a temp directory. Playwright then uses these temp
# profiles via launch_persistent_context(), preserving Facebook login state
# without modifying the user's actual profile.
def copy_firefox_profile(src_path: str) -> str:
"""Copy essential Firefox profile files to a temp dir for Playwright."""
"""Copy essential Firefox profile files (cookies, localStorage, permissions) to a temp dir for Playwright.
Also creates a minimal profiles.ini so Firefox treats it as a valid profile."""
essential = ['cookies.sqlite', 'webappsstore.sqlite', 'permissions.sqlite']
dst = tempfile.mkdtemp(prefix='fb_fx_profile_')
for f_name in essential:
@@ -135,7 +214,12 @@ def copy_firefox_profile(src_path: str) -> str:
def copy_chrome_profile(user_data_dir: str, profile_dir: str = 'Default') -> str:
"""Copy Chrome profile to temp dir for launch_persistent_context."""
"""Copy Chromium-based profile (Chrome/Edge/Opera) to a temp dir.
Copies Cookies, Login Data, Bookmarks, Web Data, Local Storage, Session Storage, and Local State.
Args:
user_data_dir: Parent directory (e.g. ".../Chrome/User Data")
profile_dir: Subdirectory name, typically "Default" (default)
"""
essential = ['Cookies', 'Login Data', 'Bookmarks', 'Web Data']
dst = tempfile.mkdtemp(prefix='fb_chrome_udir_')
src_profile = os.path.join(user_data_dir, profile_dir)
@@ -159,7 +243,8 @@ def copy_chrome_profile(user_data_dir: str, profile_dir: str = 'Default') -> str
def ensure_ublock_extension() -> str | None:
"""Download and extract uBlock Origin extension for ad blocking."""
"""Download and extract uBlock Origin extension for ad blocking via Chromium.
Cached after first download to avoid re-downloading on every scrape."""
ext_dir = os.path.join(tempfile.gettempdir(), 'fb_ublock_origin')
manifest = os.path.join(ext_dir, 'manifest.json')
if os.path.exists(manifest):
@@ -182,10 +267,15 @@ def ensure_ublock_extension() -> str | None:
return None
# ── Agent Output Parsing ──────────────────────────────────────────────
# When the browser-use Agent fallback is used, it returns natural language
# output containing post listings. This function tries to extract structured
# data from that output, first attempting JSON parsing, then falling back to
# a line-by-line heuristic (numbered or bulleted items).
def extract_agent_posts(agent_result: str, page_text: str) -> list[dict]:
"""Parse posts from agent output or page text fallback."""
posts = []
# Try JSON from agent output
json_match = re.search(r'\[.*?\]', agent_result, re.DOTALL)
if json_match:
try:
@@ -221,6 +311,12 @@ def extract_agent_posts(agent_result: str, page_text: str) -> list[dict]:
return posts
# ── Keyword Filters for Lead Detection ──────────────────────────────
# BROAD_KEYWORDS — catch any post that might relate to web development needs.
# OFFER_PATTERNS — regex patterns matching service offers (which we ignore).
# REQUEST_PATTERNS — regex patterns matching people asking for help/services.
# Together these form a two-pass filter + AI classification pipeline.
BROAD_KEYWORDS = [
"website", "web design", "web develop", "web dev",
"build my website", "build a website", "create a website",
@@ -290,6 +386,13 @@ REQUEST_PATTERNS = [
r"\bquote\s+(please|for|to)\b",
]
# ── Anti-Detection Scripts ────────────────────────────────────────────
# These JS scripts are injected into every page via add_init_script().
# They override navigator properties (webdriver, userAgent, plugins, etc.)
# and WebGL/canvas/audio fingerprints to make Playwright harder to detect.
# Firefox and Chromium variants differ slightly (e.g. Chrome needs the
# "window.chrome" object; Firefox must NOT have it).
def firefox_init_script() -> str:
return r"""// Firefox Anti-Detection
Object.defineProperty(navigator, 'webdriver', { get: () => undefined });
@@ -384,18 +487,25 @@ Object.defineProperty(navigator, 'plugins', { get: () => [1,2,3,4,5].map(i => ({
Object.defineProperty(navigator, 'mimeTypes', { get: () => [{type:'application/pdf',suffixes:'pdf',description:'Portable Document Format',enabledPlugin:navigator.plugins[0]},{type:'text/pdf',suffixes:'pdf',description:'Portable Document Format',enabledPlugin:navigator.plugins[0]}] });"""
# ── Text Classification Utilities ────────────────────────────────────
def kw_match(text: str) -> bool:
t = text.lower()
return any(kw in t for kw in BROAD_KEYWORDS)
def is_request(text: str) -> bool:
"""Detect if text contains a request for services (e.g. "I need", "looking for")."""
t = text.lower()
return any(re.search(p, t) for p in REQUEST_PATTERNS)
def is_offer(text: str) -> bool:
"""Detect if text is an offer/advertisement (we skip these, not leads)."""
t = text.lower()
return any(re.search(p, t) for p in OFFER_PATTERNS)
# ── Facebook Search Queries ───────────────────────────────────────────
# These are the search terms the scraper will use on Facebook.
# They target people looking for web development / website services.
# Each run picks 2-4 random queries to vary behavior and reduce detection.
FB_SEARCHES = [
"looking for web developer",
"need a website designed",
@@ -428,6 +538,12 @@ VIEWPORTS = [
{'width': 1920, 'height': 1080},
]
# ── Cookie Extraction ────────────────────────────────────────────────
# Reads Facebook cookies from the browser profile's SQLite database.
# Firefox stores cookies in cookies.sqlite; Chromium stores them in Cookies
# (a SQLite DB as well). This function handles Firefox's format.
# The cookies are copied to a temp file first to avoid locking issues.
async def get_fb_cookies(profile_path: str | None = None):
cookie_db_path = profile_path or FX_PROFILE
if not cookie_db_path:
@@ -472,6 +588,9 @@ async def get_fb_cookies(profile_path: str | None = None):
pass
return []
# ── Date Parsing ─────────────────────────────────────────────────────
# Facebook displays dates in relative format ("2h ago", "Yesterday", "Monday")
# or various absolute formats. These functions normalize them to YYYY-MM-DD.
WEEKDAY_ORDER = ['monday', 'tuesday', 'wednesday', 'thursday', 'friday', 'saturday', 'sunday']
def _parse_fb_date(block: list[str]) -> str:
@@ -525,6 +644,12 @@ def _clean_fb_text(text: str) -> str:
cleaned_lines.append(stripped)
return '\n'.join(cleaned_lines)
# ── Post Extraction ──────────────────────────────────────────────────
# Two strategies for extracting posts from page content:
# 1. _extract_posts_from_elements — uses structured DOM data from _get_article_elements()
# 2. _extract_posts_from_text — fallback that parses raw page text line by line
# Both apply dedup (seen_texts), offer filtering, and request scoring.
def _extract_posts_from_elements(elements: list[dict], base_url: str) -> list[dict]:
posts = []
seen_texts = set()
@@ -617,6 +742,11 @@ def _extract_posts_from_text(raw: str, url: str) -> list[dict]:
cur.pop(0)
return posts
# ── Human-like Behavior Simulation ──────────────────────────────────
# These functions add random delays, mouse movements, and scroll patterns
# to make automated browsing look more like a real human user.
# This is critical for avoiding Facebook's bot detection.
async def human_scroll(page, steps: int = None, total_delay: float = None):
steps = steps or random.randint(2, 5)
total_delay = total_delay or random.uniform(6, 18)
@@ -652,6 +782,12 @@ async def random_idle(page):
except Exception:
pass
# ── Facebook DOM Parsing ─────────────────────────────────────────────
# This JS function runs inside the page context to extract structured post
# data (text, author, URL, date) from Facebook's complex DOM structure.
# It tries multiple selectors (article, feed, etc.) and uses the first
# one that returns results, since Facebook's DOM changes frequently.
async def _get_article_elements(page) -> list[dict]:
return await page.evaluate('''() => {
const results = [];
@@ -713,6 +849,8 @@ async def _get_article_elements(page) -> list[dict]:
}''')
async def _ensure_page(page, context):
"""Check if the current page is still alive. If closed (e.g. by a popup),
create a fresh page and navigate to Facebook."""
try:
await page.evaluate('1')
return page
@@ -728,6 +866,12 @@ async def _ensure_page(page, context):
await page.wait_for_timeout(random.randint(3000, 8000))
return page
# ── Facebook Search & Scrape ─────────────────────────────────────────
# Performs a Facebook search for a given query, scrolls through results,
# extracts posts, and returns them. Includes randomization of scroll behavior,
# mouse movements, and idle actions to mimic human browsing patterns.
async def search_facebook(page, context, query: str):
page = await _ensure_page(page, context)
url = f'https://www.facebook.com/search/posts/?q={urllib.parse.quote(query)}'
@@ -764,6 +908,10 @@ async def search_facebook(page, context, query: str):
return page, []
return page, posts
# ── Detection Signals ────────────────────────────────────────────────
# Keywords/phrases in page URL or body text that indicate Facebook's
# security/bot detection systems have flagged the session.
DETECTION_SIGNALS = [
'/checkpoint/', '/login.php?', 'action=security_check',
'unusual activity', 'suspicious login', 'suspicious activity',
@@ -784,53 +932,89 @@ def check_detection_signals(page_url: str, page_text: str = '') -> str | None:
def cleanup_chrome():
"""Kill orphaned Chrome headless shell processes. These sometimes linger
after failed scrapes and interfere with subsequent runs."""
import subprocess, signal
try:
subprocess.run(["taskkill", "/F", "/IM", "chrome-headless-shell.exe"], capture_output=True, timeout=5)
except Exception:
pass
# ── Main Scrape Dispatcher ────────────────────────────────────────────
# scrape_facebook() is the main entry point. It:
# 1. Resolves the browser profile path (from SELECTED_BROWSER env var or auto-detect)
# 2. Dispatches to the correct browser-specific scraper
# 3. If the browser scrape is flagged by Facebook, falls through to the Agent fallback
#
# Priority order for browser selection:
# - SELECTED_BROWSER env var (set by setup wizard) → that browser's scraper
# - Auto-detect: Firefox → Opera → Chrome → Edge (first found wins)
# - If no profile found → Agent (browser-use + ChatOllama) fallback
async def scrape_facebook(profile_path: str | None = None, force: bool = False) -> dict:
"""Dispatcher — Firefox primary, browser-use Agent fallback."""
effective_path = profile_path or FX_PROFILE
"""Dispatcher — respect SELECTED_BROWSER, fall through on flag, then Agent."""
effective_path = profile_path or ""
browser_type = ""
# Auto-detect Firefox profile if none provided
if not effective_path:
detected = find_firefox_profile()
if detected:
effective_path = detected
os.environ["FX_PROFILE"] = detected
# Resolve profile and browser type
if effective_path:
browser_type = detect_browser_from_profile(effective_path) or ""
elif SELECTED_BROWSER == "firefox":
effective_path = FX_PROFILE or find_firefox_profile() or ""
browser_type = "firefox"
elif SELECTED_BROWSER == "opera":
effective_path = OPERA_PROFILE or find_opera_profile() or ""
browser_type = "opera"
elif SELECTED_BROWSER == "edge":
effective_path = EDGE_PROFILE or find_edge_profile() or ""
browser_type = "edge"
elif SELECTED_BROWSER == "chrome":
effective_path = CHROME_PROFILE or find_chrome_profile() or ""
browser_type = "chrome"
else:
# Auto-detect — try all in priority order
for name, env_var, find_fn in [
("firefox", "FX_PROFILE", find_firefox_profile),
("opera", "OPERA_PROFILE", find_opera_profile),
("chrome", "CHROME_PROFILE", find_chrome_profile),
("edge", "EDGE_PROFILE", find_edge_profile),
]:
p = os.getenv(env_var, "") or find_fn() or ""
if p:
effective_path = p
browser_type = name
break
browser_type = detect_browser_from_profile(effective_path)
if not browser_type or not effective_path:
logger.info("No profile found — falling back to Agent")
return await _scrape_with_agent(force)
if not browser_type and CHROME_PROFILE:
browser_type = detect_browser_from_profile(CHROME_PROFILE)
effective_path = CHROME_PROFILE
logger.info("Selected browser: %s (profile: %s)", browser_type, effective_path)
# Auto-detect Chrome profile if still nothing
if not browser_type:
detected_chrome = find_chrome_profile()
if detected_chrome:
effective_path = detected_chrome
browser_type = 'chromium'
logger.info("Detected browser: %s (profile: %s)", browser_type or "none", effective_path)
# Firefox primary (raw Playwright, stealth)
# Firefox path
if browser_type == "firefox":
result = await _scrape_with_firefox(effective_path, force)
if result.get("success") or not result.get("flagged"):
return result
logger.warning("Firefox path failed (%s), falling back to Agent", result.get("flag_reason", "unknown"))
logger.warning("Firefox flagged (%s), trying Agent", result.get("flag_reason", "unknown"))
return await _scrape_with_agent(force)
# CHROME_PROFILE set or no profile → Agent
if browser_type == "chromium":
# Chromium-based (chrome / opera / edge)
result = await _scrape_with_chromium(effective_path, browser_type, force)
if result.get("success") or not result.get("flagged"):
return result
logger.warning("%s flagged (%s), trying Agent", browser_type, result.get("flag_reason", "unknown"))
return await _scrape_with_agent(force)
# No profile at all → try Agent (fresh Chromium)
return await _scrape_with_agent(force)
# ── Firefox Scraper ──────────────────────────────────────────────────
# Launches a headless Firefox with the user's real profile cookies (copied
# to a temp dir). Uses firefox_user_prefs to disable automation flags and
# tracking protection. Injects anti-detection script via add_init_script().
# If force=False, randomly skips 30% of runs as a decoy (looks like random
# human browsing). If flagged by Facebook, returns flagged=True for the
# dispatcher to fall through to the Agent.
async def _scrape_with_firefox(profile_path: str, force: bool) -> dict:
"""Scrape Facebook using Firefox + persistent real profile (no cookie injection)."""
@@ -939,6 +1123,136 @@ async def _scrape_with_firefox(profile_path: str, force: bool) -> dict:
pass
# ── Chromium Scraper ─────────────────────────────────────────────────
# Generic scraper for Chrome/Edge/Opera. Uses the same structure as the
# Firefox scraper but with Chromium-specific launch config:
# - Chrome: channel="chrome"
# - Edge: channel="msedge"
# - Opera: executable_path from shutil.which("opera")
# Cookies come from copy_chrome_profile (temp copy of user's profile data).
# Anti-detection script differs slightly from Firefox variant.
# Same decoy-skip and flagged-fallback behavior as Firefox.
async def _scrape_with_chromium(profile_path: str, browser: str, force: bool = False) -> dict:
"""Scrape Facebook using a Chromium-based browser profile (Chrome/Edge/Opera)."""
if not profile_path:
return {"success": False, "leads": [], "flagged": False, "flag_reason": None, "error": "No profile path"}
channel = None
executable_path = None
if browser == "chrome":
channel = "chrome"
elif browser == "edge":
channel = "msedge"
elif browser == "opera":
executable_path = shutil.which("opera") or shutil.which("Opera")
profile_dir = None
try:
profile_dir = copy_chrome_profile(profile_path)
async with async_playwright() as pw:
launch_kwargs = dict(
user_data_dir=profile_dir,
headless=True,
args=CHROME_LAUNCH_ARGS,
)
if channel:
launch_kwargs["channel"] = channel
if executable_path:
launch_kwargs["executable_path"] = executable_path
context = await pw.chromium.launch_persistent_context(**launch_kwargs)
pages = context.pages
page = pages[0] if pages else await context.new_page()
await context.add_init_script(chromium_init_script())
try:
await page.goto('https://www.google.com/', wait_until='domcontentloaded', timeout=15000)
await page.wait_for_timeout(random.randint(1000, 3000))
except Exception:
logger.warning("Google navigation failed, trying Facebook directly")
await page.goto('https://www.facebook.com/', wait_until='domcontentloaded', timeout=30000)
await page.wait_for_timeout(random.randint(3000, 8000))
url = page.url
page_text = await page.evaluate('document.body.innerText') if '/login' in url.lower() else ''
det = check_detection_signals(url, page_text)
if det or '/login' in url.lower():
logger.warning("Facebook login page detected — flag: %s", det or "login_page")
await context.close()
return {"success": False, "leads": [], "flagged": True, "flag_reason": det or "login_page", "error": "Facebook login page detected"}
await human_scroll(page, steps=random.randint(2, 4), total_delay=random.uniform(8, 20))
if random.random() < 0.25:
await page.evaluate("window.scrollTo(0, 0)")
await page.wait_for_timeout(random.randint(2000, 5000))
await human_scroll(page, steps=random.randint(1, 2))
if not force and random.random() < 0.3:
await page.wait_for_timeout(random.randint(8000, 20000))
await context.close()
return {"success": True, "leads": [], "flagged": False, "flag_reason": None, "error": None}
all_posts = []
searches = random.sample(FB_SEARCHES, k=random.randint(2, 4))
for i, query in enumerate(searches):
page, posts = await search_facebook(page, context, query)
all_posts.extend(posts)
if not posts:
continue
if random.random() < 0.4:
await page.evaluate(f"window.scrollBy(0, {random.randint(-300, 300)})")
delay = random.uniform(8, 25)
await page.wait_for_timeout(int(delay * 1000))
if i == random.randint(0, 1) and random.random() < 0.15:
new_page = await context.new_page()
try:
await new_page.goto('https://www.facebook.com/groups/', wait_until='domcontentloaded', timeout=15000)
await new_page.wait_for_timeout(random.randint(3000, 8000))
except Exception:
pass
await new_page.close()
page = await _ensure_page(page, context)
if random.random() < 0.5:
await page.wait_for_timeout(random.randint(3000, 10000))
await context.close()
seen = set()
deduped = []
for p in all_posts:
key = p.get('content', '')[:100]
if key not in seen:
seen.add(key)
deduped.append(p)
deduped = [p for p in deduped if _is_within_days(p.get('date', ''), 3)]
leads = deduped[:20]
if leads:
leads = await classify_leads(leads)
return {"success": True, "leads": leads[:15], "flagged": False, "flag_reason": None, "error": None}
except Exception as e:
logger.error("%s scrape failed: %s", browser, e)
return {"success": False, "leads": [], "flagged": False, "flag_reason": None, "error": str(e)}
finally:
if profile_dir and os.path.exists(profile_dir):
try:
shutil.rmtree(profile_dir, ignore_errors=True)
except Exception:
pass
# ── Agent Fallback ──────────────────────────────────────────────────
# When all browser-based scrapers fail/are flagged, this fallback uses
# browser-use Agent with ChatOllama (local, free) to navigate Facebook
# autonomously via GPT-style prompting. No API keys needed.
# Uses Chromium headless with the same launch args as _scrape_with_chromium.
# The Agent is prompted to extract structured post data and return JSON.
async def _scrape_with_agent(force: bool = False) -> dict:
"""Fallback scraper — browser-use Agent + ChatOllama (free/local, Chromium)."""
cleanup_chrome()
@@ -1016,6 +1330,12 @@ When done, return the data as a JSON list with keys: content, author, url, date.
except Exception as e:
logger.warning("Failed to clean up Chrome profile %s: %s", profile_dir, e)
# ── AI Classification ────────────────────────────────────────────────
# Uses the local Ollama model to classify scraped posts as LEAD or NOT LEAD.
# Falls back to keyword-based filtering if the AI is unavailable.
# ask_ollama() sends a raw prompt to Ollama's /api/chat endpoint.
# classify_leads() builds a classification prompt and parses the JSON response.
async def ask_ollama(prompt: str) -> str:
import httpx
async with httpx.AsyncClient(timeout=120) as c:
@@ -1101,23 +1421,34 @@ Return a JSON array like ["yes","no","yes"] matching the order above."""
return filtered[:10]
return []
# ══════════════════════════════════════════════════════════════════════
# FastAPI Endpoints
# ══════════════════════════════════════════════════════════════════════
@app.get("/health")
async def health():
"""Simple health check for the splash page status polling."""
return {"status": "ok"}
def _detect_all_profiles() -> dict:
"""Return all detected browser profiles."""
return {
"firefox": FX_PROFILE or find_firefox_profile() or "",
"opera": OPERA_PROFILE or find_opera_profile() or "",
"chrome": CHROME_PROFILE or find_chrome_profile() or "",
"edge": EDGE_PROFILE or find_edge_profile() or "",
}
@app.get("/setup/profile")
async def setup_profile():
"""Auto-detect Firefox profile path."""
path = FX_PROFILE or find_firefox_profile()
return {"path": path, "detected": bool(path)}
"""Auto-detect all browser profiles."""
return _detect_all_profiles()
@app.post("/setup/check-login")
async def setup_check_login(body: dict):
"""Check if Facebook is logged in using the given profile."""
profile_path = (body.get("profile_path") or "").strip() or FX_PROFILE or find_firefox_profile()
if not profile_path:
return {"logged_in": False, "reason": "no_profile", "error": "No Firefox profile found"}
async def _check_browser_login(profile_path: str, browser: str) -> dict:
"""Check Facebook login for a specific browser type."""
profile_dir = None
try:
if browser == "firefox":
profile_dir = copy_firefox_profile(profile_path)
async with async_playwright() as pw:
context = await pw.firefox.launch_persistent_context(
@@ -1130,10 +1461,57 @@ async def setup_check_login(body: dict):
logged_in = "/login" not in page.url.lower()
await context.close()
shutil.rmtree(profile_dir, ignore_errors=True)
return {"logged_in": logged_in, "reason": None if logged_in else "login_page"}
return {"logged_in": logged_in, "browser": browser, "path": profile_path}
else:
channel = None
exe = None
if browser == "chrome":
channel = "chrome"
elif browser == "edge":
channel = "msedge"
elif browser == "opera":
exe = shutil.which("opera") or shutil.which("Opera")
profile_dir = copy_chrome_profile(profile_path)
async with async_playwright() as pw:
kw = dict(user_data_dir=profile_dir, headless=True, args=CHROME_LAUNCH_ARGS)
if channel:
kw["channel"] = channel
if exe:
kw["executable_path"] = exe
context = await pw.chromium.launch_persistent_context(**kw)
page = context.pages[0] if context.pages else await context.new_page()
await page.goto("https://www.facebook.com/", wait_until="domcontentloaded", timeout=30000)
await page.wait_for_timeout(3000)
logged_in = "/login" not in page.url.lower()
await context.close()
shutil.rmtree(profile_dir, ignore_errors=True)
return {"logged_in": logged_in, "browser": browser, "path": profile_path}
except Exception as e:
logger.error("Setup check-login failed: %s", e)
return {"logged_in": False, "reason": "error", "error": str(e)}
if profile_dir:
try: shutil.rmtree(profile_dir, ignore_errors=True)
except: pass
return {"logged_in": False, "browser": browser, "path": profile_path, "error": str(e)}
@app.post("/setup/check-login")
async def setup_check_login(body: dict):
"""Check if Facebook is logged in. Accepts optional 'browser' param, tries all if not given."""
browser = (body.get("browser") or "").strip().lower()
profile_path = (body.get("profile_path") or "").strip()
if browser and profile_path:
return await _check_browser_login(profile_path, browser)
# No specific browser — try all detected
profiles = _detect_all_profiles()
priority = ["firefox", "opera", "chrome", "edge"]
for b in priority:
p = profiles.get(b, "")
if p:
result = await _check_browser_login(p, b)
if result.get("logged_in"):
return result
logger.info("%s not logged in, trying next", b)
return {"logged_in": False, "reason": "none_logged_in", "message": "No browser with Facebook login found"}
@app.post("/agent/run")
async def agent_run(task: str = Body(..., embed=True)):
+13
View File
@@ -1,5 +1,18 @@
# ── Python Dependencies for the Facebook Scraper (FastAPI) ───────
# Install via: pip install -r requirements.txt
# Web framework for the REST API (health, setup, scrape endpoints)
fastapi>=0.115.0
# ASGI server for running the FastAPI app
uvicorn>=0.34.0
# Browser automation (launches Firefox, Chrome, Edge, Opera via Playwright)
playwright>=1.49.0
# AI-powered browser agent (fallback when direct browser scraping is flagged)
# Uses ChatOllama locally — no API keys needed
browser-use>=0.1.0
# LangChain integration for ChatOllama (provides the LLM for browser-use Agent)
langchain-ollama>=0.2.0
+9
View File
@@ -1,3 +1,9 @@
// ── Ollama Launcher ────────────────────────────────────────────────
// Checks if Ollama is already running. If not, finds the binary and
// starts it as a detached background process.
// Cross-platform: uses tasklist (Windows) or pgrep (Linux/Mac) to
// detect running process; uses where/which to find the binary.
import { execSync, spawn } from "node:child_process"
import { platform } from "node:os"
@@ -16,6 +22,7 @@ function isRunning() {
function findOllama() {
if (platform() === "win32") {
// Try PATH first, then common install locations
try {
return execSync("where ollama", { encoding: "utf8", timeout: 3000 }).trim().split("\n")[0]
} catch {}
@@ -36,11 +43,13 @@ if (!isRunning()) {
process.exit(1)
}
console.log("Starting Ollama...")
// Spawn detached so it outlives this script and the npm process
if (platform() === "win32") {
spawn(bin, ["serve"], { stdio: "ignore", detached: true, windowsHide: true }).unref()
} else {
spawn(bin, ["serve"], { stdio: "ignore", detached: true }).unref()
}
// Give it a moment to start listening
execSync("sleep 3", { stdio: "ignore", timeout: 5000 })
} else {
console.log("Ollama already running")
+6
View File
@@ -1,3 +1,9 @@
// ── Browser Opener ─────────────────────────────────────────────────
// Opens the splash page in the user's default browser after an 8-second
// delay. The delay gives all services time to start before the user
// sees the loading screen.
// Cross-platform: uses start (Windows), open (Mac), or xdg-open (Linux).
import { execSync } from "node:child_process"
import { platform } from "node:os"
+8
View File
@@ -1,9 +1,16 @@
// ── Port Precheck ──────────────────────────────────────────────────
// Kills any existing processes on ports 3001, 3006, 3007, 3008.
// These are the AI server, Next.js frontend, Signaling server, and
// Python scraper respectively.
// Runs before anything else starts to avoid EADDRINUSE errors.
import { execSync } from "node:child_process"
import { platform } from "node:os"
const PORTS = [3001, 3006, 3007, 3008]
if (platform() === "win32") {
// Windows: use netstat + findstr to find listening PIDs, then taskkill
for (const port of PORTS) {
try {
const out = execSync(`netstat -ano | findstr "LISTENING" | findstr ":${port} "`, { encoding: "utf8", timeout: 5000 })
@@ -18,6 +25,7 @@ if (platform() === "win32") {
} catch {}
}
} else {
// Linux/Mac: use lsof -ti to find PIDs, then kill -9
for (const port of PORTS) {
try {
const pid = execSync(`lsof -ti:${port} 2>/dev/null`, { encoding: "utf8", timeout: 5000 }).trim()
+7
View File
@@ -1,3 +1,9 @@
// ── Python Runner ──────────────────────────────────────────────────
// Detects the system's Python executable (python vs python3) and runs
// a given script with arguments. Used by the dev:browser-use npm script.
// Avoids shell:true — spawns Python directly with its full path.
// Cross-platform: uses where (Windows) or which (Linux/Mac).
import { execSync, spawn } from "node:child_process"
import { platform } from "node:os"
@@ -23,5 +29,6 @@ if (!script) {
process.exit(1)
}
// Spawn Python with inherited stdio so the script's output is visible
const proc = spawn(PYTHON, [script, ...args], { stdio: "inherit" })
proc.on("exit", (code) => process.exit(code ?? 1))
+17 -4
View File
@@ -1,9 +1,21 @@
// ── One-command Setup ──────────────────────────────────────────────
// Run via: npm run setup
// Does the following in order:
// 1. npm install (Node.js dependencies)
// 2. pip install -r requirements.txt (Python dependencies)
// 3. playwright install firefox chromium (Playwright browsers)
// 4. Copies .env.example to .env.local if not exists
//
// All steps are cross-platform (Windows, Mac, Linux).
// Uses execSync for simplicity since each step blocks the next.
import { execSync } from "node:child_process"
import { existsSync, copyFileSync } from "node:fs"
import { platform } from "node:os"
const SEP = platform() === "win32" ? "&" : ";"
// Auto-detect Python executable (python vs python3)
function detectPython() {
const candidates = platform() === "win32" ? ["python", "python3"] : ["python3", "python"]
for (const cmd of candidates) {
@@ -16,6 +28,7 @@ function detectPython() {
process.exit(1)
}
// Auto-detect pip (pip vs pip3), fall back to python -m pip
function detectPip(python) {
const candidates = platform() === "win32" ? ["pip", "pip3"] : ["pip3", "pip"]
for (const cmd of candidates) {
@@ -45,13 +58,13 @@ console.log("=== CoastIT CRM Setup ===\n")
// 1. Node dependencies
run("npm install", "Installing Node.js dependencies")
// 2. Python dependencies
// 2. Python dependencies (run from browser-use-service directory)
run(`cd browser-use-service ${SEP} ${PIP} install -r requirements.txt`, "Installing Python dependencies")
// 3. Playwright browsers
// 3. Playwright browsers (Firefox for primary scraping, Chromium for Chrome/Edge/Opera + Agent fallback)
run(`${PY} -m playwright install firefox chromium`, "Installing Playwright browsers")
// 4. .env file
// 4. .env file — create from template if it doesn't exist
if (!existsSync(".env.local")) {
console.log("\n── Creating .env.local ──")
copyFileSync(".env.example", ".env.local")
@@ -60,7 +73,7 @@ if (!existsSync(".env.local")) {
console.log("\n── .env.local already exists, skipping ──")
}
// 5. Ollama model
// 5. Remaining manual steps
console.log("\n── Next steps ──")
console.log(" 1. Make sure PostgreSQL is running with database 'crm'")
console.log(" 2. Pull the Ollama model: ollama pull dolphin-llama3:8b")
+138 -138
View File
@@ -5,6 +5,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Loading CoastIT CRM</title>
<style>
/* ── Global Reset ──────────────────────────────────────────────── */
* { margin: 0; padding: 0; box-sizing: border-box; }
body {
background: #0a0a1a;
@@ -18,7 +19,8 @@
overflow: hidden;
}
/* Robot */
/* ── Robot Animation ────────────────────────────────────────────── */
/* Animated robot mascot that bobs, swings arms, and blinks while loading */
.robot {
position: relative;
width: 120px;
@@ -128,7 +130,9 @@
100% { transform: translateY(-4px) rotate(8deg); }
}
/* Status indicators */
/* ── Status Indicators ────────────────────────────────────────── */
/* Shows AI Server, Scraper, and Frontend status dots side by side.
Each transitions from dim → cyan (ready) or red (failed). */
.services {
display: flex;
flex-direction: column;
@@ -207,7 +211,7 @@
color: #ef4444;
}
/* Loading text */
/* ── Loading Text ──────────────────────────────────────────────── */
.loading-text {
font-size: 16px;
font-weight: 600;
@@ -222,7 +226,9 @@
50% { opacity: 1; }
}
/* Launch gear (inline, below everything) */
/* ── Launch Gear ────────────────────────────────────────────────── */
/* Appears below the loading text after all services are ready.
Shows a spinning gear + "Launching gear!" text, then redirects to /login. */
.launch-overlay {
display: none;
flex-direction: column;
@@ -255,7 +261,9 @@
100% { transform: scale(1); opacity: 1; }
}
/* Error state */
/* ── Error State ────────────────────────────────────────────────── */
/* When a service fails to start within the timeout (60s), show
red text, an error message, and a retry button. */
.loading-text.error {
background: linear-gradient(135deg, #ef4444, #f97316);
-webkit-background-clip: text;
@@ -294,7 +302,12 @@
display: inline-block;
}
/* ── Setup Wizard ──────────────────────────────────────────── */
/* ── Setup Wizard ──────────────────────────────────────────────── */
/* Full-screen 4-step wizard for first-time setup:
1. Environment check (Ollama, model, browser, Facebook login)
2. Browser auto-detection & selection
3. Ollama model pull with progress bar
4. Done — close wizard, start normal loading flow */
.setup-wizard {
display: none;
position: fixed;
@@ -416,7 +429,7 @@
</head>
<body>
<!-- Robot -->
<!-- ── Robot Mascot ────────────────────────────────────────────── -->
<div class="robot">
<div class="robot-antenna"></div>
<div class="robot-head">
@@ -431,7 +444,7 @@
</div>
</div>
<!-- Services -->
<!-- ── Service Status Indicators ───────────────────────────────── -->
<div class="services">
<div class="service" id="svc-ai">
<span class="service-name">AI Server</span>
@@ -464,26 +477,27 @@
<div class="loading-text" id="loading-text">Loading...</div>
<!-- Launch gear (inline, below everything) -->
<!-- ── Launch Gear ──────────────────────────────────────────── -->
<div class="launch-overlay" id="launch-overlay">
<div class="launch-gear"></div>
<div class="launch-text">🚀 Launching gear!</div>
<div class="launch-text">Launching gear!</div>
</div>
<div class="error-msg" id="error-msg"></div>
<button class="retry-btn" id="retry-btn" onclick="location.reload()">Try Again</button>
<!-- ── Setup Wizard ── -->
<!-- ══════════════════════════════════════════════════════════════
Setup Wizard (4 steps, shown only on first run)
══════════════════════════════════════════════════════════════ -->
<div class="setup-wizard" id="setup-wizard">
<div class="setup-steps">
<div class="setup-dot current" id="sdot-1"></div>
<div class="setup-dot" id="sdot-2"></div>
<div class="setup-dot" id="sdot-3"></div>
<div class="setup-dot" id="sdot-4"></div>
<div class="setup-dot" id="sdot-5"></div>
</div>
<!-- Step 1: Welcome -->
<!-- Step 1: Environment check -->
<div class="setup-step active" id="sstep-1">
<div class="robot" style="margin-bottom:20px">
<div class="robot-antenna"></div>
@@ -504,43 +518,22 @@
<button class="setup-btn" id="welcome-next" onclick="goStep(2)">Next →</button>
</div>
<!-- Step 2: Browser Profile -->
<!-- Step 2: Browser auto-detection & selection -->
<div class="setup-step" id="sstep-2">
<div class="setup-title">Browser Profile</div>
<div class="setup-subtitle">We need a Firefox or Chrome profile with Facebook logged in</div>
<div class="setup-card">
<div class="check-row">
<span class="check-icon" id="prof-auto-icon"></span>
<span class="check-label">Auto-detected</span>
<span class="check-value" id="prof-auto-val">scanning...</span>
</div>
<div style="margin-top:14px;font-size:13px;opacity:0.5;margin-bottom:8px">Or enter the path manually:</div>
<input class="setup-input" id="prof-input" placeholder="C:\Users\You\AppData\Roaming\Mozilla\Firefox\Profiles\..." oninput="onProfileInput()">
<div style="margin-top:6px;font-size:12px;color:#22d3ee" id="prof-feedback"></div>
<div class="setup-title">Detect your browser</div>
<div class="setup-subtitle">We found these browsers with Facebook login. Click one to select it.</div>
<div class="setup-card" id="browser-list">
<div id="browser-scanning" style="text-align:center;padding:16px;opacity:0.6">🔍 Scanning for browsers...</div>
<div id="browser-results" style="display:none"></div>
</div>
<div class="setup-btns">
<button class="setup-btn outline" onclick="goStep(1)">← Back</button>
<button class="setup-btn" id="prof-next" onclick="saveProfile()" disabled>Next</button>
<button class="setup-btn" id="browser-confirm" onclick="confirmBrowser()" disabled>Confirm</button>
</div>
</div>
<!-- Step 3: Facebook Login -->
<!-- Step 3: Ollama Model Pull -->
<div class="setup-step" id="sstep-3">
<div class="setup-title">Facebook Login</div>
<div class="setup-subtitle">Make sure you're logged into Facebook in your browser</div>
<div class="setup-card" style="text-align:center">
<div style="font-size:48px;margin-bottom:12px" id="fb-icon">🔍</div>
<div style="font-size:14px;opacity:0.7" id="fb-status">Click "Check Login" to verify</div>
<div class="progress-bar" id="fb-progress" style="display:none;margin-top:16px"><div class="progress-fill" id="fb-progress-fill"></div></div>
</div>
<div class="setup-btns">
<button class="setup-btn outline" onclick="goStep(2)">← Back</button>
<button class="setup-btn" id="fb-check-btn" onclick="checkFacebookLogin()">Check Login</button>
</div>
</div>
<!-- Step 4: Ollama Model -->
<div class="setup-step" id="sstep-4">
<div class="setup-title">AI Model</div>
<div class="setup-subtitle">We need to pull the AI model for the scraper to work</div>
<div class="setup-card" style="text-align:center">
@@ -554,8 +547,8 @@
</div>
</div>
<!-- Step 5: Done -->
<div class="setup-step" id="sstep-5">
<!-- Step 4: Done → Launch -->
<div class="setup-step" id="sstep-4">
<div style="font-size:64px;margin-bottom:16px">🎉</div>
<div class="setup-title">All set!</div>
<div class="setup-subtitle">Your environment is configured. We'll now start all services.</div>
@@ -567,13 +560,18 @@
</div>
<script>
// ══════════════════════════════════════════════════════════════════
// Client-side JavaScript
// ══════════════════════════════════════════════════════════════════
// ── State ──
let setupData = null;
let currentStep = 1;
const TOTAL_STEPS = 5;
const TOTAL_STEPS = 4;
let selectedBrowser = "";
// ── Normal loading state ──
const MAX_ATTEMPTS = 30;
const MAX_ATTEMPTS = 30; // 30 attempts × 2s = 60s timeout
let attempts = 0;
const CHECKS = [
{ id: 'ai', key: 'ai', ready: false, failed: false },
@@ -582,6 +580,7 @@ const CHECKS = [
];
const MSGS = { ai: 'AI Ready', scraper: 'Scraper Ready', frontend: 'Frontend Ready' };
const NAMES = { ai: 'AI Server', scraper: 'Scraper', frontend: 'Frontend' };
const BROWSE_ICONS = { firefox: '🦊', opera: '🔵', chrome: '🌐', edge: '🔷' };
// ── Step navigation ──
function goStep(n) {
@@ -593,21 +592,32 @@ function goStep(n) {
el.classList.toggle('current', i + 1 === n);
el.classList.toggle('done', i + 1 < n);
});
if (n === 2) renderBrowserCards();
}
// ── Step 1: Welcome / Env check ──
// ── Step 1: Welcome + Environment check ──
// Fetches /setup/status from the AI server, displays checkmarks/crosses
// for each environment requirement. If first_run is false, skips the
// wizard entirely and goes straight to the normal loading flow.
async function checkEnv() {
try {
const res = await fetch('/setup/status');
setupData = await res.json();
} catch {
setupData = { first_run: true, ollama_running: false, model_available: false, profile_detected: false, facebook_logged_in: false };
setupData = { first_run: true, ollama_running: false, model_available: false, facebook_logged_in: false, browsers: {} };
}
setEnvStatus('ollama', setupData.ollama_running, setupData.ollama_running ? 'Running' : 'Not running');
setEnvStatus('model', setupData.model_available, setupData.model_available ? `${setupData.model_name} ✓` : 'Not pulled');
setEnvStatus('profile', setupData.profile_detected, setupData.profile_path || 'Not found');
setEnvStatus('fb', setupData.facebook_logged_in, setupData.facebook_logged_in ? 'Logged in' : 'Not checked');
// Browser summary
const detected = Object.entries(setupData.browsers || {}).filter(([, v]) => v.path)
const loggedIn = detected.filter(([, v]) => v.logged_in)
const browserSummary = loggedIn.length
? loggedIn.map(([b]) => `${b.charAt(0).toUpperCase()+b.slice(1)} ✓`).join(', ')
: detected.length ? 'Found but not logged into Facebook' : 'None detected'
setEnvStatus('profile', detected.length > 0, browserSummary)
setEnvStatus('fb', setupData.facebook_logged_in, setupData.facebook_logged_in ? 'Logged in' : loggedIn.length ? '' : 'Not checked')
if (!setupData.first_run) {
document.getElementById('setup-wizard').classList.remove('active');
@@ -615,8 +625,6 @@ async function checkEnv() {
return;
}
document.getElementById('setup-wizard').classList.add('active');
// Pre-fill step 2 fields
updateProfileUI();
}
function setEnvStatus(id, ok, label) {
@@ -624,94 +632,85 @@ function setEnvStatus(id, ok, label) {
document.getElementById('env-' + id + '-val').textContent = label;
}
// ── Step 2: Browser Profile ──
function updateProfileUI() {
const icon = document.getElementById('prof-auto-icon');
const val = document.getElementById('prof-auto-val');
const input = document.getElementById('prof-input');
if (setupData.profile_path) {
icon.textContent = '✅';
val.textContent = setupData.profile_path;
input.value = setupData.profile_path;
document.getElementById('prof-next').disabled = false;
} else {
icon.textContent = '❌';
val.textContent = 'None detected';
// ── Step 2: Browser selection ──
// Renders clickable cards for each detected browser.
// Auto-selects the first one that has Facebook login.
// No manual path input needed — all detection is automatic.
function renderBrowserCards() {
const scanning = document.getElementById('browser-scanning');
const results = document.getElementById('browser-results');
const browsers = setupData.browsers || {};
const detected = Object.entries(browsers).filter(([, v]) => v.path)
if (detected.length === 0) {
scanning.textContent = '❌ No browsers with Facebook detected. Log in on your browser and click Retry.';
const retryBtn = document.createElement('button');
retryBtn.className = 'setup-btn';
retryBtn.textContent = 'Retry Detection';
retryBtn.onclick = () => location.reload();
scanning.after(retryBtn);
return;
}
scanning.style.display = 'none';
results.style.display = 'block';
// Auto-select first logged-in browser, or first detected
const firstLoggedIn = detected.find(([, v]) => v.logged_in)
selectedBrowser = firstLoggedIn ? firstLoggedIn[0] : detected[0][0]
results.innerHTML = detected.map(([name, info]) => {
const isSel = name === selectedBrowser
const icon = BROWSE_ICONS[name] || '🌐'
const status = info.logged_in ? '✅ Logged in' : info.path ? '⚠️ Not logged in' : '❌ Not found'
const pathShort = info.path ? info.path.length > 45 ? '...' + info.path.slice(-42) : info.path : ''
return `<div class="browser-card ${isSel ? 'selected' : ''}" data-browser="${name}" onclick="pickBrowser('${name}')" style="display:flex;align-items:center;gap:12px;padding:12px;border:2px solid ${isSel ? '#6366f1' : 'rgba(255,255,255,0.08)'};border-radius:10px;margin-bottom:8px;cursor:pointer;transition:all 0.2s">
<span style="font-size:24px">${icon}</span>
<div style="flex:1"><strong style="text-transform:capitalize">${name}</strong><div style="font-size:12px;opacity:0.5;overflow:hidden;text-overflow:ellipsis;white-space:nowrap">${pathShort}</div></div>
<span style="font-size:13px">${status}</span>
</div>`
}).join('')
document.getElementById('browser-confirm').disabled = false
}
function onProfileInput() {
const val = document.getElementById('prof-input').value.trim();
const fb = document.getElementById('prof-feedback');
const btn = document.getElementById('prof-next');
if (!val) { fb.textContent = ''; btn.disabled = true; return }
fb.textContent = 'Entered: ' + val;
btn.disabled = false;
function pickBrowser(name) {
selectedBrowser = name
document.querySelectorAll('.browser-card').forEach(el => {
el.style.borderColor = el.dataset.browser === name ? '#6366f1' : 'rgba(255,255,255,0.08)'
})
}
async function saveProfile() {
const path = document.getElementById('prof-input').value.trim();
if (!path) return;
const btn = document.getElementById('prof-next');
btn.textContent = 'Saving...';
btn.disabled = true;
// Saves the selected browser to .env.local via the AI server's /setup/profile endpoint
async function confirmBrowser() {
if (!selectedBrowser) return
const btn = document.getElementById('browser-confirm')
btn.textContent = 'Saving...'
btn.disabled = true
const path = setupData.browsers[selectedBrowser].path
try {
const res = await fetch('/setup/profile', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ path }) });
const data = await res.json();
const res = await fetch('/setup/profile', {
method: 'POST', headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ browser: selectedBrowser, path })
})
const data = await res.json()
if (data.success) {
setupData.profile_detected = true;
setupData.profile_path = path;
goStep(3);
setupData.selected_browser = selectedBrowser
goStep(3)
} else {
document.getElementById('prof-feedback').textContent = 'Error: ' + (data.error || 'failed');
}
} catch { document.getElementById('prof-feedback').textContent = 'Error: could not save'; }
btn.textContent = 'Next →';
btn.disabled = false;
}
// ── Step 3: Facebook Login ──
async function checkFacebookLogin() {
const btn = document.getElementById('fb-check-btn');
const status = document.getElementById('fb-status');
const icon = document.getElementById('fb-icon');
const prog = document.getElementById('fb-progress');
const fill = document.getElementById('fb-progress-fill');
btn.disabled = true;
btn.textContent = 'Checking...';
prog.style.display = 'block';
fill.style.width = '50%';
status.textContent = 'Opening Facebook...';
try {
const path = setupData.profile_path || '';
const res = await fetch('/setup/check-login', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ profile_path: path }) });
const data = await res.json();
if (data.logged_in) {
fill.style.width = '100%';
icon.textContent = '✅';
status.textContent = 'You are logged into Facebook!';
status.className = 'status-ok';
setupData.facebook_logged_in = true;
setTimeout(() => goStep(4), 1000);
} else {
fill.style.width = '100%';
icon.textContent = '❌';
status.textContent = 'Not logged in. Log into Facebook in your browser and try again.';
status.className = 'status-err';
btn.textContent = 'Retry';
btn.disabled = false;
alert('Failed to save: ' + (data.error || 'unknown'))
btn.disabled = false
}
} catch {
icon.textContent = '❌';
status.textContent = 'Could not reach the scraper service';
status.className = 'status-err';
btn.textContent = 'Retry';
btn.disabled = false;
alert('Could not save profile')
btn.disabled = false
}
btn.textContent = 'Confirm →'
}
// ── Step 4: Ollama Model ──
// ── Step 3: Ollama Model Pull ──
// Kicks off an "ollama pull" via the AI server and polls for progress.
let pullPolling = false;
async function pullModel() {
if (pullPolling) return;
@@ -720,7 +719,6 @@ async function pullModel() {
const status = document.getElementById('model-status');
btn.disabled = true;
btn.textContent = 'Starting...';
try {
const res = await fetch('/setup/ollama/pull', { method: 'POST' });
const data = await res.json();
@@ -745,12 +743,12 @@ async function pollPullProgress() {
const data = await res.json();
fill.style.width = data.progress + '%';
if (data.status === 'done') {
status.textContent = '✅ Model downloaded successfully!';
status.textContent = '✅ Model downloaded!';
status.className = 'status-ok';
btn.textContent = 'Done';
setupData.model_available = true;
pullPolling = false;
setTimeout(() => goStep(5), 1500);
setTimeout(() => goStep(4), 1500);
return;
}
if (data.status === 'failed') {
@@ -769,25 +767,26 @@ async function pollPullProgress() {
}
}
// ── Step 5: Finish ──
// ── Step 4: Finish Setup ──
// Closes the wizard and starts the normal loading flow.
function finishSetup() {
document.getElementById('setup-wizard').classList.remove('active');
startNormalFlow();
}
// ── Normal loading flow ──
function startNormalFlow() {
poll();
}
// ── Normal Loading Flow ──
// Polls /status every 2 seconds for up to 60 seconds.
// Tracks each service's state (ready/waiting/failed).
// All ready → show launch gear, redirect to /login after 5s.
// Any failed → show error message + retry button.
function startNormalFlow() { poll() }
async function checkAllServices() {
try {
const res = await fetch('/status', { cache: 'no-store' });
const data = await res.json();
for (const svc of CHECKS) svc.ready = data[svc.key] === true;
} catch {
for (const svc of CHECKS) svc.ready = false;
}
} catch { for (const svc of CHECKS) svc.ready = false }
}
function updateUI() {
@@ -829,6 +828,7 @@ async function poll() {
}
// ── Boot ──
// Entry point: check environment → shows wizard or starts normal flow
checkEnv();
</script>
</body>