Compare commits
2 Commits
ed2e1fc64d
...
1f448320be
| Author | SHA1 | Date | |
|---|---|---|---|
| 1f448320be | |||
| 9ce9506e8e |
+126
-56
@@ -1,3 +1,13 @@
|
||||
// ── CRM AI Server ──────────────────────────────────────────────────
|
||||
// Provides:
|
||||
// - Chat API (POST /ai/chat) — routes user messages to Ollama for sales coaching
|
||||
// - Setup wizard endpoints (GET /setup/status, POST /setup/profile, etc.)
|
||||
// - Combined /status endpoint for splash page health polling
|
||||
// - Configuration routes (GET/POST /ai/instructions, GET /ai/jobs)
|
||||
// - Model pull support (POST /setup/ollama/pull)
|
||||
//
|
||||
// This is a zero-dependency Node.js HTTP server (no Express needed).
|
||||
|
||||
import http from "node:http"
|
||||
import fs from "node:fs"
|
||||
import path from "node:path"
|
||||
@@ -8,6 +18,9 @@ const __dirname = path.dirname(fileURLToPath(import.meta.url))
|
||||
const ROOT = path.resolve(__dirname, "..")
|
||||
|
||||
// ── Load .env.local ──────────────────────────────────────────────
|
||||
// Reads key=value pairs and sets them as process.env so they're
|
||||
// available throughout the server. Ignores comments and blank lines.
|
||||
// Values with matching quotes are unquoted.
|
||||
try {
|
||||
const envPath = path.join(ROOT, ".env.local")
|
||||
const envContent = fs.readFileSync(envPath, "utf-8")
|
||||
@@ -23,7 +36,7 @@ try {
|
||||
}
|
||||
console.log("Loaded .env.local")
|
||||
} catch {
|
||||
// .env.local may not exist, ignore
|
||||
// .env.local may not exist (first run), which is fine
|
||||
}
|
||||
|
||||
// ── Config from env ─────────────────────────────────────────────
|
||||
@@ -36,10 +49,14 @@ const JOBS_PATH = process.env.JOBS_PATH || path.join(ROOT, "data", "ai", "jobs.j
|
||||
const AI_MD_PATH = process.env.AI_MD_PATH || path.join(ROOT, "data", "ai", "ai.md")
|
||||
|
||||
// ── Setup state ──────────────────────────────────────────────────
|
||||
// Tracks the Ollama model pull process so the setup wizard can
|
||||
// poll for download progress.
|
||||
let pullProcess = null
|
||||
let pullProgress = { status: "idle", progress: 0, message: "" }
|
||||
|
||||
// ── Job loading ─────────────────────────────────────────────────
|
||||
// Loads job categories from a JSONL file (one JSON object per line).
|
||||
// Used as context for the AI sales coach chat responses.
|
||||
function loadJobs() {
|
||||
try {
|
||||
const content = fs.readFileSync(JOBS_PATH, "utf-8")
|
||||
@@ -64,6 +81,8 @@ function loadJobs() {
|
||||
}
|
||||
|
||||
// ── ai.md management ────────────────────────────────────────────
|
||||
// ai.md is a Markdown file containing system instructions for the AI.
|
||||
// It can be read, written, or appended to via the API.
|
||||
function readInstructions() {
|
||||
try {
|
||||
return fs.readFileSync(AI_MD_PATH, "utf-8")
|
||||
@@ -78,6 +97,7 @@ function writeInstructions(content) {
|
||||
}
|
||||
|
||||
function appendToImprovementLog(entry) {
|
||||
// Adds a timestamped entry to the ## Improvement Log section of ai.md
|
||||
const current = readInstructions()
|
||||
const timestamp = new Date().toISOString().replace("T", " ").substring(0, 16)
|
||||
const logEntry = `\n- ${timestamp} — ${entry}`
|
||||
@@ -101,10 +121,13 @@ function appendToImprovementLog(entry) {
|
||||
}
|
||||
|
||||
// ── Chat handler ────────────────────────────────────────────────
|
||||
// scrapeFacebook() calls the scraper service (port 3008) to get leads.
|
||||
// handleChat() processes user messages — triggers lead scraping when
|
||||
// the user asks for "leads" or "listings", otherwise routes to Ollama
|
||||
// for AI-powered sales coaching.
|
||||
async function scrapeFacebook() {
|
||||
const profilePath = process.env.FX_PROFILE || ""
|
||||
const urlPath = `/scrape/facebook?force=true${profilePath ? `&profile_path=${encodeURIComponent(profilePath)}` : ""}`
|
||||
const logPath = "C:\\Users\\USER-PC\\AppData\\Local\\Temp\\opencode\\ai-scrape-debug.log"
|
||||
try {
|
||||
const body = await new Promise((resolve, reject) => {
|
||||
const req = http.request({ hostname: "127.0.0.1", port: 3008, path: urlPath, method: "POST", timeout: 360000 }, (res) => {
|
||||
@@ -139,6 +162,7 @@ function formatLeads(leads) {
|
||||
}
|
||||
|
||||
async function handleChat(userMessage, userId, userRole) {
|
||||
// If the user asks for leads, trigger the scraper
|
||||
const lowerMsg = userMessage.toLowerCase()
|
||||
const triggerWords = ["lists", "listings", "leads", "recent leads", "pull leads", "show me leads", "show listings"]
|
||||
|
||||
@@ -150,6 +174,7 @@ async function handleChat(userMessage, userId, userRole) {
|
||||
return "Scraper returned no results or encountered an error. Try again later."
|
||||
}
|
||||
|
||||
// Otherwise, build a system prompt with job context and send to Ollama
|
||||
const jobs = loadedJobs
|
||||
const instructions = readInstructions()
|
||||
|
||||
@@ -189,7 +214,7 @@ Provide concise, actionable sales advice. When asked about a specific job catego
|
||||
const data = await ollamaRes.json()
|
||||
const responseText = data.message?.content || ""
|
||||
|
||||
// Try to persist to DB (best-effort)
|
||||
// Persist conversation to PostgreSQL (best-effort — table may not exist yet)
|
||||
try {
|
||||
if (pgPool && userId) {
|
||||
await pgPool.query(
|
||||
@@ -205,6 +230,8 @@ Provide concise, actionable sales advice. When asked about a specific job catego
|
||||
}
|
||||
|
||||
// ── PG pool (lazy init) ────────────────────────────────────────
|
||||
// PostgreSQL connection pool for storing conversation history.
|
||||
// Lazy-initialized so the server starts even without a DB.
|
||||
let pgPool = null
|
||||
async function initPg() {
|
||||
if (!DATABASE_URL) return
|
||||
@@ -246,8 +273,9 @@ function parseURL(req) {
|
||||
return { pathname: url.pathname, searchParams: url.searchParams }
|
||||
}
|
||||
|
||||
// ── HTTP Server ─────────────────────────────────────────────────
|
||||
const server = http.createServer(async (req, res) => {
|
||||
// CORS
|
||||
// CORS headers — allow the Next.js frontend (port 3006) to call us
|
||||
res.setHeader("Access-Control-Allow-Origin", "*")
|
||||
res.setHeader("Access-Control-Allow-Methods", "GET, POST, OPTIONS")
|
||||
res.setHeader("Access-Control-Allow-Headers", "Content-Type")
|
||||
@@ -280,10 +308,12 @@ const server = http.createServer(async (req, res) => {
|
||||
}
|
||||
|
||||
// GET /status — combined health of all services
|
||||
// Used by the splash page to check if AI, Scraper, and Frontend are ready.
|
||||
// Polls each service internally to avoid cross-origin CORS issues.
|
||||
if (req.method === "GET" && pathname === "/status") {
|
||||
const { default: http } = await import("http")
|
||||
const results = { ai: true }
|
||||
// Check scraper
|
||||
// Check scraper (port 3008)
|
||||
try {
|
||||
await new Promise((resolve, reject) => {
|
||||
const r = http.get("http://127.0.0.1:3008/health", { timeout: 3000 }, (res) => { res.resume(); resolve() })
|
||||
@@ -291,7 +321,7 @@ const server = http.createServer(async (req, res) => {
|
||||
})
|
||||
results.scraper = true
|
||||
} catch { results.scraper = false }
|
||||
// Check frontend
|
||||
// Check frontend (port 3006)
|
||||
try {
|
||||
await new Promise((resolve, reject) => {
|
||||
const r = http.get("http://127.0.0.1:3006", { timeout: 3000 }, (res) => { res.resume(); resolve() })
|
||||
@@ -305,6 +335,11 @@ const server = http.createServer(async (req, res) => {
|
||||
// ── Setup endpoints ─────────────────────────────────────────
|
||||
|
||||
// GET /setup/status — check environment
|
||||
// Called by the splash page on boot. Returns info about:
|
||||
// - Ollama availability
|
||||
// - Model presence
|
||||
// - Detected browsers with login status
|
||||
// - Whether this is a first run (wizard needed)
|
||||
if (req.method === "GET" && pathname === "/setup/status") {
|
||||
const envExists = fs.existsSync(path.join(ROOT, ".env.local"))
|
||||
|
||||
@@ -327,33 +362,41 @@ const server = http.createServer(async (req, res) => {
|
||||
} catch {}
|
||||
}
|
||||
|
||||
// Profile auto-detect
|
||||
let profilePath = process.env.FX_PROFILE || ""
|
||||
let profileDetected = !!profilePath
|
||||
if (!profileDetected) {
|
||||
try {
|
||||
const r = await fetch("http://127.0.0.1:3008/health", { signal: AbortSignal.timeout(2000) })
|
||||
if (r.ok) {
|
||||
const diag = await (await fetch("http://127.0.0.1:3008/setup/profile", { signal: AbortSignal.timeout(5000) })).json()
|
||||
if (diag.path) { profilePath = diag.path; profileDetected = true }
|
||||
}
|
||||
} catch {}
|
||||
}
|
||||
|
||||
// Login check
|
||||
// Detect all browsers via scraper
|
||||
let browsers = { firefox: { path: null }, opera: { path: null }, chrome: { path: null }, edge: { path: null } }
|
||||
let facebookLoggedIn = false
|
||||
if (profileDetected) {
|
||||
try {
|
||||
const r = await fetch("http://127.0.0.1:3008/setup/check-login", {
|
||||
method: "POST", headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ profile_path: profilePath }),
|
||||
signal: AbortSignal.timeout(15000),
|
||||
})
|
||||
if (r.ok) { const d = await r.json(); facebookLoggedIn = d.logged_in === true }
|
||||
} catch {}
|
||||
}
|
||||
let selectedBrowser = process.env.SELECTED_BROWSER || ""
|
||||
|
||||
const firstRun = !envExists || !ollamaRunning || !profileDetected || !modelAvailable
|
||||
try {
|
||||
await fetch("http://127.0.0.1:3008/health", { signal: AbortSignal.timeout(2000) })
|
||||
const profiles = await (await fetch("http://127.0.0.1:3008/setup/profile", { signal: AbortSignal.timeout(5000) })).json()
|
||||
for (const [b, p] of Object.entries(profiles)) {
|
||||
if (p) browsers[b] = { path: p }
|
||||
}
|
||||
// Check login for the selected browser first, then try all
|
||||
const detectedList = Object.entries(browsers).filter(([, v]) => v.path)
|
||||
for (const [b, v] of detectedList) {
|
||||
try {
|
||||
const r = await fetch("http://127.0.0.1:3008/setup/check-login", {
|
||||
method: "POST", headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ browser: b, profile_path: v.path }),
|
||||
signal: AbortSignal.timeout(20000),
|
||||
})
|
||||
if (r.ok) {
|
||||
const d = await r.json()
|
||||
browsers[b].logged_in = d.logged_in === true
|
||||
if (d.logged_in && !facebookLoggedIn) {
|
||||
facebookLoggedIn = true
|
||||
if (!selectedBrowser) selectedBrowser = b
|
||||
}
|
||||
}
|
||||
} catch {}
|
||||
}
|
||||
} catch {}
|
||||
|
||||
const anyDetected = Object.values(browsers).some(v => v.path)
|
||||
// first_run = any setup step is incomplete
|
||||
const firstRun = !envExists || !ollamaRunning || !anyDetected || !facebookLoggedIn || !modelAvailable
|
||||
|
||||
return sendJSON(res, 200, {
|
||||
first_run: firstRun,
|
||||
@@ -361,46 +404,71 @@ const server = http.createServer(async (req, res) => {
|
||||
ollama_running: ollamaRunning,
|
||||
model_available: modelAvailable,
|
||||
model_name: MODEL,
|
||||
profile_detected: profileDetected,
|
||||
profile_path: profilePath || null,
|
||||
selected_browser: selectedBrowser,
|
||||
browsers,
|
||||
facebook_logged_in: facebookLoggedIn,
|
||||
})
|
||||
}
|
||||
|
||||
// POST /setup/profile — save profile path to .env.local
|
||||
// POST /setup/profile — save selected browser + path to .env.local
|
||||
// Called by the setup wizard when the user confirms their browser choice.
|
||||
// Writes SELECTED_BROWSER and the matching profile env var to .env.local.
|
||||
if (req.method === "POST" && pathname === "/setup/profile") {
|
||||
const body = await parseBody(req)
|
||||
const browserName = (body.browser || "").trim().toLowerCase()
|
||||
const profilePath = (body.path || "").trim()
|
||||
if (!profilePath) return sendJSON(res, 400, { error: "Path required" })
|
||||
if (!fs.existsSync(profilePath)) return sendJSON(res, 400, { error: "Path does not exist" })
|
||||
if (!browserName || !["firefox", "opera", "chrome", "edge"].includes(browserName))
|
||||
return sendJSON(res, 400, { error: "Valid browser required (firefox/opera/chrome/edge)" })
|
||||
if (!profilePath)
|
||||
return sendJSON(res, 400, { error: "Path required" })
|
||||
|
||||
const envKey = browserName === "firefox" ? "FX_PROFILE"
|
||||
: browserName === "opera" ? "OPERA_PROFILE"
|
||||
: browserName === "edge" ? "EDGE_PROFILE"
|
||||
: "CHROME_PROFILE"
|
||||
|
||||
const envPath = path.join(ROOT, ".env.local")
|
||||
let content = ""
|
||||
try { content = fs.readFileSync(envPath, "utf-8") } catch {}
|
||||
const lines = content.split("\n")
|
||||
let found = false
|
||||
let lines = content.split("\n")
|
||||
// Update or add SELECTED_BROWSER
|
||||
let foundSel = false
|
||||
for (let i = 0; i < lines.length; i++) {
|
||||
if (lines[i].trim().startsWith("FX_PROFILE=")) {
|
||||
lines[i] = `FX_PROFILE=${profilePath}`
|
||||
found = true
|
||||
if (lines[i].trim().startsWith("SELECTED_BROWSER=")) {
|
||||
lines[i] = `SELECTED_BROWSER=${browserName}`
|
||||
foundSel = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if (!found) lines.push(`FX_PROFILE=${profilePath}`)
|
||||
if (!foundSel) lines.push(`SELECTED_BROWSER=${browserName}`)
|
||||
// Update or add browser profile
|
||||
let foundProf = false
|
||||
for (let i = 0; i < lines.length; i++) {
|
||||
if (lines[i].trim().startsWith(`${envKey}=`)) {
|
||||
lines[i] = `${envKey}=${profilePath}`
|
||||
foundProf = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if (!foundProf) lines.push(`${envKey}=${profilePath}`)
|
||||
fs.writeFileSync(envPath, lines.join("\n"), "utf-8")
|
||||
process.env.FX_PROFILE = profilePath
|
||||
return sendJSON(res, 200, { success: true, path: profilePath })
|
||||
process.env.SELECTED_BROWSER = browserName
|
||||
process.env[envKey] = profilePath
|
||||
return sendJSON(res, 200, { success: true, browser: browserName, path: profilePath })
|
||||
}
|
||||
|
||||
// POST /setup/check-login — verify Facebook login in the given profile
|
||||
// POST /setup/check-login — proxy to scraper, accepts browser + profile_path
|
||||
// The splash page calls this (via the AI server) to verify Facebook login status.
|
||||
if (req.method === "POST" && pathname === "/setup/check-login") {
|
||||
const body = await parseBody(req)
|
||||
const profilePath = body.profile_path || process.env.FX_PROFILE || ""
|
||||
const browserName = (body.browser || "").trim().toLowerCase() || process.env.SELECTED_BROWSER || ""
|
||||
const profilePath = (body.profile_path || "").trim()
|
||||
|
||||
if (!profilePath) return sendJSON(res, 200, { logged_in: false, reason: "no_profile" })
|
||||
try {
|
||||
const r = await fetch("http://127.0.0.1:3008/setup/check-login", {
|
||||
method: "POST", headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ profile_path: profilePath }),
|
||||
body: JSON.stringify({ browser: browserName, profile_path: profilePath }),
|
||||
signal: AbortSignal.timeout(20000),
|
||||
})
|
||||
if (r.ok) { const d = await r.json(); return sendJSON(res, 200, d) }
|
||||
@@ -409,6 +477,8 @@ const server = http.createServer(async (req, res) => {
|
||||
}
|
||||
|
||||
// POST /setup/ollama/pull — start pulling the model
|
||||
// Spawns "ollama pull" as a child process. The setup wizard polls
|
||||
// the progress endpoint to show a download progress bar.
|
||||
if (req.method === "POST" && pathname === "/setup/ollama/pull") {
|
||||
if (pullProcess) return sendJSON(res, 200, { status: "already_running" })
|
||||
pullProgress = { status: "downloading", progress: 0, message: "Starting..." }
|
||||
@@ -432,22 +502,23 @@ const server = http.createServer(async (req, res) => {
|
||||
}
|
||||
|
||||
// GET /setup/ollama/pull/progress
|
||||
// Returns current download progress for the setup wizard.
|
||||
if (req.method === "GET" && pathname === "/setup/ollama/pull/progress") {
|
||||
return sendJSON(res, 200, pullProgress)
|
||||
}
|
||||
|
||||
// GET /ai/jobs
|
||||
// GET /ai/jobs — return loaded job categories
|
||||
if (req.method === "GET" && pathname === "/ai/jobs") {
|
||||
return sendJSON(res, 200, { jobs: loadedJobs })
|
||||
}
|
||||
|
||||
// GET /ai/instructions
|
||||
// GET /ai/instructions — return current ai.md content
|
||||
if (req.method === "GET" && pathname === "/ai/instructions") {
|
||||
const instructions = readInstructions()
|
||||
return sendJSON(res, 200, { success: true, instructions })
|
||||
}
|
||||
|
||||
// POST /ai/instructions
|
||||
// POST /ai/instructions — update ai.md or append improvement log entry
|
||||
if (req.method === "POST" && pathname === "/ai/instructions") {
|
||||
const body = await parseBody(req)
|
||||
if (body.content) {
|
||||
@@ -461,18 +532,17 @@ const server = http.createServer(async (req, res) => {
|
||||
})
|
||||
}
|
||||
|
||||
// POST /ai/chat
|
||||
// POST /ai/chat — main AI chat endpoint
|
||||
// Accepts { message, user_id?, user_role? } and returns AI response.
|
||||
// user_role must be "sales", "admin", or "super_admin" if provided.
|
||||
if (req.method === "POST" && pathname === "/ai/chat") {
|
||||
const startTime = Date.now()
|
||||
const chunks = []
|
||||
req.on("data", c => chunks.push(c))
|
||||
req.on("end", () => {
|
||||
const rawBody = Buffer.concat(chunks).toString()
|
||||
try { fs.appendFileSync("C:\\Users\\USER-PC\\AppData\\Local\\Temp\\opencode\\ai-req-log.txt",
|
||||
`${new Date().toISOString()} headers=${JSON.stringify(req.headers)} body=${rawBody}\n`) } catch {}
|
||||
try {
|
||||
const body = JSON.parse(rawBody)
|
||||
// Continue processing
|
||||
processRequest(req, res, body, startTime)
|
||||
} catch {
|
||||
sendJSON(res, 400, { error: "Invalid JSON" })
|
||||
@@ -481,7 +551,7 @@ const server = http.createServer(async (req, res) => {
|
||||
return
|
||||
}
|
||||
|
||||
// Separate handler
|
||||
// Separate handler for /ai/chat (defined here due to hoisting within the IIFE)
|
||||
async function processRequest(req, res, body, startTime) {
|
||||
const { message, user_id, user_role } = body
|
||||
|
||||
@@ -498,7 +568,7 @@ async function processRequest(req, res, body, startTime) {
|
||||
return sendJSON(res, 200, { response })
|
||||
}
|
||||
|
||||
// 404
|
||||
// 404 fallback
|
||||
sendJSON(res, 404, { error: "Not found" })
|
||||
} catch (err) {
|
||||
console.error("Request error:", err)
|
||||
|
||||
+444
-66
@@ -1,3 +1,4 @@
|
||||
# ── Imports ──────────────────────────────────────────────────────────
|
||||
import os, json, asyncio, re, shutil, sqlite3, urllib.parse, random, logging, tempfile
|
||||
from datetime import datetime, timedelta
|
||||
from fastapi import FastAPI, Query, Body
|
||||
@@ -8,8 +9,12 @@ from browser_use import Agent, Browser
|
||||
from langchain_ollama import ChatOllama
|
||||
|
||||
|
||||
# ── Helpers ──────────────────────────────────────────────────────────
|
||||
def make_ollama(model: str | None = None, **kwargs) -> ChatOllama:
|
||||
"""Create ChatOllama with required attrs for browser-use Agent compatibility."""
|
||||
"""
|
||||
Create ChatOllama with required attrs for browser-use Agent compatibility.
|
||||
The browser-use Agent expects llm.name, llm.model_name, and llm.provider attrs.
|
||||
"""
|
||||
llm = ChatOllama(model=model or CLASSIFY_MODEL, **kwargs)
|
||||
object.__setattr__(llm, 'provider', 'ollama')
|
||||
object.__setattr__(llm, 'name', 'ChatOllama')
|
||||
@@ -17,6 +22,7 @@ def make_ollama(model: str | None = None, **kwargs) -> ChatOllama:
|
||||
return llm
|
||||
|
||||
|
||||
# ── Logging & App Setup ──────────────────────────────────────────────
|
||||
logging.basicConfig(level=logging.INFO, format='%(asctime)s [%(levelname)s] %(message)s')
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -29,13 +35,24 @@ app.add_middleware(
|
||||
)
|
||||
PORT = int(os.getenv("PORT", "3008"))
|
||||
|
||||
# ── AI / Ollama Config ───────────────────────────────────────────────
|
||||
OLLAMA_URL = os.getenv("OLLAMA_URL", "http://localhost:11434")
|
||||
CLASSIFY_MODEL = os.getenv("CLASSIFY_MODEL", "dolphin-llama3:8b")
|
||||
|
||||
|
||||
# ── Env Vars (set via .env.local) ──────────────────────────────────────
|
||||
# These store detected browser profile paths from the setup wizard.
|
||||
FX_PROFILE = os.getenv('FX_PROFILE', '')
|
||||
CHROME_PROFILE = os.getenv('CHROME_PROFILE', '')
|
||||
OPERA_PROFILE = os.getenv('OPERA_PROFILE', '')
|
||||
EDGE_PROFILE = os.getenv('EDGE_PROFILE', '')
|
||||
# SELECTED_BROWSER determines which browser the scrape dispatcher tries first.
|
||||
# Values: "firefox", "chrome", "opera", "edge", or "" (auto-detect all).
|
||||
SELECTED_BROWSER = os.getenv('SELECTED_BROWSER', '')
|
||||
|
||||
# ── Chromium Launch Args ──────────────────────────────────────────────
|
||||
# These flags reduce detectable automation signals in Chrome/Edge/Opera.
|
||||
# They disable background services, sync, updates, throttling, popups, etc.
|
||||
CHROME_LAUNCH_ARGS = [
|
||||
'--headless=new',
|
||||
'--disable-blink-features=AutomationControlled',
|
||||
@@ -60,20 +77,36 @@ CHROME_LAUNCH_ARGS = [
|
||||
]
|
||||
|
||||
|
||||
# ── Profile Detection ────────────────────────────────────────────────
|
||||
# All find_*_profile() functions return the parent directory (User Data or equivalent),
|
||||
# NOT the "Default" subdirectory. This is important because copy_chrome_profile()
|
||||
# appends "Default" internally, so returning the full Default path would cause
|
||||
# duplicated path segments (e.g. ".../User Data/Default/Default").
|
||||
|
||||
def detect_browser_from_profile(profile_path: str) -> str | None:
|
||||
"""Detect browser type from profile path. Returns 'firefox', 'chromium', or None."""
|
||||
"""
|
||||
Infer browser type from the profile path string.
|
||||
Checks for keywords like "firefox", "opera", "edge", "chrome" in the path.
|
||||
Returns "firefox", "chrome", "opera", "edge", or None.
|
||||
"""
|
||||
if not profile_path:
|
||||
return None
|
||||
p = profile_path.lower().replace('\\', '/')
|
||||
if 'firefox' in p or '.default' in p or '.dev-edition' in p:
|
||||
return 'firefox'
|
||||
if 'chrome' in p or 'chromium' in p or 'edge' in p:
|
||||
return 'chromium'
|
||||
if 'opera' in p:
|
||||
return 'opera'
|
||||
if 'edge' in p:
|
||||
return 'edge'
|
||||
if 'chrome' in p or 'chromium' in p:
|
||||
return 'chrome'
|
||||
return None
|
||||
|
||||
|
||||
def find_firefox_profile() -> str | None:
|
||||
"""Auto-detect Firefox profile directory cross-platform."""
|
||||
"""Auto-detect Firefox profile directory cross-platform.
|
||||
Scans the Profiles dir and returns the first .default or .dev-edition folder found,
|
||||
preferring "default-release" profiles."""
|
||||
import sys
|
||||
home = os.path.expanduser("~")
|
||||
candidates = []
|
||||
@@ -102,7 +135,8 @@ def find_firefox_profile() -> str | None:
|
||||
|
||||
|
||||
def find_chrome_profile() -> str | None:
|
||||
"""Auto-detect Chrome/Chromium profile directory cross-platform."""
|
||||
"""Auto-detect Chrome profile (User Data dir) cross-platform.
|
||||
Returns the User Data parent dir (NOT the Default subdirectory)."""
|
||||
import sys
|
||||
home = os.path.expanduser("~")
|
||||
|
||||
@@ -113,15 +147,60 @@ def find_chrome_profile() -> str | None:
|
||||
else:
|
||||
base = os.path.join(home, ".config", "google-chrome")
|
||||
|
||||
default_profile = os.path.join(base, "Default")
|
||||
if os.path.isdir(default_profile):
|
||||
logger.info("Auto-detected Chrome profile: %s", default_profile)
|
||||
return default_profile
|
||||
if os.path.isdir(os.path.join(base, "Default")):
|
||||
logger.info("Auto-detected Chrome profile: %s", base)
|
||||
return base
|
||||
return None
|
||||
|
||||
|
||||
def find_opera_profile() -> str | None:
|
||||
"""Auto-detect Opera profile directory cross-platform.
|
||||
Opera stores its profile at the "Opera Stable" level (no "User Data/Default"
|
||||
nesting like Chrome/Edge), so this returns the profile root directly."""
|
||||
import sys
|
||||
home = os.path.expanduser("~")
|
||||
|
||||
if sys.platform == "win32":
|
||||
base = os.path.join(os.environ.get("APPDATA", ""), "Opera Software", "Opera Stable")
|
||||
elif sys.platform == "darwin":
|
||||
base = os.path.join(home, "Library", "Application Support", "com.operasoftware.Opera")
|
||||
else:
|
||||
base = os.path.join(home, ".config", "opera")
|
||||
|
||||
if os.path.isdir(base) and os.listdir(base):
|
||||
logger.info("Auto-detected Opera profile: %s", base)
|
||||
return base
|
||||
return None
|
||||
|
||||
|
||||
def find_edge_profile() -> str | None:
|
||||
"""Auto-detect Edge profile (User Data dir) cross-platform.
|
||||
Returns the User Data parent dir (NOT the Default subdirectory)."""
|
||||
import sys
|
||||
home = os.path.expanduser("~")
|
||||
|
||||
if sys.platform == "win32":
|
||||
base = os.path.join(os.environ.get("LOCALAPPDATA", ""), "Microsoft", "Edge", "User Data")
|
||||
elif sys.platform == "darwin":
|
||||
base = os.path.join(home, "Library", "Application Support", "Microsoft Edge")
|
||||
else:
|
||||
base = os.path.join(home, ".config", "microsoft-edge")
|
||||
|
||||
if os.path.isdir(os.path.join(base, "Default")):
|
||||
logger.info("Auto-detected Edge profile: %s", base)
|
||||
return base
|
||||
return None
|
||||
|
||||
|
||||
# ── Profile Copying ─────────────────────────────────────────────────
|
||||
# Both functions copy essential cookies/login/storage files from the user's
|
||||
# real browser profile to a temp directory. Playwright then uses these temp
|
||||
# profiles via launch_persistent_context(), preserving Facebook login state
|
||||
# without modifying the user's actual profile.
|
||||
|
||||
def copy_firefox_profile(src_path: str) -> str:
|
||||
"""Copy essential Firefox profile files to a temp dir for Playwright."""
|
||||
"""Copy essential Firefox profile files (cookies, localStorage, permissions) to a temp dir for Playwright.
|
||||
Also creates a minimal profiles.ini so Firefox treats it as a valid profile."""
|
||||
essential = ['cookies.sqlite', 'webappsstore.sqlite', 'permissions.sqlite']
|
||||
dst = tempfile.mkdtemp(prefix='fb_fx_profile_')
|
||||
for f_name in essential:
|
||||
@@ -135,7 +214,12 @@ def copy_firefox_profile(src_path: str) -> str:
|
||||
|
||||
|
||||
def copy_chrome_profile(user_data_dir: str, profile_dir: str = 'Default') -> str:
|
||||
"""Copy Chrome profile to temp dir for launch_persistent_context."""
|
||||
"""Copy Chromium-based profile (Chrome/Edge/Opera) to a temp dir.
|
||||
Copies Cookies, Login Data, Bookmarks, Web Data, Local Storage, Session Storage, and Local State.
|
||||
Args:
|
||||
user_data_dir: Parent directory (e.g. ".../Chrome/User Data")
|
||||
profile_dir: Subdirectory name, typically "Default" (default)
|
||||
"""
|
||||
essential = ['Cookies', 'Login Data', 'Bookmarks', 'Web Data']
|
||||
dst = tempfile.mkdtemp(prefix='fb_chrome_udir_')
|
||||
src_profile = os.path.join(user_data_dir, profile_dir)
|
||||
@@ -159,7 +243,8 @@ def copy_chrome_profile(user_data_dir: str, profile_dir: str = 'Default') -> str
|
||||
|
||||
|
||||
def ensure_ublock_extension() -> str | None:
|
||||
"""Download and extract uBlock Origin extension for ad blocking."""
|
||||
"""Download and extract uBlock Origin extension for ad blocking via Chromium.
|
||||
Cached after first download to avoid re-downloading on every scrape."""
|
||||
ext_dir = os.path.join(tempfile.gettempdir(), 'fb_ublock_origin')
|
||||
manifest = os.path.join(ext_dir, 'manifest.json')
|
||||
if os.path.exists(manifest):
|
||||
@@ -182,10 +267,15 @@ def ensure_ublock_extension() -> str | None:
|
||||
return None
|
||||
|
||||
|
||||
# ── Agent Output Parsing ──────────────────────────────────────────────
|
||||
# When the browser-use Agent fallback is used, it returns natural language
|
||||
# output containing post listings. This function tries to extract structured
|
||||
# data from that output, first attempting JSON parsing, then falling back to
|
||||
# a line-by-line heuristic (numbered or bulleted items).
|
||||
|
||||
def extract_agent_posts(agent_result: str, page_text: str) -> list[dict]:
|
||||
"""Parse posts from agent output or page text fallback."""
|
||||
posts = []
|
||||
# Try JSON from agent output
|
||||
json_match = re.search(r'\[.*?\]', agent_result, re.DOTALL)
|
||||
if json_match:
|
||||
try:
|
||||
@@ -221,6 +311,12 @@ def extract_agent_posts(agent_result: str, page_text: str) -> list[dict]:
|
||||
return posts
|
||||
|
||||
|
||||
# ── Keyword Filters for Lead Detection ──────────────────────────────
|
||||
# BROAD_KEYWORDS — catch any post that might relate to web development needs.
|
||||
# OFFER_PATTERNS — regex patterns matching service offers (which we ignore).
|
||||
# REQUEST_PATTERNS — regex patterns matching people asking for help/services.
|
||||
# Together these form a two-pass filter + AI classification pipeline.
|
||||
|
||||
BROAD_KEYWORDS = [
|
||||
"website", "web design", "web develop", "web dev",
|
||||
"build my website", "build a website", "create a website",
|
||||
@@ -290,6 +386,13 @@ REQUEST_PATTERNS = [
|
||||
r"\bquote\s+(please|for|to)\b",
|
||||
]
|
||||
|
||||
# ── Anti-Detection Scripts ────────────────────────────────────────────
|
||||
# These JS scripts are injected into every page via add_init_script().
|
||||
# They override navigator properties (webdriver, userAgent, plugins, etc.)
|
||||
# and WebGL/canvas/audio fingerprints to make Playwright harder to detect.
|
||||
# Firefox and Chromium variants differ slightly (e.g. Chrome needs the
|
||||
# "window.chrome" object; Firefox must NOT have it).
|
||||
|
||||
def firefox_init_script() -> str:
|
||||
return r"""// Firefox Anti-Detection
|
||||
Object.defineProperty(navigator, 'webdriver', { get: () => undefined });
|
||||
@@ -384,18 +487,25 @@ Object.defineProperty(navigator, 'plugins', { get: () => [1,2,3,4,5].map(i => ({
|
||||
Object.defineProperty(navigator, 'mimeTypes', { get: () => [{type:'application/pdf',suffixes:'pdf',description:'Portable Document Format',enabledPlugin:navigator.plugins[0]},{type:'text/pdf',suffixes:'pdf',description:'Portable Document Format',enabledPlugin:navigator.plugins[0]}] });"""
|
||||
|
||||
|
||||
# ── Text Classification Utilities ────────────────────────────────────
|
||||
def kw_match(text: str) -> bool:
|
||||
t = text.lower()
|
||||
return any(kw in t for kw in BROAD_KEYWORDS)
|
||||
|
||||
def is_request(text: str) -> bool:
|
||||
"""Detect if text contains a request for services (e.g. "I need", "looking for")."""
|
||||
t = text.lower()
|
||||
return any(re.search(p, t) for p in REQUEST_PATTERNS)
|
||||
|
||||
def is_offer(text: str) -> bool:
|
||||
"""Detect if text is an offer/advertisement (we skip these, not leads)."""
|
||||
t = text.lower()
|
||||
return any(re.search(p, t) for p in OFFER_PATTERNS)
|
||||
|
||||
# ── Facebook Search Queries ───────────────────────────────────────────
|
||||
# These are the search terms the scraper will use on Facebook.
|
||||
# They target people looking for web development / website services.
|
||||
# Each run picks 2-4 random queries to vary behavior and reduce detection.
|
||||
FB_SEARCHES = [
|
||||
"looking for web developer",
|
||||
"need a website designed",
|
||||
@@ -428,6 +538,12 @@ VIEWPORTS = [
|
||||
{'width': 1920, 'height': 1080},
|
||||
]
|
||||
|
||||
# ── Cookie Extraction ────────────────────────────────────────────────
|
||||
# Reads Facebook cookies from the browser profile's SQLite database.
|
||||
# Firefox stores cookies in cookies.sqlite; Chromium stores them in Cookies
|
||||
# (a SQLite DB as well). This function handles Firefox's format.
|
||||
# The cookies are copied to a temp file first to avoid locking issues.
|
||||
|
||||
async def get_fb_cookies(profile_path: str | None = None):
|
||||
cookie_db_path = profile_path or FX_PROFILE
|
||||
if not cookie_db_path:
|
||||
@@ -472,6 +588,9 @@ async def get_fb_cookies(profile_path: str | None = None):
|
||||
pass
|
||||
return []
|
||||
|
||||
# ── Date Parsing ─────────────────────────────────────────────────────
|
||||
# Facebook displays dates in relative format ("2h ago", "Yesterday", "Monday")
|
||||
# or various absolute formats. These functions normalize them to YYYY-MM-DD.
|
||||
WEEKDAY_ORDER = ['monday', 'tuesday', 'wednesday', 'thursday', 'friday', 'saturday', 'sunday']
|
||||
|
||||
def _parse_fb_date(block: list[str]) -> str:
|
||||
@@ -525,6 +644,12 @@ def _clean_fb_text(text: str) -> str:
|
||||
cleaned_lines.append(stripped)
|
||||
return '\n'.join(cleaned_lines)
|
||||
|
||||
# ── Post Extraction ──────────────────────────────────────────────────
|
||||
# Two strategies for extracting posts from page content:
|
||||
# 1. _extract_posts_from_elements — uses structured DOM data from _get_article_elements()
|
||||
# 2. _extract_posts_from_text — fallback that parses raw page text line by line
|
||||
# Both apply dedup (seen_texts), offer filtering, and request scoring.
|
||||
|
||||
def _extract_posts_from_elements(elements: list[dict], base_url: str) -> list[dict]:
|
||||
posts = []
|
||||
seen_texts = set()
|
||||
@@ -617,6 +742,11 @@ def _extract_posts_from_text(raw: str, url: str) -> list[dict]:
|
||||
cur.pop(0)
|
||||
return posts
|
||||
|
||||
# ── Human-like Behavior Simulation ──────────────────────────────────
|
||||
# These functions add random delays, mouse movements, and scroll patterns
|
||||
# to make automated browsing look more like a real human user.
|
||||
# This is critical for avoiding Facebook's bot detection.
|
||||
|
||||
async def human_scroll(page, steps: int = None, total_delay: float = None):
|
||||
steps = steps or random.randint(2, 5)
|
||||
total_delay = total_delay or random.uniform(6, 18)
|
||||
@@ -652,6 +782,12 @@ async def random_idle(page):
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# ── Facebook DOM Parsing ─────────────────────────────────────────────
|
||||
# This JS function runs inside the page context to extract structured post
|
||||
# data (text, author, URL, date) from Facebook's complex DOM structure.
|
||||
# It tries multiple selectors (article, feed, etc.) and uses the first
|
||||
# one that returns results, since Facebook's DOM changes frequently.
|
||||
|
||||
async def _get_article_elements(page) -> list[dict]:
|
||||
return await page.evaluate('''() => {
|
||||
const results = [];
|
||||
@@ -713,6 +849,8 @@ async def _get_article_elements(page) -> list[dict]:
|
||||
}''')
|
||||
|
||||
async def _ensure_page(page, context):
|
||||
"""Check if the current page is still alive. If closed (e.g. by a popup),
|
||||
create a fresh page and navigate to Facebook."""
|
||||
try:
|
||||
await page.evaluate('1')
|
||||
return page
|
||||
@@ -728,6 +866,12 @@ async def _ensure_page(page, context):
|
||||
await page.wait_for_timeout(random.randint(3000, 8000))
|
||||
return page
|
||||
|
||||
|
||||
# ── Facebook Search & Scrape ─────────────────────────────────────────
|
||||
# Performs a Facebook search for a given query, scrolls through results,
|
||||
# extracts posts, and returns them. Includes randomization of scroll behavior,
|
||||
# mouse movements, and idle actions to mimic human browsing patterns.
|
||||
|
||||
async def search_facebook(page, context, query: str):
|
||||
page = await _ensure_page(page, context)
|
||||
url = f'https://www.facebook.com/search/posts/?q={urllib.parse.quote(query)}'
|
||||
@@ -764,6 +908,10 @@ async def search_facebook(page, context, query: str):
|
||||
return page, []
|
||||
return page, posts
|
||||
|
||||
# ── Detection Signals ────────────────────────────────────────────────
|
||||
# Keywords/phrases in page URL or body text that indicate Facebook's
|
||||
# security/bot detection systems have flagged the session.
|
||||
|
||||
DETECTION_SIGNALS = [
|
||||
'/checkpoint/', '/login.php?', 'action=security_check',
|
||||
'unusual activity', 'suspicious login', 'suspicious activity',
|
||||
@@ -784,54 +932,90 @@ def check_detection_signals(page_url: str, page_text: str = '') -> str | None:
|
||||
|
||||
|
||||
def cleanup_chrome():
|
||||
"""Kill orphaned Chrome headless shell processes. These sometimes linger
|
||||
after failed scrapes and interfere with subsequent runs."""
|
||||
import subprocess, signal
|
||||
try:
|
||||
subprocess.run(["taskkill", "/F", "/IM", "chrome-headless-shell.exe"], capture_output=True, timeout=5)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
# ── Main Scrape Dispatcher ────────────────────────────────────────────
|
||||
# scrape_facebook() is the main entry point. It:
|
||||
# 1. Resolves the browser profile path (from SELECTED_BROWSER env var or auto-detect)
|
||||
# 2. Dispatches to the correct browser-specific scraper
|
||||
# 3. If the browser scrape is flagged by Facebook, falls through to the Agent fallback
|
||||
#
|
||||
# Priority order for browser selection:
|
||||
# - SELECTED_BROWSER env var (set by setup wizard) → that browser's scraper
|
||||
# - Auto-detect: Firefox → Opera → Chrome → Edge (first found wins)
|
||||
# - If no profile found → Agent (browser-use + ChatOllama) fallback
|
||||
|
||||
async def scrape_facebook(profile_path: str | None = None, force: bool = False) -> dict:
|
||||
"""Dispatcher — Firefox primary, browser-use Agent fallback."""
|
||||
effective_path = profile_path or FX_PROFILE
|
||||
"""Dispatcher — respect SELECTED_BROWSER, fall through on flag, then Agent."""
|
||||
effective_path = profile_path or ""
|
||||
browser_type = ""
|
||||
|
||||
# Auto-detect Firefox profile if none provided
|
||||
if not effective_path:
|
||||
detected = find_firefox_profile()
|
||||
if detected:
|
||||
effective_path = detected
|
||||
os.environ["FX_PROFILE"] = detected
|
||||
# Resolve profile and browser type
|
||||
if effective_path:
|
||||
browser_type = detect_browser_from_profile(effective_path) or ""
|
||||
elif SELECTED_BROWSER == "firefox":
|
||||
effective_path = FX_PROFILE or find_firefox_profile() or ""
|
||||
browser_type = "firefox"
|
||||
elif SELECTED_BROWSER == "opera":
|
||||
effective_path = OPERA_PROFILE or find_opera_profile() or ""
|
||||
browser_type = "opera"
|
||||
elif SELECTED_BROWSER == "edge":
|
||||
effective_path = EDGE_PROFILE or find_edge_profile() or ""
|
||||
browser_type = "edge"
|
||||
elif SELECTED_BROWSER == "chrome":
|
||||
effective_path = CHROME_PROFILE or find_chrome_profile() or ""
|
||||
browser_type = "chrome"
|
||||
else:
|
||||
# Auto-detect — try all in priority order
|
||||
for name, env_var, find_fn in [
|
||||
("firefox", "FX_PROFILE", find_firefox_profile),
|
||||
("opera", "OPERA_PROFILE", find_opera_profile),
|
||||
("chrome", "CHROME_PROFILE", find_chrome_profile),
|
||||
("edge", "EDGE_PROFILE", find_edge_profile),
|
||||
]:
|
||||
p = os.getenv(env_var, "") or find_fn() or ""
|
||||
if p:
|
||||
effective_path = p
|
||||
browser_type = name
|
||||
break
|
||||
|
||||
browser_type = detect_browser_from_profile(effective_path)
|
||||
if not browser_type or not effective_path:
|
||||
logger.info("No profile found — falling back to Agent")
|
||||
return await _scrape_with_agent(force)
|
||||
|
||||
if not browser_type and CHROME_PROFILE:
|
||||
browser_type = detect_browser_from_profile(CHROME_PROFILE)
|
||||
effective_path = CHROME_PROFILE
|
||||
logger.info("Selected browser: %s (profile: %s)", browser_type, effective_path)
|
||||
|
||||
# Auto-detect Chrome profile if still nothing
|
||||
if not browser_type:
|
||||
detected_chrome = find_chrome_profile()
|
||||
if detected_chrome:
|
||||
effective_path = detected_chrome
|
||||
browser_type = 'chromium'
|
||||
|
||||
logger.info("Detected browser: %s (profile: %s)", browser_type or "none", effective_path)
|
||||
|
||||
# Firefox primary (raw Playwright, stealth)
|
||||
# Firefox path
|
||||
if browser_type == "firefox":
|
||||
result = await _scrape_with_firefox(effective_path, force)
|
||||
if result.get("success") or not result.get("flagged"):
|
||||
return result
|
||||
logger.warning("Firefox path failed (%s), falling back to Agent", result.get("flag_reason", "unknown"))
|
||||
logger.warning("Firefox flagged (%s), trying Agent", result.get("flag_reason", "unknown"))
|
||||
return await _scrape_with_agent(force)
|
||||
|
||||
# CHROME_PROFILE set or no profile → Agent
|
||||
if browser_type == "chromium":
|
||||
return await _scrape_with_agent(force)
|
||||
|
||||
# No profile at all → try Agent (fresh Chromium)
|
||||
# Chromium-based (chrome / opera / edge)
|
||||
result = await _scrape_with_chromium(effective_path, browser_type, force)
|
||||
if result.get("success") or not result.get("flagged"):
|
||||
return result
|
||||
logger.warning("%s flagged (%s), trying Agent", browser_type, result.get("flag_reason", "unknown"))
|
||||
return await _scrape_with_agent(force)
|
||||
|
||||
|
||||
# ── Firefox Scraper ──────────────────────────────────────────────────
|
||||
# Launches a headless Firefox with the user's real profile cookies (copied
|
||||
# to a temp dir). Uses firefox_user_prefs to disable automation flags and
|
||||
# tracking protection. Injects anti-detection script via add_init_script().
|
||||
# If force=False, randomly skips 30% of runs as a decoy (looks like random
|
||||
# human browsing). If flagged by Facebook, returns flagged=True for the
|
||||
# dispatcher to fall through to the Agent.
|
||||
|
||||
async def _scrape_with_firefox(profile_path: str, force: bool) -> dict:
|
||||
"""Scrape Facebook using Firefox + persistent real profile (no cookie injection)."""
|
||||
if not profile_path:
|
||||
@@ -939,6 +1123,136 @@ async def _scrape_with_firefox(profile_path: str, force: bool) -> dict:
|
||||
pass
|
||||
|
||||
|
||||
# ── Chromium Scraper ─────────────────────────────────────────────────
|
||||
# Generic scraper for Chrome/Edge/Opera. Uses the same structure as the
|
||||
# Firefox scraper but with Chromium-specific launch config:
|
||||
# - Chrome: channel="chrome"
|
||||
# - Edge: channel="msedge"
|
||||
# - Opera: executable_path from shutil.which("opera")
|
||||
# Cookies come from copy_chrome_profile (temp copy of user's profile data).
|
||||
# Anti-detection script differs slightly from Firefox variant.
|
||||
# Same decoy-skip and flagged-fallback behavior as Firefox.
|
||||
|
||||
async def _scrape_with_chromium(profile_path: str, browser: str, force: bool = False) -> dict:
|
||||
"""Scrape Facebook using a Chromium-based browser profile (Chrome/Edge/Opera)."""
|
||||
if not profile_path:
|
||||
return {"success": False, "leads": [], "flagged": False, "flag_reason": None, "error": "No profile path"}
|
||||
|
||||
channel = None
|
||||
executable_path = None
|
||||
if browser == "chrome":
|
||||
channel = "chrome"
|
||||
elif browser == "edge":
|
||||
channel = "msedge"
|
||||
elif browser == "opera":
|
||||
executable_path = shutil.which("opera") or shutil.which("Opera")
|
||||
|
||||
profile_dir = None
|
||||
try:
|
||||
profile_dir = copy_chrome_profile(profile_path)
|
||||
async with async_playwright() as pw:
|
||||
launch_kwargs = dict(
|
||||
user_data_dir=profile_dir,
|
||||
headless=True,
|
||||
args=CHROME_LAUNCH_ARGS,
|
||||
)
|
||||
if channel:
|
||||
launch_kwargs["channel"] = channel
|
||||
if executable_path:
|
||||
launch_kwargs["executable_path"] = executable_path
|
||||
|
||||
context = await pw.chromium.launch_persistent_context(**launch_kwargs)
|
||||
pages = context.pages
|
||||
page = pages[0] if pages else await context.new_page()
|
||||
await context.add_init_script(chromium_init_script())
|
||||
|
||||
try:
|
||||
await page.goto('https://www.google.com/', wait_until='domcontentloaded', timeout=15000)
|
||||
await page.wait_for_timeout(random.randint(1000, 3000))
|
||||
except Exception:
|
||||
logger.warning("Google navigation failed, trying Facebook directly")
|
||||
|
||||
await page.goto('https://www.facebook.com/', wait_until='domcontentloaded', timeout=30000)
|
||||
await page.wait_for_timeout(random.randint(3000, 8000))
|
||||
|
||||
url = page.url
|
||||
page_text = await page.evaluate('document.body.innerText') if '/login' in url.lower() else ''
|
||||
det = check_detection_signals(url, page_text)
|
||||
if det or '/login' in url.lower():
|
||||
logger.warning("Facebook login page detected — flag: %s", det or "login_page")
|
||||
await context.close()
|
||||
return {"success": False, "leads": [], "flagged": True, "flag_reason": det or "login_page", "error": "Facebook login page detected"}
|
||||
|
||||
await human_scroll(page, steps=random.randint(2, 4), total_delay=random.uniform(8, 20))
|
||||
if random.random() < 0.25:
|
||||
await page.evaluate("window.scrollTo(0, 0)")
|
||||
await page.wait_for_timeout(random.randint(2000, 5000))
|
||||
await human_scroll(page, steps=random.randint(1, 2))
|
||||
|
||||
if not force and random.random() < 0.3:
|
||||
await page.wait_for_timeout(random.randint(8000, 20000))
|
||||
await context.close()
|
||||
return {"success": True, "leads": [], "flagged": False, "flag_reason": None, "error": None}
|
||||
|
||||
all_posts = []
|
||||
searches = random.sample(FB_SEARCHES, k=random.randint(2, 4))
|
||||
for i, query in enumerate(searches):
|
||||
page, posts = await search_facebook(page, context, query)
|
||||
all_posts.extend(posts)
|
||||
if not posts:
|
||||
continue
|
||||
if random.random() < 0.4:
|
||||
await page.evaluate(f"window.scrollBy(0, {random.randint(-300, 300)})")
|
||||
delay = random.uniform(8, 25)
|
||||
await page.wait_for_timeout(int(delay * 1000))
|
||||
if i == random.randint(0, 1) and random.random() < 0.15:
|
||||
new_page = await context.new_page()
|
||||
try:
|
||||
await new_page.goto('https://www.facebook.com/groups/', wait_until='domcontentloaded', timeout=15000)
|
||||
await new_page.wait_for_timeout(random.randint(3000, 8000))
|
||||
except Exception:
|
||||
pass
|
||||
await new_page.close()
|
||||
page = await _ensure_page(page, context)
|
||||
|
||||
if random.random() < 0.5:
|
||||
await page.wait_for_timeout(random.randint(3000, 10000))
|
||||
|
||||
await context.close()
|
||||
|
||||
seen = set()
|
||||
deduped = []
|
||||
for p in all_posts:
|
||||
key = p.get('content', '')[:100]
|
||||
if key not in seen:
|
||||
seen.add(key)
|
||||
deduped.append(p)
|
||||
|
||||
deduped = [p for p in deduped if _is_within_days(p.get('date', ''), 3)]
|
||||
leads = deduped[:20]
|
||||
if leads:
|
||||
leads = await classify_leads(leads)
|
||||
|
||||
return {"success": True, "leads": leads[:15], "flagged": False, "flag_reason": None, "error": None}
|
||||
|
||||
except Exception as e:
|
||||
logger.error("%s scrape failed: %s", browser, e)
|
||||
return {"success": False, "leads": [], "flagged": False, "flag_reason": None, "error": str(e)}
|
||||
finally:
|
||||
if profile_dir and os.path.exists(profile_dir):
|
||||
try:
|
||||
shutil.rmtree(profile_dir, ignore_errors=True)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
# ── Agent Fallback ──────────────────────────────────────────────────
|
||||
# When all browser-based scrapers fail/are flagged, this fallback uses
|
||||
# browser-use Agent with ChatOllama (local, free) to navigate Facebook
|
||||
# autonomously via GPT-style prompting. No API keys needed.
|
||||
# Uses Chromium headless with the same launch args as _scrape_with_chromium.
|
||||
# The Agent is prompted to extract structured post data and return JSON.
|
||||
|
||||
async def _scrape_with_agent(force: bool = False) -> dict:
|
||||
"""Fallback scraper — browser-use Agent + ChatOllama (free/local, Chromium)."""
|
||||
cleanup_chrome()
|
||||
@@ -1016,6 +1330,12 @@ When done, return the data as a JSON list with keys: content, author, url, date.
|
||||
except Exception as e:
|
||||
logger.warning("Failed to clean up Chrome profile %s: %s", profile_dir, e)
|
||||
|
||||
# ── AI Classification ────────────────────────────────────────────────
|
||||
# Uses the local Ollama model to classify scraped posts as LEAD or NOT LEAD.
|
||||
# Falls back to keyword-based filtering if the AI is unavailable.
|
||||
# ask_ollama() sends a raw prompt to Ollama's /api/chat endpoint.
|
||||
# classify_leads() builds a classification prompt and parses the JSON response.
|
||||
|
||||
async def ask_ollama(prompt: str) -> str:
|
||||
import httpx
|
||||
async with httpx.AsyncClient(timeout=120) as c:
|
||||
@@ -1101,39 +1421,97 @@ Return a JSON array like ["yes","no","yes"] matching the order above."""
|
||||
return filtered[:10]
|
||||
return []
|
||||
|
||||
# ══════════════════════════════════════════════════════════════════════
|
||||
# FastAPI Endpoints
|
||||
# ══════════════════════════════════════════════════════════════════════
|
||||
|
||||
@app.get("/health")
|
||||
async def health():
|
||||
"""Simple health check for the splash page status polling."""
|
||||
return {"status": "ok"}
|
||||
|
||||
def _detect_all_profiles() -> dict:
|
||||
"""Return all detected browser profiles."""
|
||||
return {
|
||||
"firefox": FX_PROFILE or find_firefox_profile() or "",
|
||||
"opera": OPERA_PROFILE or find_opera_profile() or "",
|
||||
"chrome": CHROME_PROFILE or find_chrome_profile() or "",
|
||||
"edge": EDGE_PROFILE or find_edge_profile() or "",
|
||||
}
|
||||
|
||||
@app.get("/setup/profile")
|
||||
async def setup_profile():
|
||||
"""Auto-detect Firefox profile path."""
|
||||
path = FX_PROFILE or find_firefox_profile()
|
||||
return {"path": path, "detected": bool(path)}
|
||||
"""Auto-detect all browser profiles."""
|
||||
return _detect_all_profiles()
|
||||
|
||||
async def _check_browser_login(profile_path: str, browser: str) -> dict:
|
||||
"""Check Facebook login for a specific browser type."""
|
||||
profile_dir = None
|
||||
try:
|
||||
if browser == "firefox":
|
||||
profile_dir = copy_firefox_profile(profile_path)
|
||||
async with async_playwright() as pw:
|
||||
context = await pw.firefox.launch_persistent_context(
|
||||
user_data_dir=profile_dir, headless=True,
|
||||
firefox_user_prefs={"dom.webdriver.enabled": False},
|
||||
)
|
||||
page = context.pages[0] if context.pages else await context.new_page()
|
||||
await page.goto("https://www.facebook.com/", wait_until="domcontentloaded", timeout=30000)
|
||||
await page.wait_for_timeout(3000)
|
||||
logged_in = "/login" not in page.url.lower()
|
||||
await context.close()
|
||||
shutil.rmtree(profile_dir, ignore_errors=True)
|
||||
return {"logged_in": logged_in, "browser": browser, "path": profile_path}
|
||||
else:
|
||||
channel = None
|
||||
exe = None
|
||||
if browser == "chrome":
|
||||
channel = "chrome"
|
||||
elif browser == "edge":
|
||||
channel = "msedge"
|
||||
elif browser == "opera":
|
||||
exe = shutil.which("opera") or shutil.which("Opera")
|
||||
profile_dir = copy_chrome_profile(profile_path)
|
||||
async with async_playwright() as pw:
|
||||
kw = dict(user_data_dir=profile_dir, headless=True, args=CHROME_LAUNCH_ARGS)
|
||||
if channel:
|
||||
kw["channel"] = channel
|
||||
if exe:
|
||||
kw["executable_path"] = exe
|
||||
context = await pw.chromium.launch_persistent_context(**kw)
|
||||
page = context.pages[0] if context.pages else await context.new_page()
|
||||
await page.goto("https://www.facebook.com/", wait_until="domcontentloaded", timeout=30000)
|
||||
await page.wait_for_timeout(3000)
|
||||
logged_in = "/login" not in page.url.lower()
|
||||
await context.close()
|
||||
shutil.rmtree(profile_dir, ignore_errors=True)
|
||||
return {"logged_in": logged_in, "browser": browser, "path": profile_path}
|
||||
except Exception as e:
|
||||
if profile_dir:
|
||||
try: shutil.rmtree(profile_dir, ignore_errors=True)
|
||||
except: pass
|
||||
return {"logged_in": False, "browser": browser, "path": profile_path, "error": str(e)}
|
||||
|
||||
@app.post("/setup/check-login")
|
||||
async def setup_check_login(body: dict):
|
||||
"""Check if Facebook is logged in using the given profile."""
|
||||
profile_path = (body.get("profile_path") or "").strip() or FX_PROFILE or find_firefox_profile()
|
||||
if not profile_path:
|
||||
return {"logged_in": False, "reason": "no_profile", "error": "No Firefox profile found"}
|
||||
try:
|
||||
profile_dir = copy_firefox_profile(profile_path)
|
||||
async with async_playwright() as pw:
|
||||
context = await pw.firefox.launch_persistent_context(
|
||||
user_data_dir=profile_dir, headless=True,
|
||||
firefox_user_prefs={"dom.webdriver.enabled": False},
|
||||
)
|
||||
page = context.pages[0] if context.pages else await context.new_page()
|
||||
await page.goto("https://www.facebook.com/", wait_until="domcontentloaded", timeout=30000)
|
||||
await page.wait_for_timeout(3000)
|
||||
logged_in = "/login" not in page.url.lower()
|
||||
await context.close()
|
||||
shutil.rmtree(profile_dir, ignore_errors=True)
|
||||
return {"logged_in": logged_in, "reason": None if logged_in else "login_page"}
|
||||
except Exception as e:
|
||||
logger.error("Setup check-login failed: %s", e)
|
||||
return {"logged_in": False, "reason": "error", "error": str(e)}
|
||||
"""Check if Facebook is logged in. Accepts optional 'browser' param, tries all if not given."""
|
||||
browser = (body.get("browser") or "").strip().lower()
|
||||
profile_path = (body.get("profile_path") or "").strip()
|
||||
|
||||
if browser and profile_path:
|
||||
return await _check_browser_login(profile_path, browser)
|
||||
|
||||
# No specific browser — try all detected
|
||||
profiles = _detect_all_profiles()
|
||||
priority = ["firefox", "opera", "chrome", "edge"]
|
||||
for b in priority:
|
||||
p = profiles.get(b, "")
|
||||
if p:
|
||||
result = await _check_browser_login(p, b)
|
||||
if result.get("logged_in"):
|
||||
return result
|
||||
logger.info("%s not logged in, trying next", b)
|
||||
return {"logged_in": False, "reason": "none_logged_in", "message": "No browser with Facebook login found"}
|
||||
|
||||
@app.post("/agent/run")
|
||||
async def agent_run(task: str = Body(..., embed=True)):
|
||||
|
||||
@@ -1,5 +1,18 @@
|
||||
# ── Python Dependencies for the Facebook Scraper (FastAPI) ───────
|
||||
# Install via: pip install -r requirements.txt
|
||||
|
||||
# Web framework for the REST API (health, setup, scrape endpoints)
|
||||
fastapi>=0.115.0
|
||||
|
||||
# ASGI server for running the FastAPI app
|
||||
uvicorn>=0.34.0
|
||||
|
||||
# Browser automation (launches Firefox, Chrome, Edge, Opera via Playwright)
|
||||
playwright>=1.49.0
|
||||
|
||||
# AI-powered browser agent (fallback when direct browser scraping is flagged)
|
||||
# Uses ChatOllama locally — no API keys needed
|
||||
browser-use>=0.1.0
|
||||
|
||||
# LangChain integration for ChatOllama (provides the LLM for browser-use Agent)
|
||||
langchain-ollama>=0.2.0
|
||||
|
||||
@@ -1,3 +1,9 @@
|
||||
// ── Ollama Launcher ────────────────────────────────────────────────
|
||||
// Checks if Ollama is already running. If not, finds the binary and
|
||||
// starts it as a detached background process.
|
||||
// Cross-platform: uses tasklist (Windows) or pgrep (Linux/Mac) to
|
||||
// detect running process; uses where/which to find the binary.
|
||||
|
||||
import { execSync, spawn } from "node:child_process"
|
||||
import { platform } from "node:os"
|
||||
|
||||
@@ -16,6 +22,7 @@ function isRunning() {
|
||||
|
||||
function findOllama() {
|
||||
if (platform() === "win32") {
|
||||
// Try PATH first, then common install locations
|
||||
try {
|
||||
return execSync("where ollama", { encoding: "utf8", timeout: 3000 }).trim().split("\n")[0]
|
||||
} catch {}
|
||||
@@ -36,11 +43,13 @@ if (!isRunning()) {
|
||||
process.exit(1)
|
||||
}
|
||||
console.log("Starting Ollama...")
|
||||
// Spawn detached so it outlives this script and the npm process
|
||||
if (platform() === "win32") {
|
||||
spawn(bin, ["serve"], { stdio: "ignore", detached: true, windowsHide: true }).unref()
|
||||
} else {
|
||||
spawn(bin, ["serve"], { stdio: "ignore", detached: true }).unref()
|
||||
}
|
||||
// Give it a moment to start listening
|
||||
execSync("sleep 3", { stdio: "ignore", timeout: 5000 })
|
||||
} else {
|
||||
console.log("Ollama already running")
|
||||
|
||||
@@ -1,3 +1,9 @@
|
||||
// ── Browser Opener ─────────────────────────────────────────────────
|
||||
// Opens the splash page in the user's default browser after an 8-second
|
||||
// delay. The delay gives all services time to start before the user
|
||||
// sees the loading screen.
|
||||
// Cross-platform: uses start (Windows), open (Mac), or xdg-open (Linux).
|
||||
|
||||
import { execSync } from "node:child_process"
|
||||
import { platform } from "node:os"
|
||||
|
||||
|
||||
@@ -1,9 +1,16 @@
|
||||
// ── Port Precheck ──────────────────────────────────────────────────
|
||||
// Kills any existing processes on ports 3001, 3006, 3007, 3008.
|
||||
// These are the AI server, Next.js frontend, Signaling server, and
|
||||
// Python scraper respectively.
|
||||
// Runs before anything else starts to avoid EADDRINUSE errors.
|
||||
|
||||
import { execSync } from "node:child_process"
|
||||
import { platform } from "node:os"
|
||||
|
||||
const PORTS = [3001, 3006, 3007, 3008]
|
||||
|
||||
if (platform() === "win32") {
|
||||
// Windows: use netstat + findstr to find listening PIDs, then taskkill
|
||||
for (const port of PORTS) {
|
||||
try {
|
||||
const out = execSync(`netstat -ano | findstr "LISTENING" | findstr ":${port} "`, { encoding: "utf8", timeout: 5000 })
|
||||
@@ -18,6 +25,7 @@ if (platform() === "win32") {
|
||||
} catch {}
|
||||
}
|
||||
} else {
|
||||
// Linux/Mac: use lsof -ti to find PIDs, then kill -9
|
||||
for (const port of PORTS) {
|
||||
try {
|
||||
const pid = execSync(`lsof -ti:${port} 2>/dev/null`, { encoding: "utf8", timeout: 5000 }).trim()
|
||||
|
||||
@@ -1,3 +1,9 @@
|
||||
// ── Python Runner ──────────────────────────────────────────────────
|
||||
// Detects the system's Python executable (python vs python3) and runs
|
||||
// a given script with arguments. Used by the dev:browser-use npm script.
|
||||
// Avoids shell:true — spawns Python directly with its full path.
|
||||
// Cross-platform: uses where (Windows) or which (Linux/Mac).
|
||||
|
||||
import { execSync, spawn } from "node:child_process"
|
||||
import { platform } from "node:os"
|
||||
|
||||
@@ -23,5 +29,6 @@ if (!script) {
|
||||
process.exit(1)
|
||||
}
|
||||
|
||||
// Spawn Python with inherited stdio so the script's output is visible
|
||||
const proc = spawn(PYTHON, [script, ...args], { stdio: "inherit" })
|
||||
proc.on("exit", (code) => process.exit(code ?? 1))
|
||||
|
||||
+17
-4
@@ -1,9 +1,21 @@
|
||||
// ── One-command Setup ──────────────────────────────────────────────
|
||||
// Run via: npm run setup
|
||||
// Does the following in order:
|
||||
// 1. npm install (Node.js dependencies)
|
||||
// 2. pip install -r requirements.txt (Python dependencies)
|
||||
// 3. playwright install firefox chromium (Playwright browsers)
|
||||
// 4. Copies .env.example to .env.local if not exists
|
||||
//
|
||||
// All steps are cross-platform (Windows, Mac, Linux).
|
||||
// Uses execSync for simplicity since each step blocks the next.
|
||||
|
||||
import { execSync } from "node:child_process"
|
||||
import { existsSync, copyFileSync } from "node:fs"
|
||||
import { platform } from "node:os"
|
||||
|
||||
const SEP = platform() === "win32" ? "&" : ";"
|
||||
|
||||
// Auto-detect Python executable (python vs python3)
|
||||
function detectPython() {
|
||||
const candidates = platform() === "win32" ? ["python", "python3"] : ["python3", "python"]
|
||||
for (const cmd of candidates) {
|
||||
@@ -16,6 +28,7 @@ function detectPython() {
|
||||
process.exit(1)
|
||||
}
|
||||
|
||||
// Auto-detect pip (pip vs pip3), fall back to python -m pip
|
||||
function detectPip(python) {
|
||||
const candidates = platform() === "win32" ? ["pip", "pip3"] : ["pip3", "pip"]
|
||||
for (const cmd of candidates) {
|
||||
@@ -45,13 +58,13 @@ console.log("=== CoastIT CRM Setup ===\n")
|
||||
// 1. Node dependencies
|
||||
run("npm install", "Installing Node.js dependencies")
|
||||
|
||||
// 2. Python dependencies
|
||||
// 2. Python dependencies (run from browser-use-service directory)
|
||||
run(`cd browser-use-service ${SEP} ${PIP} install -r requirements.txt`, "Installing Python dependencies")
|
||||
|
||||
// 3. Playwright browsers
|
||||
// 3. Playwright browsers (Firefox for primary scraping, Chromium for Chrome/Edge/Opera + Agent fallback)
|
||||
run(`${PY} -m playwright install firefox chromium`, "Installing Playwright browsers")
|
||||
|
||||
// 4. .env file
|
||||
// 4. .env file — create from template if it doesn't exist
|
||||
if (!existsSync(".env.local")) {
|
||||
console.log("\n── Creating .env.local ──")
|
||||
copyFileSync(".env.example", ".env.local")
|
||||
@@ -60,7 +73,7 @@ if (!existsSync(".env.local")) {
|
||||
console.log("\n── .env.local already exists, skipping ──")
|
||||
}
|
||||
|
||||
// 5. Ollama model
|
||||
// 5. Remaining manual steps
|
||||
console.log("\n── Next steps ──")
|
||||
console.log(" 1. Make sure PostgreSQL is running with database 'crm'")
|
||||
console.log(" 2. Pull the Ollama model: ollama pull dolphin-llama3:8b")
|
||||
|
||||
+138
-138
@@ -5,6 +5,7 @@
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>Loading CoastIT CRM</title>
|
||||
<style>
|
||||
/* ── Global Reset ──────────────────────────────────────────────── */
|
||||
* { margin: 0; padding: 0; box-sizing: border-box; }
|
||||
body {
|
||||
background: #0a0a1a;
|
||||
@@ -18,7 +19,8 @@
|
||||
overflow: hidden;
|
||||
}
|
||||
|
||||
/* Robot */
|
||||
/* ── Robot Animation ────────────────────────────────────────────── */
|
||||
/* Animated robot mascot that bobs, swings arms, and blinks while loading */
|
||||
.robot {
|
||||
position: relative;
|
||||
width: 120px;
|
||||
@@ -128,7 +130,9 @@
|
||||
100% { transform: translateY(-4px) rotate(8deg); }
|
||||
}
|
||||
|
||||
/* Status indicators */
|
||||
/* ── Status Indicators ────────────────────────────────────────── */
|
||||
/* Shows AI Server, Scraper, and Frontend status dots side by side.
|
||||
Each transitions from dim → cyan (ready) or red (failed). */
|
||||
.services {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
@@ -207,7 +211,7 @@
|
||||
color: #ef4444;
|
||||
}
|
||||
|
||||
/* Loading text */
|
||||
/* ── Loading Text ──────────────────────────────────────────────── */
|
||||
.loading-text {
|
||||
font-size: 16px;
|
||||
font-weight: 600;
|
||||
@@ -222,7 +226,9 @@
|
||||
50% { opacity: 1; }
|
||||
}
|
||||
|
||||
/* Launch gear (inline, below everything) */
|
||||
/* ── Launch Gear ────────────────────────────────────────────────── */
|
||||
/* Appears below the loading text after all services are ready.
|
||||
Shows a spinning gear + "Launching gear!" text, then redirects to /login. */
|
||||
.launch-overlay {
|
||||
display: none;
|
||||
flex-direction: column;
|
||||
@@ -255,7 +261,9 @@
|
||||
100% { transform: scale(1); opacity: 1; }
|
||||
}
|
||||
|
||||
/* Error state */
|
||||
/* ── Error State ────────────────────────────────────────────────── */
|
||||
/* When a service fails to start within the timeout (60s), show
|
||||
red text, an error message, and a retry button. */
|
||||
.loading-text.error {
|
||||
background: linear-gradient(135deg, #ef4444, #f97316);
|
||||
-webkit-background-clip: text;
|
||||
@@ -294,7 +302,12 @@
|
||||
display: inline-block;
|
||||
}
|
||||
|
||||
/* ── Setup Wizard ──────────────────────────────────────────── */
|
||||
/* ── Setup Wizard ──────────────────────────────────────────────── */
|
||||
/* Full-screen 4-step wizard for first-time setup:
|
||||
1. Environment check (Ollama, model, browser, Facebook login)
|
||||
2. Browser auto-detection & selection
|
||||
3. Ollama model pull with progress bar
|
||||
4. Done — close wizard, start normal loading flow */
|
||||
.setup-wizard {
|
||||
display: none;
|
||||
position: fixed;
|
||||
@@ -416,7 +429,7 @@
|
||||
</head>
|
||||
<body>
|
||||
|
||||
<!-- Robot -->
|
||||
<!-- ── Robot Mascot ────────────────────────────────────────────── -->
|
||||
<div class="robot">
|
||||
<div class="robot-antenna"></div>
|
||||
<div class="robot-head">
|
||||
@@ -431,7 +444,7 @@
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Services -->
|
||||
<!-- ── Service Status Indicators ───────────────────────────────── -->
|
||||
<div class="services">
|
||||
<div class="service" id="svc-ai">
|
||||
<span class="service-name">AI Server</span>
|
||||
@@ -464,26 +477,27 @@
|
||||
|
||||
<div class="loading-text" id="loading-text">Loading...</div>
|
||||
|
||||
<!-- Launch gear (inline, below everything) -->
|
||||
<!-- ── Launch Gear ──────────────────────────────────────────── -->
|
||||
<div class="launch-overlay" id="launch-overlay">
|
||||
<div class="launch-gear"></div>
|
||||
<div class="launch-text">🚀 Launching gear!</div>
|
||||
<div class="launch-text">Launching gear!</div>
|
||||
</div>
|
||||
|
||||
<div class="error-msg" id="error-msg"></div>
|
||||
<button class="retry-btn" id="retry-btn" onclick="location.reload()">Try Again</button>
|
||||
|
||||
<!-- ── Setup Wizard ── -->
|
||||
<!-- ══════════════════════════════════════════════════════════════
|
||||
Setup Wizard (4 steps, shown only on first run)
|
||||
══════════════════════════════════════════════════════════════ -->
|
||||
<div class="setup-wizard" id="setup-wizard">
|
||||
<div class="setup-steps">
|
||||
<div class="setup-dot current" id="sdot-1"></div>
|
||||
<div class="setup-dot" id="sdot-2"></div>
|
||||
<div class="setup-dot" id="sdot-3"></div>
|
||||
<div class="setup-dot" id="sdot-4"></div>
|
||||
<div class="setup-dot" id="sdot-5"></div>
|
||||
</div>
|
||||
|
||||
<!-- Step 1: Welcome -->
|
||||
<!-- Step 1: Environment check -->
|
||||
<div class="setup-step active" id="sstep-1">
|
||||
<div class="robot" style="margin-bottom:20px">
|
||||
<div class="robot-antenna"></div>
|
||||
@@ -504,43 +518,22 @@
|
||||
<button class="setup-btn" id="welcome-next" onclick="goStep(2)">Next →</button>
|
||||
</div>
|
||||
|
||||
<!-- Step 2: Browser Profile -->
|
||||
<!-- Step 2: Browser auto-detection & selection -->
|
||||
<div class="setup-step" id="sstep-2">
|
||||
<div class="setup-title">Browser Profile</div>
|
||||
<div class="setup-subtitle">We need a Firefox or Chrome profile with Facebook logged in</div>
|
||||
<div class="setup-card">
|
||||
<div class="check-row">
|
||||
<span class="check-icon" id="prof-auto-icon">⏳</span>
|
||||
<span class="check-label">Auto-detected</span>
|
||||
<span class="check-value" id="prof-auto-val">scanning...</span>
|
||||
</div>
|
||||
<div style="margin-top:14px;font-size:13px;opacity:0.5;margin-bottom:8px">Or enter the path manually:</div>
|
||||
<input class="setup-input" id="prof-input" placeholder="C:\Users\You\AppData\Roaming\Mozilla\Firefox\Profiles\..." oninput="onProfileInput()">
|
||||
<div style="margin-top:6px;font-size:12px;color:#22d3ee" id="prof-feedback"></div>
|
||||
<div class="setup-title">Detect your browser</div>
|
||||
<div class="setup-subtitle">We found these browsers with Facebook login. Click one to select it.</div>
|
||||
<div class="setup-card" id="browser-list">
|
||||
<div id="browser-scanning" style="text-align:center;padding:16px;opacity:0.6">🔍 Scanning for browsers...</div>
|
||||
<div id="browser-results" style="display:none"></div>
|
||||
</div>
|
||||
<div class="setup-btns">
|
||||
<button class="setup-btn outline" onclick="goStep(1)">← Back</button>
|
||||
<button class="setup-btn" id="prof-next" onclick="saveProfile()" disabled>Next →</button>
|
||||
<button class="setup-btn" id="browser-confirm" onclick="confirmBrowser()" disabled>Confirm →</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Step 3: Facebook Login -->
|
||||
<!-- Step 3: Ollama Model Pull -->
|
||||
<div class="setup-step" id="sstep-3">
|
||||
<div class="setup-title">Facebook Login</div>
|
||||
<div class="setup-subtitle">Make sure you're logged into Facebook in your browser</div>
|
||||
<div class="setup-card" style="text-align:center">
|
||||
<div style="font-size:48px;margin-bottom:12px" id="fb-icon">🔍</div>
|
||||
<div style="font-size:14px;opacity:0.7" id="fb-status">Click "Check Login" to verify</div>
|
||||
<div class="progress-bar" id="fb-progress" style="display:none;margin-top:16px"><div class="progress-fill" id="fb-progress-fill"></div></div>
|
||||
</div>
|
||||
<div class="setup-btns">
|
||||
<button class="setup-btn outline" onclick="goStep(2)">← Back</button>
|
||||
<button class="setup-btn" id="fb-check-btn" onclick="checkFacebookLogin()">Check Login</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Step 4: Ollama Model -->
|
||||
<div class="setup-step" id="sstep-4">
|
||||
<div class="setup-title">AI Model</div>
|
||||
<div class="setup-subtitle">We need to pull the AI model for the scraper to work</div>
|
||||
<div class="setup-card" style="text-align:center">
|
||||
@@ -554,8 +547,8 @@
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Step 5: Done -->
|
||||
<div class="setup-step" id="sstep-5">
|
||||
<!-- Step 4: Done → Launch -->
|
||||
<div class="setup-step" id="sstep-4">
|
||||
<div style="font-size:64px;margin-bottom:16px">🎉</div>
|
||||
<div class="setup-title">All set!</div>
|
||||
<div class="setup-subtitle">Your environment is configured. We'll now start all services.</div>
|
||||
@@ -567,13 +560,18 @@
|
||||
</div>
|
||||
|
||||
<script>
|
||||
// ══════════════════════════════════════════════════════════════════
|
||||
// Client-side JavaScript
|
||||
// ══════════════════════════════════════════════════════════════════
|
||||
|
||||
// ── State ──
|
||||
let setupData = null;
|
||||
let currentStep = 1;
|
||||
const TOTAL_STEPS = 5;
|
||||
const TOTAL_STEPS = 4;
|
||||
let selectedBrowser = "";
|
||||
|
||||
// ── Normal loading state ──
|
||||
const MAX_ATTEMPTS = 30;
|
||||
const MAX_ATTEMPTS = 30; // 30 attempts × 2s = 60s timeout
|
||||
let attempts = 0;
|
||||
const CHECKS = [
|
||||
{ id: 'ai', key: 'ai', ready: false, failed: false },
|
||||
@@ -582,6 +580,7 @@ const CHECKS = [
|
||||
];
|
||||
const MSGS = { ai: 'AI Ready', scraper: 'Scraper Ready', frontend: 'Frontend Ready' };
|
||||
const NAMES = { ai: 'AI Server', scraper: 'Scraper', frontend: 'Frontend' };
|
||||
const BROWSE_ICONS = { firefox: '🦊', opera: '🔵', chrome: '🌐', edge: '🔷' };
|
||||
|
||||
// ── Step navigation ──
|
||||
function goStep(n) {
|
||||
@@ -593,21 +592,32 @@ function goStep(n) {
|
||||
el.classList.toggle('current', i + 1 === n);
|
||||
el.classList.toggle('done', i + 1 < n);
|
||||
});
|
||||
if (n === 2) renderBrowserCards();
|
||||
}
|
||||
|
||||
// ── Step 1: Welcome / Env check ──
|
||||
// ── Step 1: Welcome + Environment check ──
|
||||
// Fetches /setup/status from the AI server, displays checkmarks/crosses
|
||||
// for each environment requirement. If first_run is false, skips the
|
||||
// wizard entirely and goes straight to the normal loading flow.
|
||||
async function checkEnv() {
|
||||
try {
|
||||
const res = await fetch('/setup/status');
|
||||
setupData = await res.json();
|
||||
} catch {
|
||||
setupData = { first_run: true, ollama_running: false, model_available: false, profile_detected: false, facebook_logged_in: false };
|
||||
setupData = { first_run: true, ollama_running: false, model_available: false, facebook_logged_in: false, browsers: {} };
|
||||
}
|
||||
|
||||
setEnvStatus('ollama', setupData.ollama_running, setupData.ollama_running ? 'Running' : 'Not running');
|
||||
setEnvStatus('model', setupData.model_available, setupData.model_available ? `${setupData.model_name} ✓` : 'Not pulled');
|
||||
setEnvStatus('profile', setupData.profile_detected, setupData.profile_path || 'Not found');
|
||||
setEnvStatus('fb', setupData.facebook_logged_in, setupData.facebook_logged_in ? 'Logged in' : 'Not checked');
|
||||
|
||||
// Browser summary
|
||||
const detected = Object.entries(setupData.browsers || {}).filter(([, v]) => v.path)
|
||||
const loggedIn = detected.filter(([, v]) => v.logged_in)
|
||||
const browserSummary = loggedIn.length
|
||||
? loggedIn.map(([b]) => `${b.charAt(0).toUpperCase()+b.slice(1)} ✓`).join(', ')
|
||||
: detected.length ? 'Found but not logged into Facebook' : 'None detected'
|
||||
setEnvStatus('profile', detected.length > 0, browserSummary)
|
||||
setEnvStatus('fb', setupData.facebook_logged_in, setupData.facebook_logged_in ? 'Logged in' : loggedIn.length ? '' : 'Not checked')
|
||||
|
||||
if (!setupData.first_run) {
|
||||
document.getElementById('setup-wizard').classList.remove('active');
|
||||
@@ -615,8 +625,6 @@ async function checkEnv() {
|
||||
return;
|
||||
}
|
||||
document.getElementById('setup-wizard').classList.add('active');
|
||||
// Pre-fill step 2 fields
|
||||
updateProfileUI();
|
||||
}
|
||||
|
||||
function setEnvStatus(id, ok, label) {
|
||||
@@ -624,94 +632,85 @@ function setEnvStatus(id, ok, label) {
|
||||
document.getElementById('env-' + id + '-val').textContent = label;
|
||||
}
|
||||
|
||||
// ── Step 2: Browser Profile ──
|
||||
function updateProfileUI() {
|
||||
const icon = document.getElementById('prof-auto-icon');
|
||||
const val = document.getElementById('prof-auto-val');
|
||||
const input = document.getElementById('prof-input');
|
||||
if (setupData.profile_path) {
|
||||
icon.textContent = '✅';
|
||||
val.textContent = setupData.profile_path;
|
||||
input.value = setupData.profile_path;
|
||||
document.getElementById('prof-next').disabled = false;
|
||||
} else {
|
||||
icon.textContent = '❌';
|
||||
val.textContent = 'None detected';
|
||||
// ── Step 2: Browser selection ──
|
||||
// Renders clickable cards for each detected browser.
|
||||
// Auto-selects the first one that has Facebook login.
|
||||
// No manual path input needed — all detection is automatic.
|
||||
function renderBrowserCards() {
|
||||
const scanning = document.getElementById('browser-scanning');
|
||||
const results = document.getElementById('browser-results');
|
||||
const browsers = setupData.browsers || {};
|
||||
const detected = Object.entries(browsers).filter(([, v]) => v.path)
|
||||
|
||||
if (detected.length === 0) {
|
||||
scanning.textContent = '❌ No browsers with Facebook detected. Log in on your browser and click Retry.';
|
||||
const retryBtn = document.createElement('button');
|
||||
retryBtn.className = 'setup-btn';
|
||||
retryBtn.textContent = 'Retry Detection';
|
||||
retryBtn.onclick = () => location.reload();
|
||||
scanning.after(retryBtn);
|
||||
return;
|
||||
}
|
||||
|
||||
scanning.style.display = 'none';
|
||||
results.style.display = 'block';
|
||||
|
||||
// Auto-select first logged-in browser, or first detected
|
||||
const firstLoggedIn = detected.find(([, v]) => v.logged_in)
|
||||
selectedBrowser = firstLoggedIn ? firstLoggedIn[0] : detected[0][0]
|
||||
|
||||
results.innerHTML = detected.map(([name, info]) => {
|
||||
const isSel = name === selectedBrowser
|
||||
const icon = BROWSE_ICONS[name] || '🌐'
|
||||
const status = info.logged_in ? '✅ Logged in' : info.path ? '⚠️ Not logged in' : '❌ Not found'
|
||||
const pathShort = info.path ? info.path.length > 45 ? '...' + info.path.slice(-42) : info.path : ''
|
||||
return `<div class="browser-card ${isSel ? 'selected' : ''}" data-browser="${name}" onclick="pickBrowser('${name}')" style="display:flex;align-items:center;gap:12px;padding:12px;border:2px solid ${isSel ? '#6366f1' : 'rgba(255,255,255,0.08)'};border-radius:10px;margin-bottom:8px;cursor:pointer;transition:all 0.2s">
|
||||
<span style="font-size:24px">${icon}</span>
|
||||
<div style="flex:1"><strong style="text-transform:capitalize">${name}</strong><div style="font-size:12px;opacity:0.5;overflow:hidden;text-overflow:ellipsis;white-space:nowrap">${pathShort}</div></div>
|
||||
<span style="font-size:13px">${status}</span>
|
||||
</div>`
|
||||
}).join('')
|
||||
|
||||
document.getElementById('browser-confirm').disabled = false
|
||||
}
|
||||
|
||||
function onProfileInput() {
|
||||
const val = document.getElementById('prof-input').value.trim();
|
||||
const fb = document.getElementById('prof-feedback');
|
||||
const btn = document.getElementById('prof-next');
|
||||
if (!val) { fb.textContent = ''; btn.disabled = true; return }
|
||||
fb.textContent = 'Entered: ' + val;
|
||||
btn.disabled = false;
|
||||
function pickBrowser(name) {
|
||||
selectedBrowser = name
|
||||
document.querySelectorAll('.browser-card').forEach(el => {
|
||||
el.style.borderColor = el.dataset.browser === name ? '#6366f1' : 'rgba(255,255,255,0.08)'
|
||||
})
|
||||
}
|
||||
|
||||
async function saveProfile() {
|
||||
const path = document.getElementById('prof-input').value.trim();
|
||||
if (!path) return;
|
||||
const btn = document.getElementById('prof-next');
|
||||
btn.textContent = 'Saving...';
|
||||
btn.disabled = true;
|
||||
// Saves the selected browser to .env.local via the AI server's /setup/profile endpoint
|
||||
async function confirmBrowser() {
|
||||
if (!selectedBrowser) return
|
||||
const btn = document.getElementById('browser-confirm')
|
||||
btn.textContent = 'Saving...'
|
||||
btn.disabled = true
|
||||
|
||||
const path = setupData.browsers[selectedBrowser].path
|
||||
try {
|
||||
const res = await fetch('/setup/profile', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ path }) });
|
||||
const data = await res.json();
|
||||
const res = await fetch('/setup/profile', {
|
||||
method: 'POST', headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ browser: selectedBrowser, path })
|
||||
})
|
||||
const data = await res.json()
|
||||
if (data.success) {
|
||||
setupData.profile_detected = true;
|
||||
setupData.profile_path = path;
|
||||
goStep(3);
|
||||
setupData.selected_browser = selectedBrowser
|
||||
goStep(3)
|
||||
} else {
|
||||
document.getElementById('prof-feedback').textContent = 'Error: ' + (data.error || 'failed');
|
||||
}
|
||||
} catch { document.getElementById('prof-feedback').textContent = 'Error: could not save'; }
|
||||
btn.textContent = 'Next →';
|
||||
btn.disabled = false;
|
||||
}
|
||||
|
||||
// ── Step 3: Facebook Login ──
|
||||
async function checkFacebookLogin() {
|
||||
const btn = document.getElementById('fb-check-btn');
|
||||
const status = document.getElementById('fb-status');
|
||||
const icon = document.getElementById('fb-icon');
|
||||
const prog = document.getElementById('fb-progress');
|
||||
const fill = document.getElementById('fb-progress-fill');
|
||||
btn.disabled = true;
|
||||
btn.textContent = 'Checking...';
|
||||
prog.style.display = 'block';
|
||||
fill.style.width = '50%';
|
||||
status.textContent = 'Opening Facebook...';
|
||||
|
||||
try {
|
||||
const path = setupData.profile_path || '';
|
||||
const res = await fetch('/setup/check-login', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ profile_path: path }) });
|
||||
const data = await res.json();
|
||||
if (data.logged_in) {
|
||||
fill.style.width = '100%';
|
||||
icon.textContent = '✅';
|
||||
status.textContent = 'You are logged into Facebook!';
|
||||
status.className = 'status-ok';
|
||||
setupData.facebook_logged_in = true;
|
||||
setTimeout(() => goStep(4), 1000);
|
||||
} else {
|
||||
fill.style.width = '100%';
|
||||
icon.textContent = '❌';
|
||||
status.textContent = 'Not logged in. Log into Facebook in your browser and try again.';
|
||||
status.className = 'status-err';
|
||||
btn.textContent = 'Retry';
|
||||
btn.disabled = false;
|
||||
alert('Failed to save: ' + (data.error || 'unknown'))
|
||||
btn.disabled = false
|
||||
}
|
||||
} catch {
|
||||
icon.textContent = '❌';
|
||||
status.textContent = 'Could not reach the scraper service';
|
||||
status.className = 'status-err';
|
||||
btn.textContent = 'Retry';
|
||||
btn.disabled = false;
|
||||
alert('Could not save profile')
|
||||
btn.disabled = false
|
||||
}
|
||||
btn.textContent = 'Confirm →'
|
||||
}
|
||||
|
||||
// ── Step 4: Ollama Model ──
|
||||
// ── Step 3: Ollama Model Pull ──
|
||||
// Kicks off an "ollama pull" via the AI server and polls for progress.
|
||||
let pullPolling = false;
|
||||
async function pullModel() {
|
||||
if (pullPolling) return;
|
||||
@@ -720,7 +719,6 @@ async function pullModel() {
|
||||
const status = document.getElementById('model-status');
|
||||
btn.disabled = true;
|
||||
btn.textContent = 'Starting...';
|
||||
|
||||
try {
|
||||
const res = await fetch('/setup/ollama/pull', { method: 'POST' });
|
||||
const data = await res.json();
|
||||
@@ -745,12 +743,12 @@ async function pollPullProgress() {
|
||||
const data = await res.json();
|
||||
fill.style.width = data.progress + '%';
|
||||
if (data.status === 'done') {
|
||||
status.textContent = '✅ Model downloaded successfully!';
|
||||
status.textContent = '✅ Model downloaded!';
|
||||
status.className = 'status-ok';
|
||||
btn.textContent = 'Done';
|
||||
setupData.model_available = true;
|
||||
pullPolling = false;
|
||||
setTimeout(() => goStep(5), 1500);
|
||||
setTimeout(() => goStep(4), 1500);
|
||||
return;
|
||||
}
|
||||
if (data.status === 'failed') {
|
||||
@@ -769,25 +767,26 @@ async function pollPullProgress() {
|
||||
}
|
||||
}
|
||||
|
||||
// ── Step 5: Finish ──
|
||||
// ── Step 4: Finish Setup ──
|
||||
// Closes the wizard and starts the normal loading flow.
|
||||
function finishSetup() {
|
||||
document.getElementById('setup-wizard').classList.remove('active');
|
||||
startNormalFlow();
|
||||
}
|
||||
|
||||
// ── Normal loading flow ──
|
||||
function startNormalFlow() {
|
||||
poll();
|
||||
}
|
||||
// ── Normal Loading Flow ──
|
||||
// Polls /status every 2 seconds for up to 60 seconds.
|
||||
// Tracks each service's state (ready/waiting/failed).
|
||||
// All ready → show launch gear, redirect to /login after 5s.
|
||||
// Any failed → show error message + retry button.
|
||||
function startNormalFlow() { poll() }
|
||||
|
||||
async function checkAllServices() {
|
||||
try {
|
||||
const res = await fetch('/status', { cache: 'no-store' });
|
||||
const data = await res.json();
|
||||
for (const svc of CHECKS) svc.ready = data[svc.key] === true;
|
||||
} catch {
|
||||
for (const svc of CHECKS) svc.ready = false;
|
||||
}
|
||||
} catch { for (const svc of CHECKS) svc.ready = false }
|
||||
}
|
||||
|
||||
function updateUI() {
|
||||
@@ -829,6 +828,7 @@ async function poll() {
|
||||
}
|
||||
|
||||
// ── Boot ──
|
||||
// Entry point: check environment → shows wizard or starts normal flow
|
||||
checkEnv();
|
||||
</script>
|
||||
</body>
|
||||
|
||||
Reference in New Issue
Block a user