added Chnages on

ai-server/index.mjs

@@ -1,21 +1,62 @@
+// ── CRM AI Server ──────────────────────────────────────────────────
+// Provides:
+//   - Chat API (POST /ai/chat) — routes user messages to Ollama for sales coaching
+//   - Setup wizard endpoints (GET /setup/status, POST /setup/profile, etc.)
+//   - Combined /status endpoint for splash page health polling
+//   - Configuration routes (GET/POST /ai/instructions, GET /ai/jobs)
+//   - Model pull support (POST /setup/ollama/pull)
+//
+// This is a zero-dependency Node.js HTTP server (no Express needed).
+
 import http from "node:http"
 import fs from "node:fs"
 import path from "node:path"
+import { spawn } from "node:child_process"
 import { fileURLToPath } from "node:url"
 
 const __dirname = path.dirname(fileURLToPath(import.meta.url))
 const ROOT = path.resolve(__dirname, "..")
 
+// ── Load .env.local ──────────────────────────────────────────────
+// Reads key=value pairs and sets them as process.env so they're
+// available throughout the server. Ignores comments and blank lines.
+// Values with matching quotes are unquoted.
+try {
+  const envPath = path.join(ROOT, ".env.local")
+  const envContent = fs.readFileSync(envPath, "utf-8")
+  for (const line of envContent.split("\n")) {
+    const trimmed = line.trim()
+    if (!trimmed || trimmed.startsWith("#")) continue
+    const eqIdx = trimmed.indexOf("=")
+    if (eqIdx === -1) continue
+    const k = trimmed.substring(0, eqIdx).trim()
+    let v = trimmed.substring(eqIdx + 1).trim()
+    if ((v.startsWith('"') && v.endsWith('"')) || (v.startsWith("'") && v.endsWith("'"))) v = v.slice(1, -1)
+    if (!process.env[k]) process.env[k] = v
+  }
+  console.log("Loaded .env.local")
+} catch {
+  // .env.local may not exist (first run), which is fine
+}
+
 // ── Config from env ─────────────────────────────────────────────
 const PORT = parseInt(process.env.AI_PORT || "3001", 10)
 const HOST = process.env.AI_HOST || "0.0.0.0"
 const OLLAMA_URL = process.env.OLLAMA_BASE_URL || "http://localhost:11434"
-const MODEL = process.env.AI_MODEL || "sam860/dolphin3-llama3.2:3b"
+const MODEL = process.env.AI_MODEL || "llama3.2:3b"
 const DATABASE_URL = process.env.DATABASE_URL
 const JOBS_PATH = process.env.JOBS_PATH || path.join(ROOT, "data", "ai", "jobs.jsonl")
 const AI_MD_PATH = process.env.AI_MD_PATH || path.join(ROOT, "data", "ai", "ai.md")
 
+// ── Setup state ──────────────────────────────────────────────────
+// Tracks the Ollama model pull process so the setup wizard can
+// poll for download progress.
+let pullProcess = null
+let pullProgress = { status: "idle", progress: 0, message: "" }
+
 // ── Job loading ─────────────────────────────────────────────────
+// Loads job categories from a JSONL file (one JSON object per line).
+// Used as context for the AI sales coach chat responses.
 function loadJobs() {
   try {
     const content = fs.readFileSync(JOBS_PATH, "utf-8")
@@ -40,6 +81,8 @@ function loadJobs() {
 }
 
 // ── ai.md management ────────────────────────────────────────────
+// ai.md is a Markdown file containing system instructions for the AI.
+// It can be read, written, or appended to via the API.
 function readInstructions() {
   try {
     return fs.readFileSync(AI_MD_PATH, "utf-8")
@@ -54,6 +97,7 @@ function writeInstructions(content) {
 }
 
 function appendToImprovementLog(entry) {
+  // Adds a timestamped entry to the ## Improvement Log section of ai.md
   const current = readInstructions()
   const timestamp = new Date().toISOString().replace("T", " ").substring(0, 16)
   const logEntry = `\n- ${timestamp} — ${entry}`
@@ -77,7 +121,60 @@ function appendToImprovementLog(entry) {
 }
 
 // ── Chat handler ────────────────────────────────────────────────
+// scrapeFacebook() calls the scraper service (port 3008) to get leads.
+// handleChat() processes user messages — triggers lead scraping when
+// the user asks for "leads" or "listings", otherwise routes to Ollama
+// for AI-powered sales coaching.
+async function scrapeFacebook() {
+  const profilePath = process.env.FX_PROFILE || ""
+  const urlPath = `/scrape/facebook?force=true${profilePath ? `&profile_path=${encodeURIComponent(profilePath)}` : ""}`
+  try {
+    const body = await new Promise((resolve, reject) => {
+      const req = http.request({ hostname: "127.0.0.1", port: 3008, path: urlPath, method: "POST", timeout: 360000 }, (res) => {
+        let data = ""
+        res.on("data", (c) => data += c)
+        res.on("end", () => resolve(data))
+        res.on("error", reject)
+      })
+      req.on("timeout", () => { req.destroy(); reject(new Error("timeout")) })
+      req.on("error", reject)
+      req.end()
+    })
+    const data = JSON.parse(body)
+    return data
+  } catch (e) {
+    return null
+  }
+}
+
+function formatLeads(leads) {
+  if (!leads || leads.length === 0) return "No leads found from the latest scrape."
+  let output = `**${leads.length} leads found:**\n\n`
+  for (let i = 0; i < leads.length; i++) {
+    const l = leads[i]
+    output += `${i + 1}. ${l.title || "No title"}\n`
+    if (l.author) output += `   Author: ${l.author}\n`
+    if (l.date) output += `   Date: ${l.date}\n`
+    if (l.url) output += `   URL: ${l.url}\n`
+    output += "\n"
+  }
+  return output.trim()
+}
+
 async function handleChat(userMessage, userId, userRole) {
+  // If the user asks for leads, trigger the scraper
+  const lowerMsg = userMessage.toLowerCase()
+  const triggerWords = ["lists", "listings", "leads", "recent leads", "pull leads", "show me leads", "show listings"]
+
+  if (triggerWords.some(w => lowerMsg.includes(w))) {
+    const result = await scrapeFacebook()
+    if (result && result.success) {
+      return formatLeads(result.leads)
+    }
+    return "Scraper returned no results or encountered an error. Try again later."
+  }
+
+  // Otherwise, build a system prompt with job context and send to Ollama
   const jobs = loadedJobs
   const instructions = readInstructions()
 
@@ -117,7 +214,7 @@ Provide concise, actionable sales advice. When asked about a specific job catego
   const data = await ollamaRes.json()
   const responseText = data.message?.content || ""
 
-  // Try to persist to DB (best-effort)
+  // Persist conversation to PostgreSQL (best-effort — table may not exist yet)
   try {
     if (pgPool && userId) {
       await pgPool.query(
@@ -133,6 +230,8 @@ Provide concise, actionable sales advice. When asked about a specific job catego
 }
 
 // ── PG pool (lazy init) ────────────────────────────────────────
+// PostgreSQL connection pool for storing conversation history.
+// Lazy-initialized so the server starts even without a DB.
 let pgPool = null
 async function initPg() {
   if (!DATABASE_URL) return
@@ -174,8 +273,9 @@ function parseURL(req) {
   return { pathname: url.pathname, searchParams: url.searchParams }
 }
 
+// ── HTTP Server ─────────────────────────────────────────────────
 const server = http.createServer(async (req, res) => {
-  // CORS
+  // CORS headers — allow the Next.js frontend (port 3006) to call us
   res.setHeader("Access-Control-Allow-Origin", "*")
   res.setHeader("Access-Control-Allow-Methods", "GET, POST, OPTIONS")
   res.setHeader("Access-Control-Allow-Headers", "Content-Type")
@@ -189,23 +289,236 @@ const server = http.createServer(async (req, res) => {
   const { pathname } = parseURL(req)
 
   try {
+    // GET /splash — loading screen
+    if (req.method === "GET" && pathname === "/splash") {
+      const splashPath = path.join(ROOT, "splash.html")
+      if (fs.existsSync(splashPath)) {
+        const html = fs.readFileSync(splashPath, "utf-8")
+        res.writeHead(200, { "Content-Type": "text/html" })
+        res.end(html)
+        return
+      }
+      sendJSON(res, 200, { status: "ok" })
+      return
+    }
+
     // GET /health
     if (req.method === "GET" && pathname === "/health") {
       return sendJSON(res, 200, { status: "ok", model: MODEL })
     }
 
-    // GET /ai/jobs
+    // GET /status — combined health of all services
+    // Used by the splash page to check if AI, Scraper, and Frontend are ready.
+    // Polls each service internally to avoid cross-origin CORS issues.
+    if (req.method === "GET" && pathname === "/status") {
+      const { default: http } = await import("http")
+      const results = { ai: true }
+      // Check scraper (port 3008)
+      try {
+        await new Promise((resolve, reject) => {
+          const r = http.get("http://127.0.0.1:3008/health", { timeout: 3000 }, (res) => { res.resume(); resolve() })
+          r.on("error", reject)
+        })
+        results.scraper = true
+      } catch { results.scraper = false }
+      // Check frontend (port 3006)
+      try {
+        await new Promise((resolve, reject) => {
+          const r = http.get("http://127.0.0.1:3006", { timeout: 3000 }, (res) => { res.resume(); resolve() })
+          r.on("error", reject)
+        })
+        results.frontend = true
+      } catch { results.frontend = false }
+      return sendJSON(res, 200, results)
+    }
+
+    // ── Setup endpoints ─────────────────────────────────────────
+
+    // GET /setup/status — check environment
+    // Called by the splash page on boot. Returns info about:
+    //   - Ollama availability
+    //   - Model presence
+    //   - Detected browsers with login status
+    //   - Whether this is a first run (wizard needed)
+    if (req.method === "GET" && pathname === "/setup/status") {
+      const envExists = fs.existsSync(path.join(ROOT, ".env.local"))
+
+      // Ollama check
+      let ollamaRunning = false
+      try {
+        await fetch(`${OLLAMA_URL}/api/tags`, { signal: AbortSignal.timeout(3000) })
+        ollamaRunning = true
+      } catch {}
+
+      // Model check
+      let modelAvailable = false
+      if (ollamaRunning) {
+        try {
+          const r = await fetch(`${OLLAMA_URL}/api/show`, {
+            method: "POST", body: JSON.stringify({ name: MODEL }),
+            signal: AbortSignal.timeout(5000),
+          })
+          modelAvailable = r.ok
+        } catch {}
+      }
+
+      // Detect all browsers via scraper
+      let browsers = { firefox: { path: null }, opera: { path: null }, chrome: { path: null }, edge: { path: null } }
+      let facebookLoggedIn = false
+      let selectedBrowser = process.env.SELECTED_BROWSER || ""
+
+      try {
+        await fetch("http://127.0.0.1:3008/health", { signal: AbortSignal.timeout(2000) })
+        const profiles = await (await fetch("http://127.0.0.1:3008/setup/profile", { signal: AbortSignal.timeout(5000) })).json()
+        for (const [b, p] of Object.entries(profiles)) {
+          if (p) browsers[b] = { path: p }
+        }
+        // Check login for the selected browser first, then try all
+        const detectedList = Object.entries(browsers).filter(([, v]) => v.path)
+        for (const [b, v] of detectedList) {
+          try {
+            const r = await fetch("http://127.0.0.1:3008/setup/check-login", {
+              method: "POST", headers: { "Content-Type": "application/json" },
+              body: JSON.stringify({ browser: b, profile_path: v.path }),
+              signal: AbortSignal.timeout(20000),
+            })
+            if (r.ok) {
+              const d = await r.json()
+              browsers[b].logged_in = d.logged_in === true
+              if (d.logged_in && !facebookLoggedIn) {
+                facebookLoggedIn = true
+                if (!selectedBrowser) selectedBrowser = b
+              }
+            }
+          } catch {}
+        }
+      } catch {}
+
+      const anyDetected = Object.values(browsers).some(v => v.path)
+      // first_run = any setup step is incomplete
+      const firstRun = !envExists || !ollamaRunning || !anyDetected || !facebookLoggedIn || !modelAvailable
+
+      return sendJSON(res, 200, {
+        first_run: firstRun,
+        env_exists: envExists,
+        ollama_running: ollamaRunning,
+        model_available: modelAvailable,
+        model_name: MODEL,
+        selected_browser: selectedBrowser,
+        browsers,
+        facebook_logged_in: facebookLoggedIn,
+      })
+    }
+
+    // POST /setup/profile — save selected browser + path to .env.local
+    // Called by the setup wizard when the user confirms their browser choice.
+    // Writes SELECTED_BROWSER and the matching profile env var to .env.local.
+    if (req.method === "POST" && pathname === "/setup/profile") {
+      const body = await parseBody(req)
+      const browserName = (body.browser || "").trim().toLowerCase()
+      const profilePath = (body.path || "").trim()
+      if (!browserName || !["firefox", "opera", "chrome", "edge"].includes(browserName))
+        return sendJSON(res, 400, { error: "Valid browser required (firefox/opera/chrome/edge)" })
+      if (!profilePath)
+        return sendJSON(res, 400, { error: "Path required" })
+
+      const envKey = browserName === "firefox" ? "FX_PROFILE"
+        : browserName === "opera" ? "OPERA_PROFILE"
+        : browserName === "edge" ? "EDGE_PROFILE"
+        : "CHROME_PROFILE"
+
+      const envPath = path.join(ROOT, ".env.local")
+      let content = ""
+      try { content = fs.readFileSync(envPath, "utf-8") } catch {}
+      let lines = content.split("\n")
+      // Update or add SELECTED_BROWSER
+      let foundSel = false
+      for (let i = 0; i < lines.length; i++) {
+        if (lines[i].trim().startsWith("SELECTED_BROWSER=")) {
+          lines[i] = `SELECTED_BROWSER=${browserName}`
+          foundSel = true
+          break
+        }
+      }
+      if (!foundSel) lines.push(`SELECTED_BROWSER=${browserName}`)
+      // Update or add browser profile
+      let foundProf = false
+      for (let i = 0; i < lines.length; i++) {
+        if (lines[i].trim().startsWith(`${envKey}=`)) {
+          lines[i] = `${envKey}=${profilePath}`
+          foundProf = true
+          break
+        }
+      }
+      if (!foundProf) lines.push(`${envKey}=${profilePath}`)
+      fs.writeFileSync(envPath, lines.join("\n"), "utf-8")
+      process.env.SELECTED_BROWSER = browserName
+      process.env[envKey] = profilePath
+      return sendJSON(res, 200, { success: true, browser: browserName, path: profilePath })
+    }
+
+    // POST /setup/check-login — proxy to scraper, accepts browser + profile_path
+    // The splash page calls this (via the AI server) to verify Facebook login status.
+    if (req.method === "POST" && pathname === "/setup/check-login") {
+      const body = await parseBody(req)
+      const browserName = (body.browser || "").trim().toLowerCase() || process.env.SELECTED_BROWSER || ""
+      const profilePath = (body.profile_path || "").trim()
+
+      if (!profilePath) return sendJSON(res, 200, { logged_in: false, reason: "no_profile" })
+      try {
+        const r = await fetch("http://127.0.0.1:3008/setup/check-login", {
+          method: "POST", headers: { "Content-Type": "application/json" },
+          body: JSON.stringify({ browser: browserName, profile_path: profilePath }),
+          signal: AbortSignal.timeout(20000),
+        })
+        if (r.ok) { const d = await r.json(); return sendJSON(res, 200, d) }
+      } catch {}
+      return sendJSON(res, 200, { logged_in: false, reason: "scraper_unavailable" })
+    }
+
+    // POST /setup/ollama/pull — start pulling the model
+    // Spawns "ollama pull" as a child process. The setup wizard polls
+    // the progress endpoint to show a download progress bar.
+    if (req.method === "POST" && pathname === "/setup/ollama/pull") {
+      if (pullProcess) return sendJSON(res, 200, { status: "already_running" })
+      pullProgress = { status: "downloading", progress: 0, message: "Starting..." }
+      const isWin = process.platform === "win32"
+      const cmd = isWin ? "ollama.exe" : "ollama"
+      pullProcess = spawn(cmd, ["pull", MODEL], { stdio: ["ignore", "pipe", "pipe"] })
+
+      pullProcess.stdout.on("data", (data) => {
+        const text = data.toString()
+        pullProgress.message = text.trim()
+        // Extract percentage from patterns like "pulling xxxx... 45%"
+        const m = text.match(/(\d+)%/)
+        if (m) pullProgress.progress = parseInt(m[1], 10)
+      })
+      pullProcess.on("close", (code) => {
+        pullProcess = null
+        pullProgress.status = code === 0 ? "done" : "failed"
+        if (code === 0) pullProgress.progress = 100
+      })
+      return sendJSON(res, 200, { status: "started" })
+    }
+
+    // GET /setup/ollama/pull/progress
+    // Returns current download progress for the setup wizard.
+    if (req.method === "GET" && pathname === "/setup/ollama/pull/progress") {
+      return sendJSON(res, 200, pullProgress)
+    }
+
+    // GET /ai/jobs — return loaded job categories
     if (req.method === "GET" && pathname === "/ai/jobs") {
       return sendJSON(res, 200, { jobs: loadedJobs })
     }
 
-    // GET /ai/instructions
+    // GET /ai/instructions — return current ai.md content
     if (req.method === "GET" && pathname === "/ai/instructions") {
       const instructions = readInstructions()
       return sendJSON(res, 200, { success: true, instructions })
     }
 
-    // POST /ai/instructions
+    // POST /ai/instructions — update ai.md or append improvement log entry
     if (req.method === "POST" && pathname === "/ai/instructions") {
       const body = await parseBody(req)
       if (body.content) {
@@ -219,9 +532,27 @@ const server = http.createServer(async (req, res) => {
       })
     }
 
-    // POST /ai/chat
+    // POST /ai/chat — main AI chat endpoint
+    // Accepts { message, user_id?, user_role? } and returns AI response.
+    // user_role must be "sales", "admin", or "super_admin" if provided.
     if (req.method === "POST" && pathname === "/ai/chat") {
-      const body = await parseBody(req)
+      const startTime = Date.now()
+      const chunks = []
+      req.on("data", c => chunks.push(c))
+      req.on("end", () => {
+        const rawBody = Buffer.concat(chunks).toString()
+        try {
+          const body = JSON.parse(rawBody)
+          processRequest(req, res, body, startTime)
+        } catch {
+          sendJSON(res, 400, { error: "Invalid JSON" })
+        }
+      })
+      return
+    }
+
+// Separate handler for /ai/chat (defined here due to hoisting within the IIFE)
+async function processRequest(req, res, body, startTime) {
       const { message, user_id, user_role } = body
 
       if (!message) {
@@ -237,7 +568,7 @@ const server = http.createServer(async (req, res) => {
       return sendJSON(res, 200, { response })
     }
 
-    // 404
+    // 404 fallback
     sendJSON(res, 404, { error: "Not found" })
   } catch (err) {
     console.error("Request error:", err)

package.json

@@ -4,12 +4,15 @@
   "private": true,
   "scripts": {
     "dev": "npm run dev:precheck & npm run dev:ollama & npm run dev:start",
-    "dev:start": "concurrently -n AI,BROWSE,NEXT -c cyan,magenta,green \"npm run dev:rust\" \"npm run dev:browser-use\" \"npm run dev:next\"",
+    "dev:signaling": "node signaling-server.mjs",
+    "dev:open": "node scripts/open-browser.mjs",
+    "dev:start": "concurrently -n AI,BROWSE,SIGNAL,NEXT,OPEN -c cyan,magenta,yellow,green,white \"npm run dev:rust\" \"npm run dev:browser-use\" \"npm run dev:signaling\" \"npm run dev:next\" \"npm run dev:open\"",
     "dev:next": "next dev -p 3006",
-    "dev:precheck": "powershell -NoProfile -Command \"Get-NetTCPConnection -State Listen | Where-Object { $_.LocalPort -in 3001,3006,3008 } | ForEach-Object { Stop-Process -Id $_.OwningProcess -Force -ErrorAction SilentlyContinue; Write-Host ('Freed port '+$_.LocalPort) }\"",
-    "dev:ollama": "powershell -NoProfile -Command \"$ollama = if (Get-Command ollama -ErrorAction SilentlyContinue) { 'ollama' } else { Join-Path $env:LOCALAPPDATA 'Programs\\Ollama\\ollama.exe' }; if (-not (Get-Process ollama -ErrorAction SilentlyContinue)) { Start-Process $ollama -ArgumentList 'serve' -WindowStyle Hidden; Start-Sleep 3 }; exit 0\"",
+    "dev:precheck": "node scripts/precheck.mjs",
+    "dev:ollama": "node scripts/ensure-ollama.mjs",
     "dev:rust": "node ai-server/index.mjs",
-    "dev:browser-use": "cd browser-use-service && python main.py",
+    "dev:browser-use": "cd browser-use-service && node ../scripts/run-python.mjs main.py",
+    "setup": "node scripts/setup.mjs",
     "build": "next build",
     "start": "next start -p 3006",
     "lint": "eslint"
@@ -32,6 +35,8 @@
     "@radix-ui/react-switch": "^1.1.3",
     "@radix-ui/react-tabs": "^1.1.3",
     "@radix-ui/react-tooltip": "^1.1.8",
+    "@supabase/ssr": "^0.12.0",
+    "@supabase/supabase-js": "^2.108.2",
     "@tanstack/react-table": "^8.20.6",
     "autoprefixer": "^10.4.20",
     "bcryptjs": "^3.0.3",
@@ -47,6 +52,8 @@
     "react-dom": "^18.3.1",
     "react-hook-form": "^7.54.2",
     "recharts": "^2.15.0",
+    "socket.io": "^4.8.3",
+    "socket.io-client": "^4.8.3",
     "sonner": "^1.7.4",
     "tailwind-merge": "^2.6.0",
     "vaul": "^1.1.2",