Files
Ace dba4c84cd5
Build & Auto-Repair / build (push) Has been cancelled
Added finishing touch on other languages
2026-07-08 14:24:03 +02:00

587 lines
23 KiB
JavaScript

// ── CRM AI Server ──────────────────────────────────────────────────
// Provides:
// - Chat API (POST /ai/chat) — routes user messages to Ollama for sales coaching
// - Setup wizard endpoints (GET /setup/status, POST /setup/profile, etc.)
// - Combined /status endpoint for splash page health polling
// - Configuration routes (GET/POST /ai/instructions, GET /ai/jobs)
// - Model pull support (POST /setup/ollama/pull)
//
// This is a zero-dependency Node.js HTTP server (no Express needed).
import http from "node:http"
import fs from "node:fs"
import path from "node:path"
import { spawn } from "node:child_process"
import crypto from "node:crypto"
import { fileURLToPath } from "node:url"
const __dirname = path.dirname(fileURLToPath(import.meta.url))
const ROOT = path.resolve(__dirname, "..")
// ── Load .env.local ──────────────────────────────────────────────
// Reads key=value pairs and sets them as process.env so they're
// available throughout the server. Ignores comments and blank lines.
// Values with matching quotes are unquoted.
try {
const envPath = path.join(ROOT, ".env.local")
const envContent = fs.readFileSync(envPath, "utf-8")
for (const line of envContent.split("\n")) {
const trimmed = line.trim()
if (!trimmed || trimmed.startsWith("#")) continue
const eqIdx = trimmed.indexOf("=")
if (eqIdx === -1) continue
const k = trimmed.substring(0, eqIdx).trim()
let v = trimmed.substring(eqIdx + 1).trim()
if ((v.startsWith('"') && v.endsWith('"')) || (v.startsWith("'") && v.endsWith("'"))) v = v.slice(1, -1)
if (!process.env[k]) process.env[k] = v
}
console.log("Loaded .env.local")
} catch {
// .env.local may not exist (first run), which is fine
}
// ── Config from env ─────────────────────────────────────────────
const PORT = parseInt(process.env.AI_PORT || "3001", 10)
const HOST = process.env.AI_HOST || "0.0.0.0"
const OLLAMA_URL = process.env.OLLAMA_BASE_URL || "http://localhost:11434"
const MODEL = process.env.AI_MODEL || "llama3.2:3b"
const SCRAPER_URL = process.env.SCRAPER_URL || "http://127.0.0.1:3008"
const FRONTEND_URL = process.env.FRONTEND_URL || "http://127.0.0.1:3006"
const DATABASE_URL = process.env.DATABASE_URL
const JOBS_PATH = process.env.JOBS_PATH || path.join(ROOT, "data", "ai", "jobs.jsonl")
const AI_MD_PATH = process.env.AI_MD_PATH || path.join(ROOT, "data", "ai", "ai.md")
// ── Setup state ──────────────────────────────────────────────────
// Tracks the Ollama model pull process so the setup wizard can
// poll for download progress.
let pullProcess = null
let pullProgress = { status: "idle", progress: 0, message: "" }
// ── Job loading ─────────────────────────────────────────────────
// Loads job categories from a JSONL file (one JSON object per line).
// Used as context for the AI sales coach chat responses.
function loadJobs() {
try {
const content = fs.readFileSync(JOBS_PATH, "utf-8")
const jobs = content
.split("\n")
.map((l) => l.trim())
.filter(Boolean)
.map((l) => {
try {
return JSON.parse(l)
} catch {
return null
}
})
.filter(Boolean)
console.log(`Loaded ${jobs.length} job categories`)
return jobs
} catch (e) {
console.error(`Failed to load jobs from ${JOBS_PATH}:`, e.message)
return []
}
}
// ── ai.md management ────────────────────────────────────────────
// ai.md is a Markdown file containing system instructions for the AI.
// It can be read, written, or appended to via the API.
function readInstructions() {
try {
return fs.readFileSync(AI_MD_PATH, "utf-8")
} catch {
return ""
}
}
function writeInstructions(content) {
fs.writeFileSync(AI_MD_PATH, content, "utf-8")
return content
}
function appendToImprovementLog(entry) {
// Adds a timestamped entry to the ## Improvement Log section of ai.md
const current = readInstructions()
const timestamp = new Date().toISOString().replace("T", " ").substring(0, 16)
const logEntry = `\n- ${timestamp}${entry}`
const logSection = "\n## Improvement Log"
const logIndex = current.indexOf(logSection)
if (logIndex !== -1) {
const afterLog = current.substring(logIndex + logSection.length)
const nextSectionIndex = afterLog.search(/\n## /)
if (nextSectionIndex !== -1) {
const insertAt = logIndex + logSection.length + nextSectionIndex
const updated = current.substring(0, insertAt) + logEntry + "\n" + current.substring(insertAt)
writeInstructions(updated)
return updated
}
writeInstructions(current + logEntry + "\n")
return current + logEntry + "\n"
}
const updated = current + `\n${logSection}\n${logEntry}\n`
writeInstructions(updated)
return updated
}
// ── Chat handler ────────────────────────────────────────────────
// scrapeFacebook() calls the scraper service (port 3008) to get leads.
// handleChat() processes user messages — triggers lead scraping when
// the user asks for "leads" or "listings", otherwise routes to Ollama
// for AI-powered sales coaching.
async function scrapeFacebook() {
const profilePath = process.env.FX_PROFILE || ""
const urlPath = `/scrape/facebook?force=true${profilePath ? `&profile_path=${encodeURIComponent(profilePath)}` : ""}`
try {
const body = await new Promise((resolve, reject) => {
const parsed = new URL(SCRAPER_URL)
let done = false
const req = http.request({ hostname: parsed.hostname, port: parsed.port || 3008, path: urlPath, method: "POST", timeout: 60000 }, (res) => {
let data = ""
res.on("data", (c) => data += c)
res.on("end", () => { done = true; resolve(data) })
res.on("error", (e) => { if (!done) { done = true; reject(e) } })
})
req.on("timeout", () => { if (!done) { done = true; req.destroy(); reject(new Error("scraper timeout")) } })
req.on("error", (e) => { if (!done) { done = true; reject(e) } })
req.end()
})
const data = JSON.parse(body)
return data
} catch (e) {
console.error("scrapeFacebook error:", e.message)
return null
}
}
function formatLeads(leads) {
if (!leads || leads.length === 0) return "No leads found from the latest scrape."
let output = `**${leads.length} leads found:**\n\n`
for (let i = 0; i < leads.length; i++) {
const l = leads[i]
output += `${i + 1}. ${l.title || "No title"}\n`
if (l.author) output += ` Author: ${l.author}\n`
if (l.date) output += ` Date: ${l.date}\n`
if (l.url) output += ` URL: ${l.url}\n`
output += "\n"
}
return output.trim()
}
async function handleChat(userMessage, userId, userRole) {
// If the user asks for leads, trigger the scraper
const lowerMsg = userMessage.toLowerCase()
const triggerWords = ["lists", "listings", "leads", "recent leads", "pull leads", "show me leads", "show listings"]
if (triggerWords.some(w => lowerMsg.includes(w))) {
const result = await scrapeFacebook()
if (result && result.success) {
return formatLeads(result.leads)
}
return "Scraper returned no results or encountered an error. Try again later."
}
// Otherwise, build a system prompt with job context and send to Ollama
const jobs = loadedJobs
const instructions = readInstructions()
const jobList = jobs
.map((j) => `- ${j.job_title} (${j.industry}): ${j.description}`)
.join("\n")
const systemPrompt = `You are a Sales AI Assistant for Coast IT CRM. Your role is to help salespeople with tips, strategies, and guidance.
Available job categories to target:
${jobList}
Current instructions:
${instructions}
Provide concise, actionable sales advice. When asked about a specific job category, give targeted tips on finding and engaging prospects in that field. Keep responses under 300 words unless asked for detail.`
const ollamaRes = await fetch(`${OLLAMA_URL}/api/chat`, {
method: "POST",
headers: { "Content-Type": "application/json" },
signal: AbortSignal.timeout(60000),
body: JSON.stringify({
model: MODEL,
messages: [
{ role: "system", content: systemPrompt },
{ role: "user", content: userMessage },
],
stream: false,
options: { temperature: 0.7, num_predict: 1024 },
}),
})
if (!ollamaRes.ok) {
const text = await ollamaRes.text()
throw new Error(`Ollama error (${ollamaRes.status}): ${text}`)
}
const data = await ollamaRes.json()
const responseText = data.message?.content || ""
// Persist conversation to PostgreSQL (best-effort — table may not exist yet)
try {
if (pgPool && userId) {
await pgPool.query(
"INSERT INTO ai_conversations (id, user_id, role, message, response) VALUES ($1, $2, $3, $4, $5)",
[crypto.randomUUID(), userId, userRole || "sales", userMessage, responseText]
)
}
} catch {
// table might not exist, ignore
}
return responseText
}
// ── PG pool (lazy init) ────────────────────────────────────────
// PostgreSQL connection pool for storing conversation history.
// Lazy-initialized so the server starts even without a DB.
let pgPool = null
async function initPg() {
if (!DATABASE_URL) return
try {
const { default: pg } = await import("pg")
pgPool = new pg.Pool({ connectionString: DATABASE_URL, max: 5 })
await pgPool.query("SELECT 1")
console.log("Connected to PostgreSQL")
} catch (e) {
console.warn("PostgreSQL unavailable (AI convos won't be saved):", e.message)
}
}
// ── Request router ─────────────────────────────────────────────
const loadedJobs = loadJobs()
function sendJSON(res, status, data) {
res.writeHead(status, { "Content-Type": "application/json" })
res.end(JSON.stringify(data))
}
function parseBody(req) {
return new Promise((resolve, reject) => {
let body = ""
req.on("data", (chunk) => (body += chunk))
req.on("end", () => {
try {
resolve(JSON.parse(body))
} catch {
reject(new Error("Invalid JSON"))
}
})
req.on("error", reject)
})
}
function parseURL(req) {
const url = new URL(req.url, `http://${req.headers.host || "localhost"}`)
return { pathname: url.pathname, searchParams: url.searchParams }
}
// ── HTTP Server ─────────────────────────────────────────────────
const server = http.createServer(async (req, res) => {
// CORS headers — allow the Next.js frontend (port 3006) to call us
res.setHeader("Access-Control-Allow-Origin", "*")
res.setHeader("Access-Control-Allow-Methods", "GET, POST, OPTIONS")
res.setHeader("Access-Control-Allow-Headers", "Content-Type")
if (req.method === "OPTIONS") {
res.writeHead(204)
res.end()
return
}
const { pathname } = parseURL(req)
try {
// GET /splash — loading screen
if (req.method === "GET" && pathname === "/splash") {
const splashPath = path.join(ROOT, "splash.html")
if (fs.existsSync(splashPath)) {
const html = fs.readFileSync(splashPath, "utf-8")
res.writeHead(200, { "Content-Type": "text/html" })
res.end(html)
return
}
sendJSON(res, 200, { status: "ok" })
return
}
// GET /health
if (req.method === "GET" && pathname === "/health") {
return sendJSON(res, 200, { status: "ok", model: MODEL })
}
// GET /status — combined health of all services
// Used by the splash page to check if AI, Scraper, and Frontend are ready.
// Polls each service internally to avoid cross-origin CORS issues.
if (req.method === "GET" && pathname === "/status") {
const { default: http } = await import("http")
const results = { ai: true }
// Check scraper
try {
await new Promise((resolve, reject) => {
const r = http.get(`${SCRAPER_URL}/health`, { timeout: 3000 }, (res) => { res.resume(); resolve() })
r.on("timeout", () => { r.destroy(); reject(new Error("timeout")) })
r.on("error", reject)
})
results.scraper = true
} catch { results.scraper = false }
// Check frontend
try {
await new Promise((resolve, reject) => {
const r = http.get(FRONTEND_URL, { timeout: 3000 }, (res) => { res.resume(); resolve() })
r.on("timeout", () => { r.destroy(); reject(new Error("timeout")) })
r.on("error", reject)
})
results.frontend = true
} catch { results.frontend = false }
return sendJSON(res, 200, results)
}
// ── Setup endpoints ─────────────────────────────────────────
// GET /setup/status — check environment
// Called by the splash page on boot. Returns info about:
// - Ollama availability
// - Model presence
// - Detected browsers with login status
// - Whether this is a first run (wizard needed)
if (req.method === "GET" && pathname === "/setup/status") {
const envExists = fs.existsSync(path.join(ROOT, ".env.local"))
// Ollama check
let ollamaRunning = false
try {
await fetch(`${OLLAMA_URL}/api/tags`, { signal: AbortSignal.timeout(3000) })
ollamaRunning = true
} catch {}
// Model check
let modelAvailable = false
if (ollamaRunning) {
try {
const r = await fetch(`${OLLAMA_URL}/api/show`, {
method: "POST", body: JSON.stringify({ name: MODEL }),
signal: AbortSignal.timeout(5000),
})
modelAvailable = r.ok
} catch {}
}
// Detect all browsers via scraper
let browsers = { firefox: { path: null }, opera: { path: null }, chrome: { path: null }, edge: { path: null } }
let facebookLoggedIn = false
let selectedBrowser = process.env.SELECTED_BROWSER || ""
try {
await fetch(`${SCRAPER_URL}/health`, { signal: AbortSignal.timeout(2000) })
const profiles = await (await fetch(`${SCRAPER_URL}/setup/profile`, { signal: AbortSignal.timeout(5000) })).json()
for (const [b, p] of Object.entries(profiles)) {
if (p) browsers[b] = { path: p }
}
// Check login for the selected browser first, then try all
const detectedList = Object.entries(browsers).filter(([, v]) => v.path)
for (const [b, v] of detectedList) {
try {
const r = await fetch(`${SCRAPER_URL}/setup/check-login`, {
method: "POST", headers: { "Content-Type": "application/json" },
body: JSON.stringify({ browser: b, profile_path: v.path }),
signal: AbortSignal.timeout(20000),
})
if (r.ok) {
const d = await r.json()
browsers[b].logged_in = d.logged_in === true
if (d.logged_in && !facebookLoggedIn) {
facebookLoggedIn = true
if (!selectedBrowser) selectedBrowser = b
}
}
} catch {}
}
} catch {}
const anyDetected = Object.values(browsers).some(v => v.path)
// first_run = any setup step is incomplete
const firstRun = !envExists || !ollamaRunning || !anyDetected || !facebookLoggedIn || !modelAvailable
return sendJSON(res, 200, {
first_run: firstRun,
env_exists: envExists,
ollama_running: ollamaRunning,
model_available: modelAvailable,
model_name: MODEL,
selected_browser: selectedBrowser,
browsers,
facebook_logged_in: facebookLoggedIn,
})
}
// POST /setup/profile — save selected browser + path to .env.local
// Called by the setup wizard when the user confirms their browser choice.
// Writes SELECTED_BROWSER and the matching profile env var to .env.local.
if (req.method === "POST" && pathname === "/setup/profile") {
const body = await parseBody(req)
const browserName = (body.browser || "").trim().toLowerCase()
const profilePath = (body.path || "").trim()
if (!browserName || !["firefox", "opera", "chrome", "edge"].includes(browserName))
return sendJSON(res, 400, { error: "Valid browser required (firefox/opera/chrome/edge)" })
if (!profilePath)
return sendJSON(res, 400, { error: "Path required" })
const envKey = browserName === "firefox" ? "FX_PROFILE"
: browserName === "opera" ? "OPERA_PROFILE"
: browserName === "edge" ? "EDGE_PROFILE"
: "CHROME_PROFILE"
const envPath = path.join(ROOT, ".env.local")
let content = ""
try { content = fs.readFileSync(envPath, "utf-8") } catch {}
let lines = content.split("\n")
// Update or add SELECTED_BROWSER
let foundSel = false
for (let i = 0; i < lines.length; i++) {
if (lines[i].trim().startsWith("SELECTED_BROWSER=")) {
lines[i] = `SELECTED_BROWSER=${browserName}`
foundSel = true
break
}
}
if (!foundSel) lines.push(`SELECTED_BROWSER=${browserName}`)
// Update or add browser profile
let foundProf = false
for (let i = 0; i < lines.length; i++) {
if (lines[i].trim().startsWith(`${envKey}=`)) {
lines[i] = `${envKey}=${profilePath}`
foundProf = true
break
}
}
if (!foundProf) lines.push(`${envKey}=${profilePath}`)
fs.writeFileSync(envPath, lines.join("\n"), "utf-8")
process.env.SELECTED_BROWSER = browserName
process.env[envKey] = profilePath
return sendJSON(res, 200, { success: true, browser: browserName, path: profilePath })
}
// POST /setup/check-login — proxy to scraper, accepts browser + profile_path
// The splash page calls this (via the AI server) to verify Facebook login status.
if (req.method === "POST" && pathname === "/setup/check-login") {
const body = await parseBody(req)
const browserName = (body.browser || "").trim().toLowerCase() || process.env.SELECTED_BROWSER || ""
const profilePath = (body.profile_path || "").trim()
if (!profilePath) return sendJSON(res, 200, { logged_in: false, reason: "no_profile" })
try {
const r = await fetch("http://127.0.0.1:3008/setup/check-login", {
method: "POST", headers: { "Content-Type": "application/json" },
body: JSON.stringify({ browser: browserName, profile_path: profilePath }),
signal: AbortSignal.timeout(20000),
})
if (r.ok) { const d = await r.json(); return sendJSON(res, 200, d) }
} catch {}
return sendJSON(res, 200, { logged_in: false, reason: "scraper_unavailable" })
}
// POST /setup/ollama/pull — start pulling the model
// Spawns "ollama pull" as a child process. The setup wizard polls
// the progress endpoint to show a download progress bar.
if (req.method === "POST" && pathname === "/setup/ollama/pull") {
if (pullProcess) return sendJSON(res, 200, { status: "already_running" })
pullProgress = { status: "downloading", progress: 0, message: "Starting..." }
const isWin = process.platform === "win32"
const cmd = isWin ? "ollama.exe" : "ollama"
pullProcess = spawn(cmd, ["pull", MODEL], { stdio: ["ignore", "pipe", "pipe"] })
pullProcess.stdout.on("data", (data) => {
const text = data.toString()
pullProgress.message = text.trim()
// Extract percentage from patterns like "pulling xxxx... 45%"
const m = text.match(/(\d+)%/)
if (m) pullProgress.progress = parseInt(m[1], 10)
})
pullProcess.on("close", (code) => {
pullProcess = null
pullProgress.status = code === 0 ? "done" : "failed"
if (code === 0) pullProgress.progress = 100
})
return sendJSON(res, 200, { status: "started" })
}
// GET /setup/ollama/pull/progress
// Returns current download progress for the setup wizard.
if (req.method === "GET" && pathname === "/setup/ollama/pull/progress") {
return sendJSON(res, 200, pullProgress)
}
// GET /ai/jobs — return loaded job categories
if (req.method === "GET" && pathname === "/ai/jobs") {
return sendJSON(res, 200, { jobs: loadedJobs })
}
// GET /ai/instructions — return current ai.md content
if (req.method === "GET" && pathname === "/ai/instructions") {
const instructions = readInstructions()
return sendJSON(res, 200, { success: true, instructions })
}
// POST /ai/instructions — update ai.md or append improvement log entry
if (req.method === "POST" && pathname === "/ai/instructions") {
const body = await parseBody(req)
if (body.content) {
writeInstructions(body.content)
} else if (body.entry) {
appendToImprovementLog(body.entry)
}
return sendJSON(res, 200, {
success: true,
instructions: readInstructions(),
})
}
// POST /ai/chat — main AI chat endpoint
// Accepts { message, user_id?, user_role? } and returns AI response.
// user_role must be "sales", "admin", or "super_admin" if provided.
if (req.method === "POST" && pathname === "/ai/chat") {
const chunks = []
req.on("data", c => chunks.push(c))
req.on("end", async () => {
try {
const rawBody = Buffer.concat(chunks).toString()
const body = JSON.parse(rawBody)
const { message, user_id, user_role } = body
if (!message) {
return sendJSON(res, 400, { error: "message is required" })
}
const validRoles = ["sales", "admin", "super_admin"]
if (user_role && !validRoles.includes(user_role)) {
return sendJSON(res, 403, { error: "Forbidden" })
}
const response = await handleChat(message, user_id || "", user_role || "sales")
sendJSON(res, 200, { response })
} catch (e) {
if (!res.headersSent) sendJSON(res, 500, { error: e.message })
}
})
return
}
// 404 fallback
sendJSON(res, 404, { error: "Not found" })
} catch (err) {
console.error("Request error:", err)
sendJSON(res, 500, { error: err.message })
}
})
// ── Start ───────────────────────────────────────────────────────
server.listen(PORT, HOST, () => {
console.log(`CRM AI server listening on http://${HOST}:${PORT}`)
console.log(` Model: ${MODEL}`)
console.log(` Ollama: ${OLLAMA_URL}`)
})
initPg()