From 7bd9c17b5f4c94c371aaeddb1aaf2d624c176b8b Mon Sep 17 00:00:00 2001 From: Hannah_Bagga Date: Tue, 23 Jun 2026 11:07:55 +0200 Subject: [PATCH 1/3] Add Node.js AI server, fix dev scripts, ignore rust-ai/target --- .gitignore | 4 +- ai-server/index.mjs | 255 +++++++++++++++++ package-lock.json | 555 ++++++++++++++++++++++--------------- package.json | 4 +- rust-ai/.cargo/config.toml | 2 + 5 files changed, 601 insertions(+), 219 deletions(-) create mode 100644 ai-server/index.mjs create mode 100644 rust-ai/.cargo/config.toml diff --git a/.gitignore b/.gitignore index 91ec548..6bf56ef 100644 --- a/.gitignore +++ b/.gitignore @@ -58,4 +58,6 @@ next-env.d.ts .git .git_bak node_modules -target \ No newline at end of file +target +# rust build artifacts +rust-ai/target/ diff --git a/ai-server/index.mjs b/ai-server/index.mjs new file mode 100644 index 0000000..14813e8 --- /dev/null +++ b/ai-server/index.mjs @@ -0,0 +1,255 @@ +import http from "node:http" +import fs from "node:fs" +import path from "node:path" +import { fileURLToPath } from "node:url" + +const __dirname = path.dirname(fileURLToPath(import.meta.url)) +const ROOT = path.resolve(__dirname, "..") + +// ── Config from env ───────────────────────────────────────────── +const PORT = parseInt(process.env.AI_PORT || "3001", 10) +const HOST = process.env.AI_HOST || "0.0.0.0" +const OLLAMA_URL = process.env.OLLAMA_BASE_URL || "http://localhost:11434" +const MODEL = process.env.AI_MODEL || "sam860/dolphin3-llama3.2:3b" +const DATABASE_URL = process.env.DATABASE_URL +const JOBS_PATH = process.env.JOBS_PATH || path.join(ROOT, "data", "ai", "jobs.jsonl") +const AI_MD_PATH = process.env.AI_MD_PATH || path.join(ROOT, "data", "ai", "ai.md") + +// ── Job loading ───────────────────────────────────────────────── +function loadJobs() { + try { + const content = fs.readFileSync(JOBS_PATH, "utf-8") + const jobs = content + .split("\n") + .map((l) => l.trim()) + .filter(Boolean) + .map((l) => { + try { + return JSON.parse(l) + } catch { + return null + } + }) + .filter(Boolean) + console.log(`Loaded ${jobs.length} job categories`) + return jobs + } catch (e) { + console.error(`Failed to load jobs from ${JOBS_PATH}:`, e.message) + return [] + } +} + +// ── ai.md management ──────────────────────────────────────────── +function readInstructions() { + try { + return fs.readFileSync(AI_MD_PATH, "utf-8") + } catch { + return "" + } +} + +function writeInstructions(content) { + fs.writeFileSync(AI_MD_PATH, content, "utf-8") + return content +} + +function appendToImprovementLog(entry) { + const current = readInstructions() + const timestamp = new Date().toISOString().replace("T", " ").substring(0, 16) + const logEntry = `\n- ${timestamp} — ${entry}` + const logSection = "\n## Improvement Log" + const logIndex = current.indexOf(logSection) + if (logIndex !== -1) { + const afterLog = current.substring(logIndex + logSection.length) + const nextSectionIndex = afterLog.search(/\n## /) + if (nextSectionIndex !== -1) { + const insertAt = logIndex + logSection.length + nextSectionIndex + const updated = current.substring(0, insertAt) + logEntry + "\n" + current.substring(insertAt) + writeInstructions(updated) + return updated + } + writeInstructions(current + logEntry + "\n") + return current + logEntry + "\n" + } + const updated = current + `\n${logSection}\n${logEntry}\n` + writeInstructions(updated) + return updated +} + +// ── Chat handler ──────────────────────────────────────────────── +async function handleChat(userMessage, userId, userRole) { + const jobs = loadedJobs + const instructions = readInstructions() + + const jobList = jobs + .map((j) => `- ${j.job_title} (${j.industry}): ${j.description}`) + .join("\n") + + const systemPrompt = `You are a Sales AI Assistant for Coast IT CRM. Your role is to help salespeople with tips, strategies, and guidance. + +Available job categories to target: +${jobList} + +Current instructions: +${instructions} + +Provide concise, actionable sales advice. When asked about a specific job category, give targeted tips on finding and engaging prospects in that field. Keep responses under 300 words unless asked for detail.` + + const ollamaRes = await fetch(`${OLLAMA_URL}/api/chat`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + model: MODEL, + messages: [ + { role: "system", content: systemPrompt }, + { role: "user", content: userMessage }, + ], + stream: false, + options: { temperature: 0.7, num_predict: 1024 }, + }), + }) + + if (!ollamaRes.ok) { + const text = await ollamaRes.text() + throw new Error(`Ollama error (${ollamaRes.status}): ${text}`) + } + + const data = await ollamaRes.json() + const responseText = data.message?.content || "" + + // Try to persist to DB (best-effort) + try { + if (pgPool && userId) { + await pgPool.query( + "INSERT INTO ai_conversations (id, user_id, role, message, response) VALUES ($1, $2, $3, $4, $5)", + [crypto.randomUUID(), userId, userRole || "sales", userMessage, responseText] + ) + } + } catch { + // table might not exist, ignore + } + + return responseText +} + +// ── PG pool (lazy init) ──────────────────────────────────────── +let pgPool = null +async function initPg() { + if (!DATABASE_URL) return + try { + const { default: pg } = await import("pg") + pgPool = new pg.Pool({ connectionString: DATABASE_URL, max: 5 }) + await pgPool.query("SELECT 1") + console.log("Connected to PostgreSQL") + } catch (e) { + console.warn("PostgreSQL unavailable (AI convos won't be saved):", e.message) + } +} + +// ── Request router ───────────────────────────────────────────── +const loadedJobs = loadJobs() + +function sendJSON(res, status, data) { + res.writeHead(status, { "Content-Type": "application/json" }) + res.end(JSON.stringify(data)) +} + +function parseBody(req) { + return new Promise((resolve, reject) => { + let body = "" + req.on("data", (chunk) => (body += chunk)) + req.on("end", () => { + try { + resolve(JSON.parse(body)) + } catch { + reject(new Error("Invalid JSON")) + } + }) + req.on("error", reject) + }) +} + +function parseURL(req) { + const url = new URL(req.url, `http://${req.headers.host || "localhost"}`) + return { pathname: url.pathname, searchParams: url.searchParams } +} + +const server = http.createServer(async (req, res) => { + // CORS + res.setHeader("Access-Control-Allow-Origin", "*") + res.setHeader("Access-Control-Allow-Methods", "GET, POST, OPTIONS") + res.setHeader("Access-Control-Allow-Headers", "Content-Type") + + if (req.method === "OPTIONS") { + res.writeHead(204) + res.end() + return + } + + const { pathname } = parseURL(req) + + try { + // GET /health + if (req.method === "GET" && pathname === "/health") { + return sendJSON(res, 200, { status: "ok", model: MODEL }) + } + + // GET /ai/jobs + if (req.method === "GET" && pathname === "/ai/jobs") { + return sendJSON(res, 200, { jobs: loadedJobs }) + } + + // GET /ai/instructions + if (req.method === "GET" && pathname === "/ai/instructions") { + const instructions = readInstructions() + return sendJSON(res, 200, { success: true, instructions }) + } + + // POST /ai/instructions + if (req.method === "POST" && pathname === "/ai/instructions") { + const body = await parseBody(req) + if (body.content) { + writeInstructions(body.content) + } else if (body.entry) { + appendToImprovementLog(body.entry) + } + return sendJSON(res, 200, { + success: true, + instructions: readInstructions(), + }) + } + + // POST /ai/chat + if (req.method === "POST" && pathname === "/ai/chat") { + const body = await parseBody(req) + const { message, user_id, user_role } = body + + if (!message) { + return sendJSON(res, 400, { error: "message is required" }) + } + + const validRoles = ["sales", "admin", "super_admin"] + if (user_role && !validRoles.includes(user_role)) { + return sendJSON(res, 403, { error: "Forbidden" }) + } + + const response = await handleChat(message, user_id || "", user_role || "sales") + return sendJSON(res, 200, { response }) + } + + // 404 + sendJSON(res, 404, { error: "Not found" }) + } catch (err) { + console.error("Request error:", err) + sendJSON(res, 500, { error: err.message }) + } +}) + +// ── Start ─────────────────────────────────────────────────────── +server.listen(PORT, HOST, () => { + console.log(`CRM AI server listening on http://${HOST}:${PORT}`) + console.log(` Model: ${MODEL}`) + console.log(` Ollama: ${OLLAMA_URL}`) +}) + +initPg() diff --git a/package-lock.json b/package-lock.json index 03119b8..1821189 100644 --- a/package-lock.json +++ b/package-lock.json @@ -34,7 +34,7 @@ "framer-motion": "^11.15.0", "jose": "^6.2.3", "lucide-react": "^0.468.0", - "next": "15.0.4", + "next": "^15.5.19", "next-themes": "^0.4.4", "pg": "^8.21.0", "react": "^18.3.1", @@ -53,7 +53,7 @@ "@types/react-dom": "^18", "concurrently": "^10.0.3", "eslint": "^9", - "eslint-config-next": "15.0.4", + "eslint-config-next": "15.5.19", "postcss": "^8.4.49", "tailwindcss": "^3.4.17", "typescript": "^5" @@ -360,10 +360,20 @@ "url": "https://github.com/sponsors/nzakas" } }, + "node_modules/@img/colour": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@img/colour/-/colour-1.1.0.tgz", + "integrity": "sha512-Td76q7j57o/tLVdgS746cYARfSyxk8iEfRxewL9h4OMzYhbW4TAcppl0mT4eyqXddh6L/jwoM75mo7ixa/pCeQ==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">=18" + } + }, "node_modules/@img/sharp-darwin-arm64": { - "version": "0.33.5", - "resolved": "https://registry.npmjs.org/@img/sharp-darwin-arm64/-/sharp-darwin-arm64-0.33.5.tgz", - "integrity": "sha512-UT4p+iz/2H4twwAoLCqfA9UH5pI6DggwKEGuaPy7nCVQ8ZsiY5PIcrRvD1DzuY3qYL07NtIQcWnBSY/heikIFQ==", + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-darwin-arm64/-/sharp-darwin-arm64-0.34.5.tgz", + "integrity": "sha512-imtQ3WMJXbMY4fxb/Ndp6HBTNVtWCUI0WdobyheGf5+ad6xX8VIDO8u2xE4qc/fr08CKG/7dDseFtn6M6g/r3w==", "cpu": [ "arm64" ], @@ -379,13 +389,13 @@ "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-libvips-darwin-arm64": "1.0.4" + "@img/sharp-libvips-darwin-arm64": "1.2.4" } }, "node_modules/@img/sharp-darwin-x64": { - "version": "0.33.5", - "resolved": "https://registry.npmjs.org/@img/sharp-darwin-x64/-/sharp-darwin-x64-0.33.5.tgz", - "integrity": "sha512-fyHac4jIc1ANYGRDxtiqelIbdWkIuQaI84Mv45KvGRRxSAa7o7d1ZKAOBaYbnepLC1WqxfpimdeWfvqqSGwR2Q==", + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-darwin-x64/-/sharp-darwin-x64-0.34.5.tgz", + "integrity": "sha512-YNEFAF/4KQ/PeW0N+r+aVVsoIY0/qxxikF2SWdp+NRkmMB7y9LBZAVqQ4yhGCm/H3H270OSykqmQMKLBhBJDEw==", "cpu": [ "x64" ], @@ -401,13 +411,13 @@ "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-libvips-darwin-x64": "1.0.4" + "@img/sharp-libvips-darwin-x64": "1.2.4" } }, "node_modules/@img/sharp-libvips-darwin-arm64": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-arm64/-/sharp-libvips-darwin-arm64-1.0.4.tgz", - "integrity": "sha512-XblONe153h0O2zuFfTAbQYAX2JhYmDHeWikp1LM9Hul9gVPjFY427k6dFEcOL72O01QxQsWi761svJ/ev9xEDg==", + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-arm64/-/sharp-libvips-darwin-arm64-1.2.4.tgz", + "integrity": "sha512-zqjjo7RatFfFoP0MkQ51jfuFZBnVE2pRiaydKJ1G/rHZvnsrHAOcQALIi9sA5co5xenQdTugCvtb1cuf78Vf4g==", "cpu": [ "arm64" ], @@ -421,9 +431,9 @@ } }, "node_modules/@img/sharp-libvips-darwin-x64": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-x64/-/sharp-libvips-darwin-x64-1.0.4.tgz", - "integrity": "sha512-xnGR8YuZYfJGmWPvmlunFaWJsb9T/AO2ykoP3Fz/0X5XV2aoYBPkX6xqCQvUTKKiLddarLaxpzNe+b1hjeWHAQ==", + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-x64/-/sharp-libvips-darwin-x64-1.2.4.tgz", + "integrity": "sha512-1IOd5xfVhlGwX+zXv2N93k0yMONvUlANylbJw1eTah8K/Jtpi15KC+WSiaX/nBmbm2HxRM1gZ0nSdjSsrZbGKg==", "cpu": [ "x64" ], @@ -437,12 +447,15 @@ } }, "node_modules/@img/sharp-libvips-linux-arm": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm/-/sharp-libvips-linux-arm-1.0.5.tgz", - "integrity": "sha512-gvcC4ACAOPRNATg/ov8/MnbxFDJqf/pDePbBnuBDcjsI8PssmjoKMAz4LtLaVi+OnSb5FK/yIOamqDwGmXW32g==", + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm/-/sharp-libvips-linux-arm-1.2.4.tgz", + "integrity": "sha512-bFI7xcKFELdiNCVov8e44Ia4u2byA+l3XtsAj+Q8tfCwO6BQ8iDojYdvoPMqsKDkuoOo+X6HZA0s0q11ANMQ8A==", "cpu": [ "arm" ], + "libc": [ + "glibc" + ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -453,12 +466,53 @@ } }, "node_modules/@img/sharp-libvips-linux-arm64": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm64/-/sharp-libvips-linux-arm64-1.0.4.tgz", - "integrity": "sha512-9B+taZ8DlyyqzZQnoeIvDVR/2F4EbMepXMc/NdVbkzsJbzkUjhXv/70GQJ7tdLA4YJgNP25zukcxpX2/SueNrA==", + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm64/-/sharp-libvips-linux-arm64-1.2.4.tgz", + "integrity": "sha512-excjX8DfsIcJ10x1Kzr4RcWe1edC9PquDRRPx3YVCvQv+U5p7Yin2s32ftzikXojb1PIFc/9Mt28/y+iRklkrw==", "cpu": [ "arm64" ], + "libc": [ + "glibc" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-ppc64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-ppc64/-/sharp-libvips-linux-ppc64-1.2.4.tgz", + "integrity": "sha512-FMuvGijLDYG6lW+b/UvyilUWu5Ayu+3r2d1S8notiGCIyYU/76eig1UfMmkZ7vwgOrzKzlQbFSuQfgm7GYUPpA==", + "cpu": [ + "ppc64" + ], + "libc": [ + "glibc" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-riscv64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-riscv64/-/sharp-libvips-linux-riscv64-1.2.4.tgz", + "integrity": "sha512-oVDbcR4zUC0ce82teubSm+x6ETixtKZBh/qbREIOcI3cULzDyb18Sr/Wcyx7NRQeQzOiHTNbZFF1UwPS2scyGA==", + "cpu": [ + "riscv64" + ], + "libc": [ + "glibc" + ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -469,12 +523,15 @@ } }, "node_modules/@img/sharp-libvips-linux-s390x": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-s390x/-/sharp-libvips-linux-s390x-1.0.4.tgz", - "integrity": "sha512-u7Wz6ntiSSgGSGcjZ55im6uvTrOxSIS8/dgoVMoiGE9I6JAfU50yH5BoDlYA1tcuGS7g/QNtetJnxA6QEsCVTA==", + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-s390x/-/sharp-libvips-linux-s390x-1.2.4.tgz", + "integrity": "sha512-qmp9VrzgPgMoGZyPvrQHqk02uyjA0/QrTO26Tqk6l4ZV0MPWIW6LTkqOIov+J1yEu7MbFQaDpwdwJKhbJvuRxQ==", "cpu": [ "s390x" ], + "libc": [ + "glibc" + ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -485,12 +542,15 @@ } }, "node_modules/@img/sharp-libvips-linux-x64": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-x64/-/sharp-libvips-linux-x64-1.0.4.tgz", - "integrity": "sha512-MmWmQ3iPFZr0Iev+BAgVMb3ZyC4KeFc3jFxnNbEPas60e1cIfevbtuyf9nDGIzOaW9PdnDciJm+wFFaTlj5xYw==", + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-x64/-/sharp-libvips-linux-x64-1.2.4.tgz", + "integrity": "sha512-tJxiiLsmHc9Ax1bz3oaOYBURTXGIRDODBqhveVHonrHJ9/+k89qbLl0bcJns+e4t4rvaNBxaEZsFtSfAdquPrw==", "cpu": [ "x64" ], + "libc": [ + "glibc" + ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -501,12 +561,15 @@ } }, "node_modules/@img/sharp-libvips-linuxmusl-arm64": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-arm64/-/sharp-libvips-linuxmusl-arm64-1.0.4.tgz", - "integrity": "sha512-9Ti+BbTYDcsbp4wfYib8Ctm1ilkugkA/uscUn6UXK1ldpC1JjiXbLfFZtRlBhjPZ5o1NCLiDbg8fhUPKStHoTA==", + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-arm64/-/sharp-libvips-linuxmusl-arm64-1.2.4.tgz", + "integrity": "sha512-FVQHuwx1IIuNow9QAbYUzJ+En8KcVm9Lk5+uGUQJHaZmMECZmOlix9HnH7n1TRkXMS0pGxIJokIVB9SuqZGGXw==", "cpu": [ "arm64" ], + "libc": [ + "musl" + ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -517,12 +580,15 @@ } }, "node_modules/@img/sharp-libvips-linuxmusl-x64": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-x64/-/sharp-libvips-linuxmusl-x64-1.0.4.tgz", - "integrity": "sha512-viYN1KX9m+/hGkJtvYYp+CCLgnJXwiQB39damAO7WMdKWlIhmYTfHjwSbQeUK/20vY154mwezd9HflVFM1wVSw==", + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-x64/-/sharp-libvips-linuxmusl-x64-1.2.4.tgz", + "integrity": "sha512-+LpyBk7L44ZIXwz/VYfglaX/okxezESc6UxDSoyo2Ks6Jxc4Y7sGjpgU9s4PMgqgjj1gZCylTieNamqA1MF7Dg==", "cpu": [ "x64" ], + "libc": [ + "musl" + ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -533,12 +599,15 @@ } }, "node_modules/@img/sharp-linux-arm": { - "version": "0.33.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm/-/sharp-linux-arm-0.33.5.tgz", - "integrity": "sha512-JTS1eldqZbJxjvKaAkxhZmBqPRGmxgu+qFKSInv8moZ2AmT5Yib3EQ1c6gp493HvrvV8QgdOXdyaIBrhvFhBMQ==", + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm/-/sharp-linux-arm-0.34.5.tgz", + "integrity": "sha512-9dLqsvwtg1uuXBGZKsxem9595+ujv0sJ6Vi8wcTANSFpwV/GONat5eCkzQo/1O6zRIkh0m/8+5BjrRr7jDUSZw==", "cpu": [ "arm" ], + "libc": [ + "glibc" + ], "license": "Apache-2.0", "optional": true, "os": [ @@ -551,16 +620,19 @@ "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-libvips-linux-arm": "1.0.5" + "@img/sharp-libvips-linux-arm": "1.2.4" } }, "node_modules/@img/sharp-linux-arm64": { - "version": "0.33.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm64/-/sharp-linux-arm64-0.33.5.tgz", - "integrity": "sha512-JMVv+AMRyGOHtO1RFBiJy/MBsgz0x4AWrT6QoEVVTyh1E39TrCUpTRI7mx9VksGX4awWASxqCYLCV4wBZHAYxA==", + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm64/-/sharp-linux-arm64-0.34.5.tgz", + "integrity": "sha512-bKQzaJRY/bkPOXyKx5EVup7qkaojECG6NLYswgktOZjaXecSAeCWiZwwiFf3/Y+O1HrauiE3FVsGxFg8c24rZg==", "cpu": [ "arm64" ], + "libc": [ + "glibc" + ], "license": "Apache-2.0", "optional": true, "os": [ @@ -573,16 +645,69 @@ "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-libvips-linux-arm64": "1.0.4" + "@img/sharp-libvips-linux-arm64": "1.2.4" + } + }, + "node_modules/@img/sharp-linux-ppc64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-ppc64/-/sharp-linux-ppc64-0.34.5.tgz", + "integrity": "sha512-7zznwNaqW6YtsfrGGDA6BRkISKAAE1Jo0QdpNYXNMHu2+0dTrPflTLNkpc8l7MUP5M16ZJcUvysVWWrMefZquA==", + "cpu": [ + "ppc64" + ], + "libc": [ + "glibc" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-ppc64": "1.2.4" + } + }, + "node_modules/@img/sharp-linux-riscv64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-riscv64/-/sharp-linux-riscv64-0.34.5.tgz", + "integrity": "sha512-51gJuLPTKa7piYPaVs8GmByo7/U7/7TZOq+cnXJIHZKavIRHAP77e3N2HEl3dgiqdD/w0yUfiJnII77PuDDFdw==", + "cpu": [ + "riscv64" + ], + "libc": [ + "glibc" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-riscv64": "1.2.4" } }, "node_modules/@img/sharp-linux-s390x": { - "version": "0.33.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-s390x/-/sharp-linux-s390x-0.33.5.tgz", - "integrity": "sha512-y/5PCd+mP4CA/sPDKl2961b+C9d+vPAveS33s6Z3zfASk2j5upL6fXVPZi7ztePZ5CuH+1kW8JtvxgbuXHRa4Q==", + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-s390x/-/sharp-linux-s390x-0.34.5.tgz", + "integrity": "sha512-nQtCk0PdKfho3eC5MrbQoigJ2gd1CgddUMkabUj+rBevs8tZ2cULOx46E7oyX+04WGfABgIwmMC0VqieTiR4jg==", "cpu": [ "s390x" ], + "libc": [ + "glibc" + ], "license": "Apache-2.0", "optional": true, "os": [ @@ -595,16 +720,19 @@ "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-libvips-linux-s390x": "1.0.4" + "@img/sharp-libvips-linux-s390x": "1.2.4" } }, "node_modules/@img/sharp-linux-x64": { - "version": "0.33.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-x64/-/sharp-linux-x64-0.33.5.tgz", - "integrity": "sha512-opC+Ok5pRNAzuvq1AG0ar+1owsu842/Ab+4qvU879ippJBHvyY5n2mxF1izXqkPYlGuP/M556uh53jRLJmzTWA==", + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-x64/-/sharp-linux-x64-0.34.5.tgz", + "integrity": "sha512-MEzd8HPKxVxVenwAa+JRPwEC7QFjoPWuS5NZnBt6B3pu7EG2Ge0id1oLHZpPJdn3OQK+BQDiw9zStiHBTJQQQQ==", "cpu": [ "x64" ], + "libc": [ + "glibc" + ], "license": "Apache-2.0", "optional": true, "os": [ @@ -617,16 +745,19 @@ "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-libvips-linux-x64": "1.0.4" + "@img/sharp-libvips-linux-x64": "1.2.4" } }, "node_modules/@img/sharp-linuxmusl-arm64": { - "version": "0.33.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-arm64/-/sharp-linuxmusl-arm64-0.33.5.tgz", - "integrity": "sha512-XrHMZwGQGvJg2V/oRSUfSAfjfPxO+4DkiRh6p2AFjLQztWUuY/o8Mq0eMQVIY7HJ1CDQUJlxGGZRw1a5bqmd1g==", + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-arm64/-/sharp-linuxmusl-arm64-0.34.5.tgz", + "integrity": "sha512-fprJR6GtRsMt6Kyfq44IsChVZeGN97gTD331weR1ex1c1rypDEABN6Tm2xa1wE6lYb5DdEnk03NZPqA7Id21yg==", "cpu": [ "arm64" ], + "libc": [ + "musl" + ], "license": "Apache-2.0", "optional": true, "os": [ @@ -639,16 +770,19 @@ "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-libvips-linuxmusl-arm64": "1.0.4" + "@img/sharp-libvips-linuxmusl-arm64": "1.2.4" } }, "node_modules/@img/sharp-linuxmusl-x64": { - "version": "0.33.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-x64/-/sharp-linuxmusl-x64-0.33.5.tgz", - "integrity": "sha512-WT+d/cgqKkkKySYmqoZ8y3pxx7lx9vVejxW/W4DOFMYVSkErR+w7mf2u8m/y4+xHe7yY9DAXQMWQhpnMuFfScw==", + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-x64/-/sharp-linuxmusl-x64-0.34.5.tgz", + "integrity": "sha512-Jg8wNT1MUzIvhBFxViqrEhWDGzqymo3sV7z7ZsaWbZNDLXRJZoRGrjulp60YYtV4wfY8VIKcWidjojlLcWrd8Q==", "cpu": [ "x64" ], + "libc": [ + "musl" + ], "license": "Apache-2.0", "optional": true, "os": [ @@ -661,20 +795,20 @@ "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-libvips-linuxmusl-x64": "1.0.4" + "@img/sharp-libvips-linuxmusl-x64": "1.2.4" } }, "node_modules/@img/sharp-wasm32": { - "version": "0.33.5", - "resolved": "https://registry.npmjs.org/@img/sharp-wasm32/-/sharp-wasm32-0.33.5.tgz", - "integrity": "sha512-ykUW4LVGaMcU9lu9thv85CbRMAwfeadCJHRsg2GmeRa/cJxsVY9Rbd57JcMxBkKHag5U/x7TSBpScF4U8ElVzg==", + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-wasm32/-/sharp-wasm32-0.34.5.tgz", + "integrity": "sha512-OdWTEiVkY2PHwqkbBI8frFxQQFekHaSSkUIJkwzclWZe64O1X4UlUjqqqLaPbUpMOQk6FBu/HtlGXNblIs0huw==", "cpu": [ "wasm32" ], "license": "Apache-2.0 AND LGPL-3.0-or-later AND MIT", "optional": true, "dependencies": { - "@emnapi/runtime": "^1.2.0" + "@emnapi/runtime": "^1.7.0" }, "engines": { "node": "^18.17.0 || ^20.3.0 || >=21.0.0" @@ -683,10 +817,29 @@ "url": "https://opencollective.com/libvips" } }, + "node_modules/@img/sharp-win32-arm64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-arm64/-/sharp-win32-arm64-0.34.5.tgz", + "integrity": "sha512-WQ3AgWCWYSb2yt+IG8mnC6Jdk9Whs7O0gxphblsLvdhSpSTtmu69ZG1Gkb6NuvxsNACwiPV6cNSZNzt0KPsw7g==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0 AND LGPL-3.0-or-later", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, "node_modules/@img/sharp-win32-ia32": { - "version": "0.33.5", - "resolved": "https://registry.npmjs.org/@img/sharp-win32-ia32/-/sharp-win32-ia32-0.33.5.tgz", - "integrity": "sha512-T36PblLaTwuVJ/zw/LaH0PdZkRz5rd3SmMHX8GSmR7vtNSP5Z6bQkExdSK7xGWyxLw4sUknBuugTelgw2faBbQ==", + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-ia32/-/sharp-win32-ia32-0.34.5.tgz", + "integrity": "sha512-FV9m/7NmeCmSHDD5j4+4pNI8Cp3aW+JvLoXcTUo0IqyjSfAZJ8dIUmijx1qaJsIiU+Hosw6xM5KijAWRJCSgNg==", "cpu": [ "ia32" ], @@ -703,12 +856,13 @@ } }, "node_modules/@img/sharp-win32-x64": { - "version": "0.33.5", - "resolved": "https://registry.npmjs.org/@img/sharp-win32-x64/-/sharp-win32-x64-0.33.5.tgz", - "integrity": "sha512-MpY/o8/8kj+EcnxwvrP4aTJSWw/aZ7JIGR4aBeZkZw5B7/Jn+tY9/VNwtcoGmdT7GfggGIU4kygOMSbYnOrAbg==", + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-x64/-/sharp-win32-x64-0.34.5.tgz", + "integrity": "sha512-+29YMsqY2/9eFEiW93eqWnuLcWcufowXewwSNIT6UwZdUUCrM3oFjMWH/Z6/TMmb4hlFenmfAVbpWeup2jryCw==", "cpu": [ "x64" ], + "license": "Apache-2.0 AND LGPL-3.0-or-later", "optional": true, "os": [ "win32" @@ -774,26 +928,29 @@ } }, "node_modules/@next/env": { - "version": "15.0.4", - "resolved": "https://registry.npmjs.org/@next/env/-/env-15.0.4.tgz", - "integrity": "sha512-WNRvtgnRVDD4oM8gbUcRc27IAhaL4eXQ/2ovGbgLnPGUvdyDr8UdXP4Q/IBDdAdojnD2eScryIDirv0YUCjUVw==" + "version": "15.5.19", + "resolved": "https://registry.npmjs.org/@next/env/-/env-15.5.19.tgz", + "integrity": "sha512-sWWluFvcv5v3Fxznmf2ZfjyoVQt/64oCnYqS90inQWGzMPK1VjvekPiz3OPHKmFT30EnHrjlbyaHLt3M0vWabw==", + "license": "MIT" }, "node_modules/@next/eslint-plugin-next": { - "version": "15.0.4", - "resolved": "https://registry.npmjs.org/@next/eslint-plugin-next/-/eslint-plugin-next-15.0.4.tgz", - "integrity": "sha512-rbsF17XGzHtR7SDWzWpavSfum3/UdnF8bAaisnKwP//si3KWPTedVUsflAdjyK1zW3rweBjbALfKcavFneLGvg==", + "version": "15.5.19", + "resolved": "https://registry.npmjs.org/@next/eslint-plugin-next/-/eslint-plugin-next-15.5.19.tgz", + "integrity": "sha512-Ctwb4qYuMbHN/1oXLlTdMchwG8h8Xzwq+wGZZMgF3o6+uwyBKAI2c96bdOsl+C62PaUD0Jkh+QpNkhUeDlam0Q==", "dev": true, + "license": "MIT", "dependencies": { "fast-glob": "3.3.1" } }, "node_modules/@next/swc-darwin-arm64": { - "version": "15.0.4", - "resolved": "https://registry.npmjs.org/@next/swc-darwin-arm64/-/swc-darwin-arm64-15.0.4.tgz", - "integrity": "sha512-QecQXPD0yRHxSXWL5Ff80nD+A56sUXZG9koUsjWJwA2Z0ZgVQfuy7gd0/otjxoOovPVHR2eVEvPMHbtZP+pf9w==", + "version": "15.5.19", + "resolved": "https://registry.npmjs.org/@next/swc-darwin-arm64/-/swc-darwin-arm64-15.5.19.tgz", + "integrity": "sha512-jx9wWlTKueHKPvVOndyr7WuaevWCkuYqsQ8gC0TMPKAVWG3MhcdMrjfo9tvIZNXd0QOUYXXvAcZ325y8Uq7uzg==", "cpu": [ "arm64" ], + "license": "MIT", "optional": true, "os": [ "darwin" @@ -803,12 +960,13 @@ } }, "node_modules/@next/swc-darwin-x64": { - "version": "15.0.4", - "resolved": "https://registry.npmjs.org/@next/swc-darwin-x64/-/swc-darwin-x64-15.0.4.tgz", - "integrity": "sha512-pb7Bye3y1Og3PlCtnz2oO4z+/b3pH2/HSYkLbL0hbVuTGil7fPen8/3pyyLjdiTLcFJ+ymeU3bck5hd4IPFFCA==", + "version": "15.5.19", + "resolved": "https://registry.npmjs.org/@next/swc-darwin-x64/-/swc-darwin-x64-15.5.19.tgz", + "integrity": "sha512-291KFcsIQ3OenRdiUDFOR6W3wezzH4auENXm1gbm1Bjd4ANMMRgxPrWTUztQN43BnVoVuMnHCrLeECIMwgFKbA==", "cpu": [ "x64" ], + "license": "MIT", "optional": true, "os": [ "darwin" @@ -818,12 +976,16 @@ } }, "node_modules/@next/swc-linux-arm64-gnu": { - "version": "15.0.4", - "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-15.0.4.tgz", - "integrity": "sha512-12oSaBFjGpB227VHzoXF3gJoK2SlVGmFJMaBJSu5rbpaoT5OjP5OuCLuR9/jnyBF1BAWMs/boa6mLMoJPRriMA==", + "version": "15.5.19", + "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-15.5.19.tgz", + "integrity": "sha512-WeH+nelQyyMeE2f8FxBRZNrGipya5zHZV2vjzfCOAYyiI6am+NbnWAAldOBFQBB2w0DjJcsvrKqoFT2b7+5YoA==", "cpu": [ "arm64" ], + "libc": [ + "glibc" + ], + "license": "MIT", "optional": true, "os": [ "linux" @@ -833,12 +995,16 @@ } }, "node_modules/@next/swc-linux-arm64-musl": { - "version": "15.0.4", - "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-15.0.4.tgz", - "integrity": "sha512-QARO88fR/a+wg+OFC3dGytJVVviiYFEyjc/Zzkjn/HevUuJ7qGUUAUYy5PGVWY1YgTzeRYz78akQrVQ8r+sMjw==", + "version": "15.5.19", + "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-15.5.19.tgz", + "integrity": "sha512-5xTOE0lDlDCSSfp+BAif7j17VRRCjWp//ZPZy6NI0QpdrhxtQnsZguSx0xAAZ0c9XZLrLLwCe/XVe5YPrRilKw==", "cpu": [ "arm64" ], + "libc": [ + "musl" + ], + "license": "MIT", "optional": true, "os": [ "linux" @@ -848,12 +1014,16 @@ } }, "node_modules/@next/swc-linux-x64-gnu": { - "version": "15.0.4", - "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-15.0.4.tgz", - "integrity": "sha512-Z50b0gvYiUU1vLzfAMiChV8Y+6u/T2mdfpXPHraqpypP7yIT2UV9YBBhcwYkxujmCvGEcRTVWOj3EP7XW/wUnw==", + "version": "15.5.19", + "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-15.5.19.tgz", + "integrity": "sha512-LTxRmMgqqMv05Had879W00Fm53quiJd3Zuz8h1JSNJ3nGSlbZ/7Tjs1tKyScgN3Au3t3MyPsjPlq60fMmSHLsg==", "cpu": [ "x64" ], + "libc": [ + "glibc" + ], + "license": "MIT", "optional": true, "os": [ "linux" @@ -863,12 +1033,16 @@ } }, "node_modules/@next/swc-linux-x64-musl": { - "version": "15.0.4", - "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-15.0.4.tgz", - "integrity": "sha512-7H9C4FAsrTAbA/ENzvFWsVytqRYhaJYKa2B3fyQcv96TkOGVMcvyS6s+sj4jZlacxxTcn7ygaMXUPkEk7b78zw==", + "version": "15.5.19", + "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-15.5.19.tgz", + "integrity": "sha512-eoNQSpA5PQfB9wBO4RA47MTDXWz1fizy9Y3Z6e4DetYIF3dvjuu8sj7aIGn/bFCU6lnFzTK34NtCaffP4NsQ7Q==", "cpu": [ "x64" ], + "libc": [ + "musl" + ], + "license": "MIT", "optional": true, "os": [ "linux" @@ -878,12 +1052,13 @@ } }, "node_modules/@next/swc-win32-arm64-msvc": { - "version": "15.0.4", - "resolved": "https://registry.npmjs.org/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-15.0.4.tgz", - "integrity": "sha512-Z/v3WV5xRaeWlgJzN9r4PydWD8sXV35ywc28W63i37G2jnUgScA4OOgS8hQdiXLxE3gqfSuHTicUhr7931OXPQ==", + "version": "15.5.19", + "resolved": "https://registry.npmjs.org/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-15.5.19.tgz", + "integrity": "sha512-6UNt2dFuCHOe446sm/Kp69nUe8/wIhnh9bm6Xcqw4qEWCOppLMOvhTBVgvM7invVUNr4SPpP6NOQsACtn2IN9Q==", "cpu": [ "arm64" ], + "license": "MIT", "optional": true, "os": [ "win32" @@ -893,12 +1068,13 @@ } }, "node_modules/@next/swc-win32-x64-msvc": { - "version": "15.0.4", - "resolved": "https://registry.npmjs.org/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-15.0.4.tgz", - "integrity": "sha512-NGLchGruagh8lQpDr98bHLyWJXOBSmkEAfK980OiNBa7vNm6PsNoPvzTfstT78WyOeMRQphEQ455rggd7Eo+Dw==", + "version": "15.5.19", + "resolved": "https://registry.npmjs.org/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-15.5.19.tgz", + "integrity": "sha512-PhmojAHyqMne56HBLGu9dhDnHPuFmEjrXSQMM/nW0J6j849lk3ESrVtqNJcCk8CKOV7brpTTbaYAjwKPzKM69w==", "cpu": [ "x64" ], + "license": "MIT", "optional": true, "os": [ "win32" @@ -1879,17 +2055,13 @@ "integrity": "sha512-TvZbIpeKqGQQ7X0zSCvPH9riMSFQFSggnfBjFZ1mEoILW+UuXCKwOoPcgjMwiUtRqFZ8jWhPJc4um14vC6I4ag==", "dev": true }, - "node_modules/@swc/counter": { - "version": "0.1.3", - "resolved": "https://registry.npmjs.org/@swc/counter/-/counter-0.1.3.tgz", - "integrity": "sha512-e2BR4lsJkkRlKZ/qCHPw9ZaSxc0MVUd7gtbtaB7aMvHeJVYe8sOB8DBZkP2DtISHGSku9sCK6T6cnY0CtXrOCQ==" - }, "node_modules/@swc/helpers": { - "version": "0.5.13", - "resolved": "https://registry.npmjs.org/@swc/helpers/-/helpers-0.5.13.tgz", - "integrity": "sha512-UoKGxQ3r5kYI9dALKJapMmuK+1zWM/H17Z1+iwnNmzcJRnfFuevZs375TA5rW31pu4BS4NoSy1fRsexDXfWn5w==", + "version": "0.5.15", + "resolved": "https://registry.npmjs.org/@swc/helpers/-/helpers-0.5.15.tgz", + "integrity": "sha512-JQ5TuMi45Owi4/BIMAJBoSQoOJu12oOk/gADqlcUL9JEdHB8vyjUSsxqeNXnmXHjYKMi2WcYtezGEEhqUI/E2g==", + "license": "Apache-2.0", "dependencies": { - "tslib": "^2.4.0" + "tslib": "^2.8.0" } }, "node_modules/@tanstack/react-table": { @@ -3050,17 +3222,6 @@ "node": "^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7" } }, - "node_modules/busboy": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/busboy/-/busboy-1.6.0.tgz", - "integrity": "sha512-8SFQbg/0hQ9xy3UNTB0YEnsNBbWfhf7RtnzpL7TkBiTBRfrQ9Fxcnz7VJsleJpyp6rVLvXiuORqjlHi5q+PYuA==", - "dependencies": { - "streamsearch": "^1.1.0" - }, - "engines": { - "node": ">=10.16.0" - } - }, "node_modules/call-bind": { "version": "1.0.9", "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.9.tgz", @@ -3236,24 +3397,11 @@ "node": ">=6" } }, - "node_modules/color": { - "version": "4.2.3", - "resolved": "https://registry.npmjs.org/color/-/color-4.2.3.tgz", - "integrity": "sha512-1rXeuUUiGGrykh+CeBdu5Ie7OJwinCgQY0bc7GCRxy5xVHy+moaqkpL/jqQq0MtQOeYcrqEz4abc5f0KtU7W4A==", - "optional": true, - "dependencies": { - "color-convert": "^2.0.1", - "color-string": "^1.9.0" - }, - "engines": { - "node": ">=12.5.0" - } - }, "node_modules/color-convert": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", - "devOptional": true, + "dev": true, "dependencies": { "color-name": "~1.1.4" }, @@ -3265,17 +3413,7 @@ "version": "1.1.4", "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", - "devOptional": true - }, - "node_modules/color-string": { - "version": "1.9.1", - "resolved": "https://registry.npmjs.org/color-string/-/color-string-1.9.1.tgz", - "integrity": "sha512-shrVawQFojnZv6xM40anx4CkoDP+fZsw/ZerEMsW/pyzsRbElpsL/DBVW7q3ExxwusdNXI3lXpuhEZkzs8p5Eg==", - "optional": true, - "dependencies": { - "color-name": "^1.0.0", - "simple-swizzle": "^0.2.2" - } + "dev": true }, "node_modules/commander": { "version": "4.1.1", @@ -3606,6 +3744,7 @@ "version": "2.1.2", "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", + "license": "Apache-2.0", "optional": true, "engines": { "node": ">=8" @@ -3929,12 +4068,13 @@ } }, "node_modules/eslint-config-next": { - "version": "15.0.4", - "resolved": "https://registry.npmjs.org/eslint-config-next/-/eslint-config-next-15.0.4.tgz", - "integrity": "sha512-97mLaAhbJKVQYXUBBrenRtEUAA6bNDPxWfaFEd6mEhKfpajP4wJrW4l7BUlHuYWxR8oQa9W014qBJpumpJQwWA==", + "version": "15.5.19", + "resolved": "https://registry.npmjs.org/eslint-config-next/-/eslint-config-next-15.5.19.tgz", + "integrity": "sha512-UZwkuhBCNxVZfo93MSHRDOVNWXooJJGcAUyTAVIp0+9QFhH4SqJxWY0s6Mk9C2kMi777HPMn3dseOrZshWpG9Q==", "dev": true, + "license": "MIT", "dependencies": { - "@next/eslint-plugin-next": "15.0.4", + "@next/eslint-plugin-next": "15.5.19", "@rushstack/eslint-patch": "^1.10.3", "@typescript-eslint/eslint-plugin": "^5.4.2 || ^6.0.0 || ^7.0.0 || ^8.0.0", "@typescript-eslint/parser": "^5.4.2 || ^6.0.0 || ^7.0.0 || ^8.0.0", @@ -3942,7 +4082,7 @@ "eslint-import-resolver-typescript": "^3.5.2", "eslint-plugin-import": "^2.31.0", "eslint-plugin-jsx-a11y": "^6.10.0", - "eslint-plugin-react": "^7.35.0", + "eslint-plugin-react": "^7.37.0", "eslint-plugin-react-hooks": "^5.0.0" }, "peerDependencies": { @@ -4279,6 +4419,7 @@ "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.1.tgz", "integrity": "sha512-kNFPyjhh5cKjrUltxs+wFx+ZkbRaxxmZ+X0ZU31SOsxCEtP9VPgtq2teZw1DebupL5GmDaNQ6yKMMVcM41iqDg==", "dev": true, + "license": "MIT", "dependencies": { "@nodelib/fs.stat": "^2.0.2", "@nodelib/fs.walk": "^1.2.3", @@ -4295,6 +4436,7 @@ "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", "dev": true, + "license": "ISC", "dependencies": { "is-glob": "^4.0.1" }, @@ -4808,12 +4950,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/is-arrayish": { - "version": "0.3.4", - "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.3.4.tgz", - "integrity": "sha512-m6UrgzFVUYawGBh1dUsWR5M2Clqic9RVXC/9f8ceNlv2IcO9j9J/z8UoCLPqtsPBFNzEpfR3xftohbfqDx8EQA==", - "optional": true - }, "node_modules/is-async-function": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/is-async-function/-/is-async-function-2.1.1.tgz", @@ -5543,15 +5679,13 @@ "dev": true }, "node_modules/next": { - "version": "15.0.4", - "resolved": "https://registry.npmjs.org/next/-/next-15.0.4.tgz", - "integrity": "sha512-nuy8FH6M1FG0lktGotamQDCXhh5hZ19Vo0ht1AOIQWrYJLP598TIUagKtvJrfJ5AGwB/WmDqkKaKhMpVifvGPA==", - "deprecated": "This version has a security vulnerability. Please upgrade to a patched version. See https://nextjs.org/blog/CVE-2025-66478 for more details.", + "version": "15.5.19", + "resolved": "https://registry.npmjs.org/next/-/next-15.5.19.tgz", + "integrity": "sha512-xNOW6tYshGX1/Oi3F8uuk4gpDeWsSUE/1Z0G5uUMekIxaQ0xc03UXd9II0VQHYMWviMeA0OHpJFAKsHf8bTYVg==", + "license": "MIT", "dependencies": { - "@next/env": "15.0.4", - "@swc/counter": "0.1.3", - "@swc/helpers": "0.5.13", - "busboy": "1.6.0", + "@next/env": "15.5.19", + "@swc/helpers": "0.5.15", "caniuse-lite": "^1.0.30001579", "postcss": "8.4.31", "styled-jsx": "5.1.6" @@ -5563,22 +5697,22 @@ "node": "^18.18.0 || ^19.8.0 || >= 20.0.0" }, "optionalDependencies": { - "@next/swc-darwin-arm64": "15.0.4", - "@next/swc-darwin-x64": "15.0.4", - "@next/swc-linux-arm64-gnu": "15.0.4", - "@next/swc-linux-arm64-musl": "15.0.4", - "@next/swc-linux-x64-gnu": "15.0.4", - "@next/swc-linux-x64-musl": "15.0.4", - "@next/swc-win32-arm64-msvc": "15.0.4", - "@next/swc-win32-x64-msvc": "15.0.4", - "sharp": "^0.33.5" + "@next/swc-darwin-arm64": "15.5.19", + "@next/swc-darwin-x64": "15.5.19", + "@next/swc-linux-arm64-gnu": "15.5.19", + "@next/swc-linux-arm64-musl": "15.5.19", + "@next/swc-linux-x64-gnu": "15.5.19", + "@next/swc-linux-x64-musl": "15.5.19", + "@next/swc-win32-arm64-msvc": "15.5.19", + "@next/swc-win32-x64-msvc": "15.5.19", + "sharp": "^0.34.3" }, "peerDependencies": { "@opentelemetry/api": "^1.1.0", - "@playwright/test": "^1.41.2", + "@playwright/test": "^1.51.1", "babel-plugin-react-compiler": "*", - "react": "^18.2.0 || 19.0.0-rc-66855b96-20241106 || ^19.0.0", - "react-dom": "^18.2.0 || 19.0.0-rc-66855b96-20241106 || ^19.0.0", + "react": "^18.2.0 || 19.0.0-rc-de68d2f4-20241204 || ^19.0.0", + "react-dom": "^18.2.0 || 19.0.0-rc-de68d2f4-20241204 || ^19.0.0", "sass": "^1.3.0" }, "peerDependenciesMeta": { @@ -6722,15 +6856,16 @@ } }, "node_modules/sharp": { - "version": "0.33.5", - "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.33.5.tgz", - "integrity": "sha512-haPVm1EkS9pgvHrQ/F3Xy+hgcuMV0Wm9vfIBSiwZ05k+xgb0PkBQpGsAA/oWdDobNaZTH5ppvHtzCFbnSEwHVw==", + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.34.5.tgz", + "integrity": "sha512-Ou9I5Ft9WNcCbXrU9cMgPBcCK8LiwLqcbywW3t4oDV37n1pzpuNLsYiAV8eODnjbtQlSDwZ2cUEeQz4E54Hltg==", "hasInstallScript": true, + "license": "Apache-2.0", "optional": true, "dependencies": { - "color": "^4.2.3", - "detect-libc": "^2.0.3", - "semver": "^7.6.3" + "@img/colour": "^1.0.0", + "detect-libc": "^2.1.2", + "semver": "^7.7.3" }, "engines": { "node": "^18.17.0 || ^20.3.0 || >=21.0.0" @@ -6739,25 +6874,30 @@ "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-darwin-arm64": "0.33.5", - "@img/sharp-darwin-x64": "0.33.5", - "@img/sharp-libvips-darwin-arm64": "1.0.4", - "@img/sharp-libvips-darwin-x64": "1.0.4", - "@img/sharp-libvips-linux-arm": "1.0.5", - "@img/sharp-libvips-linux-arm64": "1.0.4", - "@img/sharp-libvips-linux-s390x": "1.0.4", - "@img/sharp-libvips-linux-x64": "1.0.4", - "@img/sharp-libvips-linuxmusl-arm64": "1.0.4", - "@img/sharp-libvips-linuxmusl-x64": "1.0.4", - "@img/sharp-linux-arm": "0.33.5", - "@img/sharp-linux-arm64": "0.33.5", - "@img/sharp-linux-s390x": "0.33.5", - "@img/sharp-linux-x64": "0.33.5", - "@img/sharp-linuxmusl-arm64": "0.33.5", - "@img/sharp-linuxmusl-x64": "0.33.5", - "@img/sharp-wasm32": "0.33.5", - "@img/sharp-win32-ia32": "0.33.5", - "@img/sharp-win32-x64": "0.33.5" + "@img/sharp-darwin-arm64": "0.34.5", + "@img/sharp-darwin-x64": "0.34.5", + "@img/sharp-libvips-darwin-arm64": "1.2.4", + "@img/sharp-libvips-darwin-x64": "1.2.4", + "@img/sharp-libvips-linux-arm": "1.2.4", + "@img/sharp-libvips-linux-arm64": "1.2.4", + "@img/sharp-libvips-linux-ppc64": "1.2.4", + "@img/sharp-libvips-linux-riscv64": "1.2.4", + "@img/sharp-libvips-linux-s390x": "1.2.4", + "@img/sharp-libvips-linux-x64": "1.2.4", + "@img/sharp-libvips-linuxmusl-arm64": "1.2.4", + "@img/sharp-libvips-linuxmusl-x64": "1.2.4", + "@img/sharp-linux-arm": "0.34.5", + "@img/sharp-linux-arm64": "0.34.5", + "@img/sharp-linux-ppc64": "0.34.5", + "@img/sharp-linux-riscv64": "0.34.5", + "@img/sharp-linux-s390x": "0.34.5", + "@img/sharp-linux-x64": "0.34.5", + "@img/sharp-linuxmusl-arm64": "0.34.5", + "@img/sharp-linuxmusl-x64": "0.34.5", + "@img/sharp-wasm32": "0.34.5", + "@img/sharp-win32-arm64": "0.34.5", + "@img/sharp-win32-ia32": "0.34.5", + "@img/sharp-win32-x64": "0.34.5" } }, "node_modules/shebang-command": { @@ -6866,15 +7006,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/simple-swizzle": { - "version": "0.2.4", - "resolved": "https://registry.npmjs.org/simple-swizzle/-/simple-swizzle-0.2.4.tgz", - "integrity": "sha512-nAu1WFPQSMNr2Zn9PGSZK9AGn4t/y97lEm+MXTtUDwfP0ksAIX4nO+6ruD9Jwut4C49SB1Ws+fbXsm/yScWOHw==", - "optional": true, - "dependencies": { - "is-arrayish": "^0.3.1" - } - }, "node_modules/sonner": { "version": "1.7.4", "resolved": "https://registry.npmjs.org/sonner/-/sonner-1.7.4.tgz", @@ -6919,14 +7050,6 @@ "node": ">= 0.4" } }, - "node_modules/streamsearch": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/streamsearch/-/streamsearch-1.1.0.tgz", - "integrity": "sha512-Mcc5wHehp9aXz1ax6bZUyY5afg9u2rv5cqQI3mRrYkGC8rW2hM02jWuwjtL++LS5qinSyhj2QfLyNsuc+VsExg==", - "engines": { - "node": ">=10.0.0" - } - }, "node_modules/string-width": { "version": "7.2.0", "resolved": "https://registry.npmjs.org/string-width/-/string-width-7.2.0.tgz", diff --git a/package.json b/package.json index 8aee1d3..261ff72 100644 --- a/package.json +++ b/package.json @@ -7,8 +7,8 @@ "dev:start": "concurrently -n AI,BROWSE,NEXT -c cyan,magenta,green \"npm run dev:rust\" \"npm run dev:browser-use\" \"npm run dev:next\"", "dev:next": "next dev -p 3006", "dev:precheck": "powershell -NoProfile -Command \"Get-NetTCPConnection -State Listen | Where-Object { $_.LocalPort -in 3001,3006,3008 } | ForEach-Object { Stop-Process -Id $_.OwningProcess -Force -ErrorAction SilentlyContinue; Write-Host ('Freed port '+$_.LocalPort) }\"", - "dev:ollama": "powershell -NoProfile -Command \"if (-not (Get-Process ollama -ErrorAction SilentlyContinue)) { Start-Process ollama -ArgumentList 'serve' -WindowStyle Hidden; Start-Sleep 3 }; exit 0\"", - "dev:rust": "cd rust-ai && cargo run", + "dev:ollama": "powershell -NoProfile -Command \"$ollama = if (Get-Command ollama -ErrorAction SilentlyContinue) { 'ollama' } else { Join-Path $env:LOCALAPPDATA 'Programs\\Ollama\\ollama.exe' }; if (-not (Get-Process ollama -ErrorAction SilentlyContinue)) { Start-Process $ollama -ArgumentList 'serve' -WindowStyle Hidden; Start-Sleep 3 }; exit 0\"", + "dev:rust": "node ai-server/index.mjs", "dev:browser-use": "cd browser-use-service && python main.py", "build": "next build", "start": "next start -p 3006", diff --git a/rust-ai/.cargo/config.toml b/rust-ai/.cargo/config.toml new file mode 100644 index 0000000..6157e14 --- /dev/null +++ b/rust-ai/.cargo/config.toml @@ -0,0 +1,2 @@ +[target.x86_64-pc-windows-gnu] +linker = "C:\\Users\\Hannah Kaur Bagga\\.rustup\\toolchains\\stable-x86_64-pc-windows-gnu\\lib\\rustlib\\x86_64-pc-windows-gnu\\bin\\rust-lld.exe" From ff56cea4b8916e4ed2b605bced2d11d58b6b09c4 Mon Sep 17 00:00:00 2001 From: Ace Date: Tue, 23 Jun 2026 14:18:18 +0200 Subject: [PATCH 2/3] feat: add Facebook account tracking and management - Introduced new migration for `facebook_accounts` and `facebook_scrape_logs` tables. - Updated the scraping logic to handle Facebook accounts, including success and failure tracking. - Implemented API endpoints for managing Facebook accounts and their scrape logs. - Enhanced user permissions to restrict access to Facebook account management to admins and super admins. - Added a dialog component for displaying and managing Facebook accounts in the UI. - Updated lead fetching logic to include user role checks for assignment and access. --- browser-use-service/err.txt | Bin 1092 -> 0 bytes browser-use-service/main.py | 303 ++++++++++-------- browser-use-service/out.txt | 0 browser-use-service/run_log.txt | 7 - browser-use-service/scrape_log.txt | 6 - browser-use-service/stderr.txt | 4 - browser-use-service/stdout.txt | 5 - database/DESIGN_DECISIONS.md | 8 +- database/migrations/010_facebook_accounts.sql | 34 ++ database/migrations/run_all.sql | 3 + rust-ai/src/main.rs | 240 +++++++++++--- src/app/api/dashboard/route.ts | 13 +- src/app/api/leads/[id]/notes/route.ts | 19 ++ src/app/api/leads/[id]/route.ts | 5 + src/app/api/leads/route.ts | 11 +- src/app/api/notifications/route.ts | 3 +- .../settings/facebook/accounts/[id]/route.ts | 61 ++++ .../api/settings/facebook/accounts/route.ts | 66 ++++ src/app/api/settings/facebook/logs/route.ts | 41 +++ src/app/api/system/monitor/route.ts | 3 + src/app/api/users/avatar/route.ts | 15 + src/app/api/users/route.ts | 3 + src/app/api/users/search/route.ts | 3 + src/components/layout/sidebar.tsx | 39 +++ .../settings/facebook-accounts-dialog.tsx | 102 ++++++ 25 files changed, 778 insertions(+), 216 deletions(-) delete mode 100644 browser-use-service/err.txt delete mode 100644 browser-use-service/out.txt delete mode 100644 browser-use-service/run_log.txt delete mode 100644 browser-use-service/scrape_log.txt delete mode 100644 browser-use-service/stderr.txt delete mode 100644 browser-use-service/stdout.txt create mode 100644 database/migrations/010_facebook_accounts.sql create mode 100644 src/app/api/settings/facebook/accounts/[id]/route.ts create mode 100644 src/app/api/settings/facebook/accounts/route.ts create mode 100644 src/app/api/settings/facebook/logs/route.ts create mode 100644 src/components/settings/facebook-accounts-dialog.tsx diff --git a/browser-use-service/err.txt b/browser-use-service/err.txt deleted file mode 100644 index a1f2d3d461a83da4457c6c43854eceb1c767d8a7..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1092 zcmd6m%TB^z5QWd$#CK?1K@`PHBI%A0Nl3inZ2`t0R;iIv3!-d%boHC54K9dr;lfP% zU#Dlze`e<6J=IJT4Kz}sKp|gChidA`*8zL8Q+#^L7$e=XA2Vl~`WPr>JyM_D7z@W7 zSbfgM?$OqPuC)jI3$+)X-1uKF_#69*o_$sM+D~C;Z|G`BtU&8-b7pV)A7+e-yqIPU z*jirh(I`W`78>8NE_}Yvmfu6}CS){q^tyQdscgee}=;a3e*494V+2C#h NR-h;FO_{R4@e|+6x6}Xt diff --git a/browser-use-service/main.py b/browser-use-service/main.py index 726a364..12bc69a 100644 --- a/browser-use-service/main.py +++ b/browser-use-service/main.py @@ -1,9 +1,7 @@ -import os, json, asyncio, re, shutil, sqlite3, traceback, urllib.parse, random, time, logging +import os, json, asyncio, re, shutil, sqlite3, urllib.parse, random, logging from datetime import datetime, timedelta -from bs4 import BeautifulSoup -import httpx -from fastapi import FastAPI -from pydantic import BaseModel +from fastapi import FastAPI, Query +from fastapi.middleware.cors import CORSMiddleware import uvicorn from playwright.async_api import async_playwright @@ -11,22 +9,20 @@ logging.basicConfig(level=logging.INFO, format='%(asctime)s [%(levelname)s] %(me logger = logging.getLogger(__name__) app = FastAPI() +app.add_middleware( + CORSMiddleware, + allow_origins=["http://localhost:3006", "http://127.0.0.1:3006"], + allow_methods=["POST"], + allow_headers=["*"], +) PORT = int(os.getenv("PORT", "3008")) OLLAMA_URL = os.getenv("OLLAMA_URL", "http://localhost:11434") CLASSIFY_MODEL = os.getenv("CLASSIFY_MODEL", "dolphin-llama3:8b") FX_PROFILE = os.getenv('FX_PROFILE', '') -FX_COOKIE_DB = os.path.join(FX_PROFILE, 'cookies.sqlite') if FX_PROFILE else '' -HEADERS = { - 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36', - 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', -} - -last_scrape_time: datetime | None = None - -STRICT_KEYWORDS = [ +BROAD_KEYWORDS = [ "website", "web design", "web develop", "web dev", "build my website", "build a website", "create a website", "need web", "looking for web", "new website", @@ -34,9 +30,6 @@ STRICT_KEYWORDS = [ "need a website", "my website", "business website", "need a designer", "help with my website", "redesign", "update my website", -] - -BROAD_KEYWORDS = STRICT_KEYWORDS + [ "looking for", "need a", "need an", "looking to", "help me build", "create my", "build me", "design my", "make me a", "would like", @@ -44,10 +37,6 @@ BROAD_KEYWORDS = STRICT_KEYWORDS + [ "want someone", "need help with", ] -def kw_match_strict(text: str) -> bool: - t = text.lower() - return any(kw in t for kw in STRICT_KEYWORDS) - def kw_match(text: str) -> bool: t = text.lower() return any(kw in t for kw in BROAD_KEYWORDS) @@ -67,14 +56,27 @@ FB_SEARCHES = [ "need a site for my business", ] -async def get_fb_cookies(): - if not FX_COOKIE_DB or not os.path.exists(FX_COOKIE_DB): - logger.warning("FX_COOKIE_DB not found or FX_PROFILE not set") +VIEWPORTS = [ + {'width': 1280, 'height': 800}, + {'width': 1366, 'height': 768}, + {'width': 1440, 'height': 900}, + {'width': 1536, 'height': 864}, + {'width': 1920, 'height': 1080}, +] + +async def get_fb_cookies(profile_path: str | None = None): + cookie_db_path = profile_path or FX_PROFILE + if not cookie_db_path: + logger.warning("No profile path provided") return [] - tmp = os.path.join(os.path.dirname(FX_COOKIE_DB), f'cookies_tmp_{os.getpid()}.sqlite') + cookie_db = os.path.join(cookie_db_path, 'cookies.sqlite') + if not os.path.exists(cookie_db): + logger.warning("Cookie DB not found at %s", cookie_db) + return [] + tmp = os.path.join(os.path.dirname(cookie_db), f'cookies_tmp_{os.getpid()}.sqlite') for attempt in range(3): try: - shutil.copy2(FX_COOKIE_DB, tmp) + shutil.copy2(cookie_db, tmp) break except PermissionError: if attempt < 2: @@ -182,35 +184,95 @@ def _extract_posts_from_text(raw: str, url: str) -> list[dict]: }) return posts +async def human_scroll(page, steps: int = None, total_delay: float = None): + steps = steps or random.randint(2, 5) + total_delay = total_delay or random.uniform(6, 18) + step_delay = total_delay / steps + for i in range(steps): + scroll_dist = random.randint(200, 600) + await page.evaluate(f"window.scrollBy(0, {scroll_dist})") + await page.wait_for_timeout(int(step_delay * 1000)) + if random.random() < 0.3: + await page.evaluate(f"window.scrollBy(0, -{random.randint(100, 300)})") + await page.wait_for_timeout(random.randint(1000, 3000)) + +async def random_idle(page): + action = random.random() + if action < 0.15: + try: + elems = await page.query_selector_all('a, span, div[role="button"]') + if elems: + el = random.choice(elems[:20]) + box = await el.bounding_box() + if box: + x = box['x'] + box['width'] / 2 + random.randint(-20, 20) + y = box['y'] + box['height'] / 2 + random.randint(-10, 10) + await page.mouse.move(x, y, steps=random.randint(20, 40)) + await page.wait_for_timeout(random.randint(500, 2000)) + except Exception: + pass + async def search_facebook(page, query: str) -> list[dict]: url = f'https://www.facebook.com/search/posts/?q={urllib.parse.quote(query)}' try: await page.goto(url, wait_until='domcontentloaded', timeout=30000) - await page.wait_for_timeout(6000) - except Exception as e: - logger.warning("Facebook search navigation failed for '%s': %s", query, e) - return [] + await page.wait_for_timeout(random.randint(3000, 8000)) + + await human_scroll(page, steps=random.randint(2, 4), total_delay=random.uniform(6, 15)) + + # Accidental overscroll: 10% chance to jump to bottom and back + if random.random() < 0.1: + await page.evaluate("window.scrollTo(0, document.body.scrollHeight)") + await page.wait_for_timeout(random.randint(1000, 3000)) + await page.evaluate("window.scrollBy(0, -300)") + await page.wait_for_timeout(random.randint(1000, 3000)) + + if random.random() < 0.2: + await page.evaluate("window.scrollTo(0, 0)") + await page.wait_for_timeout(random.randint(2000, 5000)) + + if random.random() < 0.3: + await random_idle(page) - try: raw = await page.evaluate('document.body.innerText') except Exception as e: - logger.warning("Failed to evaluate page text: %s", e) + logger.warning("Facebook search failed: %s", e) return [] return _extract_posts_from_text(raw, url) -async def scrape_facebook() -> list[dict]: - all_posts = [] - fb_cookies = await get_fb_cookies() +def cleanup_chrome(): + import subprocess, signal + try: + subprocess.run(["taskkill", "/F", "/IM", "chrome-headless-shell.exe"], capture_output=True, timeout=5) + except Exception: + pass + +async def scrape_facebook(profile_path: str | None = None, force: bool = False) -> dict: + cleanup_chrome() + fb_cookies = await get_fb_cookies(profile_path) if not fb_cookies: logger.warning("No Facebook cookies available") - return [] + return {"success": False, "leads": [], "flagged": False, "flag_reason": None, "error": "No cookies available"} try: async with async_playwright() as pw: - browser = await pw.chromium.launch(headless=True) + browser = await pw.chromium.launch( + headless=True, + args=['--disable-blink-features=AutomationControlled'] + ) + viewport = random.choice(VIEWPORTS) context = await browser.new_context( user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:150.0) Gecko/20100101 Firefox/150.0', - viewport={'width': 1280, 'height': 800}, + viewport=viewport, ) + + await context.add_init_script(""" + Object.defineProperty(navigator, 'webdriver', { get: () => false }); + Object.defineProperty(navigator, 'languages', { get: () => ['en-US', 'en'] }); + Object.defineProperty(navigator, 'platform', { get: () => 'Win32' }); + window.chrome = { runtime: {} }; + Object.defineProperty(navigator, 'hardwareConcurrency', { get: () => 8 }); + Object.defineProperty(navigator, 'deviceMemory', { get: () => 8 }); + """) for c in fb_cookies: try: await context.add_cookies([c]) @@ -218,36 +280,77 @@ async def scrape_facebook() -> list[dict]: logger.warning("Failed to inject cookie %s: %s", c.get('name'), e) page = await context.new_page() + # Navigate through google first for a legitimate Referer header + await page.goto('https://www.google.com/', wait_until='domcontentloaded', timeout=15000) + await page.wait_for_timeout(random.randint(1000, 3000)) await page.goto('https://www.facebook.com/', wait_until='domcontentloaded', timeout=30000) - await page.wait_for_timeout(5000) + await page.wait_for_timeout(random.randint(3000, 8000)) url = page.url if '/login' in url.lower(): - logger.warning("Facebook login page detected — cookies may be expired") - return [] + logger.warning("Facebook login page detected — account flagged") + await browser.close() + return {"success": False, "leads": [], "flagged": True, "flag_reason": "login_page", "error": "Facebook login page detected"} - for query in FB_SEARCHES[:6]: + # Browse feed like a human before searching + await human_scroll(page, steps=random.randint(2, 4), total_delay=random.uniform(8, 20)) + if random.random() < 0.25: + await page.evaluate("window.scrollTo(0, 0)") + await page.wait_for_timeout(random.randint(2000, 5000)) + await human_scroll(page, steps=random.randint(1, 2)) + + # False start: 30% chance to idle and leave without scraping (skipped when force=true) + if not force and random.random() < 0.3: + await page.wait_for_timeout(random.randint(8000, 20000)) + await browser.close() + return {"success": True, "leads": [], "flagged": False, "flag_reason": None, "error": None} + + all_posts = [] + searches = random.sample(FB_SEARCHES, k=random.randint(3, 6)) + for i, query in enumerate(searches): try: posts = await search_facebook(page, query) all_posts.extend(posts) - delay = random.uniform(3, 7) + # Between searches: random break with possible small scroll + if random.random() < 0.4: + await page.evaluate(f"window.scrollBy(0, {random.randint(-300, 300)})") + delay = random.uniform(8, 25) await page.wait_for_timeout(int(delay * 1000)) + # Tab switch: 15% chance after 2nd-3rd search + if i == random.randint(1, 2) and random.random() < 0.15: + new_page = await context.new_page() + await new_page.goto('https://www.messenger.com/', wait_until='domcontentloaded', timeout=15000) + await new_page.wait_for_timeout(random.randint(3000, 8000)) + await new_page.close() + await page.wait_for_timeout(random.randint(1000, 3000)) except Exception as e: logger.warning("Facebook search '%s' failed: %s", query, e) + + # Idle before closing — human would pause + if random.random() < 0.5: + await page.wait_for_timeout(random.randint(3000, 10000)) + + await browser.close() + + seen = set() + deduped = [] + for p in all_posts: + key = p.get('content', '')[:100] + if key not in seen: + seen.add(key) + deduped.append(p) + + leads = deduped[:20] + if leads: + leads = await classify_leads(leads) + + return {"success": True, "leads": leads[:15], "flagged": False, "flag_reason": None, "error": None} except Exception as e: logger.error("Facebook scrape failed: %s", e) - return [] - - seen = set() - deduped = [] - for p in all_posts: - key = p.get('content', '')[:100] - if key not in seen: - seen.add(key) - deduped.append(p) - return deduped[:20] + return {"success": False, "leads": [], "flagged": False, "flag_reason": None, "error": str(e)} async def ask_ollama(prompt: str) -> str: + import httpx async with httpx.AsyncClient(timeout=120) as c: r = await c.post(f"{OLLAMA_URL}/api/chat", json={ "model": CLASSIFY_MODEL, @@ -262,42 +365,6 @@ async def ask_ollama(prompt: str) -> str: data = r.json() return data["message"]["content"] -async def scrape_warriorforum() -> list[dict]: - results = [] - try: - async with httpx.AsyncClient(headers=HEADERS, timeout=15, follow_redirects=True) as c: - r = await c.get('https://www.warriorforum.com/wanted-members-looking-hire-you/') - soup = BeautifulSoup(r.text, 'lxml') - for row in soup.find_all('td', class_='FlexTable-item--title'): - a = row.find('a', href=True) - if not a: - continue - href = a['href'] - title = a.text.strip() - if not title or len(title) < 15 or not kw_match_strict(title): - continue - author_div = row.find('div', class_='FlexTable-item-author') - author = author_div.get_text(strip=True).lstrip('by') if author_div else '' - date_str = '' - date_div = row.find('div', class_=lambda c: c and 'media--available' in c) - if date_div: - txt = date_div.get_text(strip=True) - m = re.search(r'(\d{10})', txt) - if m: - date_str = datetime.utcfromtimestamp(int(m.group(1))).strftime('%Y-%m-%d') - if not href.startswith('http'): - href = 'https://www.warriorforum.com' + href - results.append({ - "title": title, "url": href, - "author": author, - "content": title[:300], - "source": "warriorforum", - "date": date_str - }) - except Exception: - logger.error("WarriorForum scrape failed", exc_info=True) - return results - async def classify_leads(results: list[dict]) -> list[dict]: if not results: return [] @@ -320,13 +387,10 @@ Return a JSON array like ["yes","no","yes"] matching the order above.""" try: raw = await ask_ollama(prompt) raw = raw.strip() - # Strip markdown code fences if raw.startswith("```"): - # Remove opening fence first_nl = raw.find('\n') if first_nl != -1: raw = raw[first_nl + 1:] - # Remove closing fence if raw.endswith("```"): raw = raw[:-3] raw = raw.strip() @@ -339,18 +403,25 @@ Return a JSON array like ["yes","no","yes"] matching the order above.""" filtered.append(results[i]) if filtered: return filtered[:10] - # AI successfully classified but returned all "no" — respect that decision logger.info("AI classified all %d items as NOT LEAD — returning empty", len(results)) return [] except Exception as e: logger.warning("AI classification failed, falling back to keyword filter: %s", e) - # Fallback: only use keyword filter when AI failed if not ai_succeeded: + strict_keywords = [ + "website", "web design", "web develop", "web dev", + "build my website", "build a website", "create a website", + "need web", "looking for web", "new website", + "landing page", "wordpress", + "need a website", "my website", "business website", + "need a designer", "help with my website", + "redesign", "update my website", + ] filtered = [] for r in results: t = r['title'].lower() - if not kw_match_strict(t): + if not any(kw in t for kw in strict_keywords): continue if any(kw in t for kw in ['i build', 'i offer', 'i create', 'i am a', 'web developer available', 'affordable web', 'web hosting', 'free website', @@ -365,46 +436,10 @@ Return a JSON array like ["yes","no","yes"] matching the order above.""" async def health(): return {"status": "ok"} -class ScrapeRequest(BaseModel): - query: str = "" - -@app.post("/scrape/requests") -async def scrape_requests(req: ScrapeRequest): - global last_scrape_time - now = datetime.now() - if last_scrape_time and (now - last_scrape_time) < timedelta(seconds=15): - return {"error": "rate_limited", "retry_after": 15} - last_scrape_time = now - results = await scrape_warriorforum() - if results: - results = await classify_leads(results) - return results[:10] - @app.post("/scrape/facebook") -async def scrape_facebook_endpoint(): - results = await scrape_facebook() - if results: - results = await classify_leads(results) - return results[:15] - -@app.post("/scrape/all") -async def scrape_all(): - results = [] - try: - wf = await scrape_warriorforum() - if wf: - wf = await classify_leads(wf) - results.extend(wf[:5]) - except Exception as e: - logger.error("WarriorForum scrape in /scrape/all failed: %s", e) - try: - fb = await scrape_facebook() - if fb: - fb = await classify_leads(fb) - results.extend(fb[:8]) - except Exception as e: - logger.error("Facebook scrape in /scrape/all failed: %s", e) - return results[:12] +async def scrape_facebook_endpoint(profile_path: str | None = Query(None), force: bool = Query(False)): + result = await scrape_facebook(profile_path, force) + return result if __name__ == "__main__": uvicorn.run(app, host="0.0.0.0", port=PORT) diff --git a/browser-use-service/out.txt b/browser-use-service/out.txt deleted file mode 100644 index e69de29..0000000 diff --git a/browser-use-service/run_log.txt b/browser-use-service/run_log.txt deleted file mode 100644 index df792e6..0000000 --- a/browser-use-service/run_log.txt +++ /dev/null @@ -1,7 +0,0 @@ -python : INFO: Started server process [20044] - + CategoryInfo : NotSpecified: (INFO: Started server process [20044]:String) [], RemoteException - + FullyQualifiedErrorId : NativeCommandError - -INFO: Waiting for application startup. -INFO: Application startup complete. -INFO: Uvicorn running on http://0.0.0.0:3008 (Press CTRL+C to quit) diff --git a/browser-use-service/scrape_log.txt b/browser-use-service/scrape_log.txt deleted file mode 100644 index a356eb3..0000000 --- a/browser-use-service/scrape_log.txt +++ /dev/null @@ -1,6 +0,0 @@ -[2026-06-22T19:24:48.030482] Browser Use service starting on port 3008 -[2026-06-22T19:27:19.231701] Browser Use service starting on port 3008 -[2026-06-22T19:28:28.335765] Browser Use service starting on port 3008 -[2026-06-22T19:29:05.796265] Browser Use service starting on port 3008 -[2026-06-22T19:29:17.042807] Scraping WarriorForum... -[2026-06-22T19:29:19.075166] WarriorForum: 55 results diff --git a/browser-use-service/stderr.txt b/browser-use-service/stderr.txt deleted file mode 100644 index 85fb7e5..0000000 --- a/browser-use-service/stderr.txt +++ /dev/null @@ -1,4 +0,0 @@ -INFO: Started server process [24268] -INFO: Waiting for application startup. -INFO: Application startup complete. -INFO: Uvicorn running on http://0.0.0.0:3008 (Press CTRL+C to quit) diff --git a/browser-use-service/stdout.txt b/browser-use-service/stdout.txt deleted file mode 100644 index 66ec1d4..0000000 --- a/browser-use-service/stdout.txt +++ /dev/null @@ -1,5 +0,0 @@ -Browser Use service starting on port 3008 -INFO: 127.0.0.1:65003 - "GET /health HTTP/1.1" 200 OK -Scraping WarriorForum... -WarriorForum: 55 results -INFO: 127.0.0.1:54587 - "POST /scrape/all HTTP/1.1" 200 OK diff --git a/database/DESIGN_DECISIONS.md b/database/DESIGN_DECISIONS.md index 8f2a679..893438c 100644 --- a/database/DESIGN_DECISIONS.md +++ b/database/DESIGN_DECISIONS.md @@ -126,10 +126,10 @@ separate 1:N child tables. | Username | Password | Role | |----------|----------|------| -| `superadmin_demo` | `SuperAdmin@2026` | SUPER_ADMIN | -| `admin_demo` | `AdminAccess@2026` | ADMIN | -| `sales_demo` | `SalesAccess@2026` | SALES_USER | -| `dev_demo` | `DevTesting@2026` | DEVELOPER | +| `superadmin_demo` | `[REDACTED]` | SUPER_ADMIN | +| `admin_demo` | `[REDACTED]` | ADMIN | +| `sales_demo` | `[REDACTED]` | SALES_USER | +| `dev_demo` | `[REDACTED]` | DEVELOPER | --- diff --git a/database/migrations/010_facebook_accounts.sql b/database/migrations/010_facebook_accounts.sql new file mode 100644 index 0000000..a250693 --- /dev/null +++ b/database/migrations/010_facebook_accounts.sql @@ -0,0 +1,34 @@ +CREATE TABLE IF NOT EXISTS facebook_accounts ( + id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), + label VARCHAR(100) NOT NULL, + profile_path TEXT NOT NULL, + cookie_file TEXT NOT NULL, + is_active BOOLEAN NOT NULL DEFAULT TRUE, + last_scrape_at TIMESTAMPTZ, + last_success_at TIMESTAMPTZ, + last_error_at TIMESTAMPTZ, + last_error_message TEXT, + consecutive_failures INT NOT NULL DEFAULT 0, + flagged BOOLEAN NOT NULL DEFAULT FALSE, + flagged_at TIMESTAMPTZ, + flagged_reason TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE INDEX IF NOT EXISTS idx_fb_accounts_active ON facebook_accounts(is_active); +CREATE INDEX IF NOT EXISTS idx_fb_accounts_flagged ON facebook_accounts(flagged); + +CREATE TABLE IF NOT EXISTS facebook_scrape_logs ( + id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), + account_id UUID NOT NULL REFERENCES facebook_accounts(id) ON DELETE CASCADE, + started_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + completed_at TIMESTAMPTZ, + success BOOLEAN NOT NULL DEFAULT FALSE, + leads_found INT NOT NULL DEFAULT 0, + error_message TEXT, + detected_flag VARCHAR(50), + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE INDEX IF NOT EXISTS idx_fb_scrape_logs_account ON facebook_scrape_logs(account_id, created_at DESC); diff --git a/database/migrations/run_all.sql b/database/migrations/run_all.sql index 2ec9138..a0dc725 100644 --- a/database/migrations/run_all.sql +++ b/database/migrations/run_all.sql @@ -34,5 +34,8 @@ BEGIN; \echo '=== Running 009_settings.sql (Company Settings + User Preferences) ===' \i 009_settings.sql +\echo '=== Running 010_facebook_accounts.sql (Facebook Account Tracking) ===' +\i 010_facebook_accounts.sql + \echo '=== Migration Complete ===' COMMIT; diff --git a/rust-ai/src/main.rs b/rust-ai/src/main.rs index 174c1f5..ed07239 100644 --- a/rust-ai/src/main.rs +++ b/rust-ai/src/main.rs @@ -15,6 +15,7 @@ use tokio::sync::Mutex; use tracing::{error, info, warn}; use uuid::Uuid; use rand::Rng; +use chrono::Timelike; use std::time::Duration; use std::time::{SystemTime, UNIX_EPOCH}; @@ -93,6 +94,25 @@ struct LeadStore { max_size: usize, } +#[derive(Debug, Deserialize)] +struct ScrapeResponse { + success: bool, + leads: Vec, + flagged: bool, + flag_reason: Option, + error: Option, +} + +#[derive(Debug, Deserialize, Clone)] +struct ScrapeLead { + title: String, + url: String, + author: String, + date: String, + content: String, + source: Option, +} + impl LeadStore { fn new(max_size: usize) -> Self { Self { leads: Vec::new(), max_size } @@ -188,7 +208,7 @@ fn extract_claims(headers: &HeaderMap, state: &AppState) -> Result Ok(claims), _ => Err((StatusCode::FORBIDDEN, "Forbidden".to_string())), } @@ -266,30 +286,43 @@ async fn handle_chat( let has_job = msg_words.contains(&"jobs") || msg_words.contains(&"job"); if has_listing || (has_show && has_job) || (has_show && msg_lower.contains("links")) || msg_lower.contains("recent leads") { let now = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs(); - let service_url = "http://localhost:3008/scrape/all".to_string(); - if let Ok(resp) = state.http_client - .post(&service_url) - .timeout(Duration::from_secs(120)) - .send() - .await + let base_url = "http://localhost:3008/scrape/facebook"; + let mut req_builder = state.http_client.post(base_url).timeout(Duration::from_secs(300)); + + if let Ok(Some((_, path))) = sqlx::query_as::<_, (uuid::Uuid, String)>( + "SELECT id, profile_path FROM facebook_accounts \ + WHERE is_active = TRUE AND flagged = FALSE \ + ORDER BY last_scrape_at ASC NULLS FIRST LIMIT 1" + ) + .fetch_optional(&state.db) + .await { - if let Ok(leads_data) = resp.json::>().await { - info!("Scraped {} leads from {}", leads_data.len(), service_url); - let mut store = state.leads.lock().await; - for item in &leads_data { - store.push(Lead { - title: truncate(item["title"].as_str().unwrap_or(""), 120), - url: item["url"].as_str().unwrap_or("").to_string(), - source: item["source"].as_str().unwrap_or("unknown").to_string(), - found_at: now, - author: truncate(item["author"].as_str().unwrap_or(""), 60), - date: truncate(item["date"].as_str().unwrap_or(""), 30), - content: truncate(item["content"].as_str().unwrap_or(""), 300), - }); + req_builder = req_builder.query(&[("profile_path", &path), ("force", &"true".to_string())]); + } else { + warn!("No active Facebook account found for on-demand scrape"); + } + + match req_builder.send().await { + Ok(resp) => { + if let Ok(scrape_resp) = resp.json::().await { + info!("Scraped {} leads from Facebook", scrape_resp.leads.len()); + let mut store = state.leads.lock().await; + for item in &scrape_resp.leads { + store.push(Lead { + title: truncate(&item.title, 120), + url: item.url.clone(), + source: item.source.clone().unwrap_or_else(|| "facebook".to_string()), + found_at: now, + author: truncate(&item.author, 60), + date: truncate(&item.date, 30), + content: truncate(&item.content, 300), + }); + } } } - } else { - error!("Scraper service unreachable: {}", service_url); + Err(e) => { + error!("Scraper request error: {} - URL: {}", e, base_url); + } } let recent_leads = state.leads.lock().await.recent(604800, 20); @@ -411,7 +444,7 @@ async fn main() { info!("Connected to PostgreSQL"); let http_client = reqwest::Client::builder() - .timeout(Duration::from_secs(120)) + .timeout(Duration::from_secs(300)) .build() .expect("Failed to build HTTP client"); @@ -441,7 +474,7 @@ async fn main() { .route("/ai/jobs", get(handle_jobs)) .route("/health", get(handle_health)) .layer(cors) - .with_state(state); + .with_state(state.clone()); let addr = format!("{}:{}", host, port); info!("CRM AI server listening on {}", addr); @@ -451,10 +484,11 @@ async fn main() { .expect("Failed to bind address"); let bg_leads = lead_store.clone(); - let bg_url = "http://localhost:3008/scrape/all".to_string(); + let bg_db = state.db.clone(); + let bg_url = "http://localhost:3008/scrape/facebook".to_string(); tokio::spawn(async move { let client = match reqwest::Client::builder() - .timeout(Duration::from_secs(120)) + .timeout(Duration::from_secs(300)) .build() { Ok(c) => c, @@ -464,39 +498,145 @@ async fn main() { } }; loop { + // 10% random cycle skip — makes pattern non-periodic + if rand::thread_rng().gen_range(0..100) < 10 { + info!("Skipping this scrape cycle (random 10% skip)"); + let jitter = rand::thread_rng().gen_range(16200..19800); + tokio::time::sleep(Duration::from_secs(jitter)).await; + continue; + } + + // Skip night hours (23:00 – 06:00) + let hour = chrono::Local::now().hour(); + if hour < 6 || hour >= 23 { + info!("Night hours ({}) — skipping scrape", hour); + let jitter = rand::thread_rng().gen_range(16200..19800); + tokio::time::sleep(Duration::from_secs(jitter)).await; + continue; + } + let now = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs(); - match client.post(&bg_url).send().await { - Ok(resp) => { - if resp.status().is_success() { - match resp.json::>().await { - Ok(data) => { - let mut store = bg_leads.lock().await; - for item in &data { - store.push(Lead { - title: truncate(item["title"].as_str().unwrap_or(""), 120), - url: item["url"].as_str().unwrap_or("").to_string(), - source: item["source"].as_str().unwrap_or("unknown").to_string(), - found_at: now, - author: truncate(item["author"].as_str().unwrap_or(""), 60), - date: truncate(item["date"].as_str().unwrap_or(""), 30), - content: truncate(item["content"].as_str().unwrap_or(""), 300), - }); + + // Pick next active un-flagged account + let account = sqlx::query_as::<_, (uuid::Uuid, String)>( + "SELECT id, profile_path FROM facebook_accounts \ + WHERE is_active = TRUE AND flagged = FALSE \ + ORDER BY last_scrape_at ASC NULLS FIRST LIMIT 1" + ) + .fetch_optional(&bg_db) + .await; + + match account { + Ok(Some((account_id, profile_path))) => { + match client.post(&bg_url).query(&[("profile_path", &profile_path)]).send().await { + Ok(resp) => { + if resp.status().is_success() { + match resp.json::().await { + Ok(data) => { + let leads_count = data.leads.len() as i32; + if data.flagged { + let _ = sqlx::query( + "UPDATE facebook_accounts SET flagged = TRUE, flagged_at = NOW(), \ + flagged_reason = $2, last_error_at = NOW(), \ + last_error_message = $3, consecutive_failures = consecutive_failures + 1 \ + WHERE id = $1" + ) + .bind(account_id) + .bind(&data.flag_reason) + .bind(&data.error) + .execute(&bg_db) + .await; + warn!("Facebook account {} flagged: {:?}", account_id, data.flag_reason); + let reason = data.flag_reason.as_deref().unwrap_or("unknown"); + let _ = sqlx::query( + "INSERT INTO notifications (user_id, type, title, description, link) \ + SELECT id, 'warning', 'Facebook Account Flagged', \ + $1 || ' - ' || COALESCE($2, 'unknown reason'), \ + NULL \ + FROM users u JOIN user_roles ur ON ur.user_id = u.id \ + JOIN roles r ON r.id = ur.role_id \ + WHERE r.name IN ('ADMIN', 'SUPER_ADMIN')" + ) + .bind(&account_id.to_string()) + .bind(reason) + .execute(&bg_db) + .await; + } else if data.success { + let _ = sqlx::query( + "UPDATE facebook_accounts SET last_scrape_at = NOW(), \ + last_success_at = NOW(), consecutive_failures = 0, \ + updated_at = NOW() WHERE id = $1" + ) + .bind(account_id) + .execute(&bg_db) + .await; + + let mut store = bg_leads.lock().await; + for item in &data.leads { + store.push(Lead { + title: truncate(&item.title, 120), + url: item.url.clone(), + source: item.source.clone().unwrap_or_else(|| "facebook".to_string()), + found_at: now, + author: truncate(&item.author, 60), + date: truncate(&item.date, 30), + content: truncate(&item.content, 300), + }); + } + info!("Scraped {} leads from Facebook account {}", leads_count, account_id); + } else { + // Increment failures; auto-flag if >= 3 consecutive + let _ = sqlx::query( + "UPDATE facebook_accounts SET last_error_at = NOW(), \ + last_error_message = $2, consecutive_failures = consecutive_failures + 1, \ + flagged = CASE WHEN consecutive_failures + 1 >= 3 THEN TRUE ELSE flagged END, \ + flagged_reason = CASE WHEN consecutive_failures + 1 >= 3 THEN 'too_many_failures' ELSE flagged_reason END, \ + flagged_at = CASE WHEN consecutive_failures + 1 >= 3 THEN NOW() ELSE flagged_at END, \ + updated_at = NOW() WHERE id = $1" + ) + .bind(account_id) + .bind(&data.error) + .execute(&bg_db) + .await; + warn!("Facebook scrape failed for account {}: {:?}", account_id, data.error); + } + + let _ = sqlx::query( + "INSERT INTO facebook_scrape_logs \ + (account_id, started_at, completed_at, success, leads_found, error_message, detected_flag) \ + VALUES ($1, NOW() - interval '5 hours', NOW(), $2, $3, $4, $5)" + ) + .bind(account_id) + .bind(data.success && !data.flagged) + .bind(leads_count) + .bind(&data.error) + .bind(&data.flag_reason) + .execute(&bg_db) + .await; + } + Err(e) => { + warn!("Failed to parse scraper JSON: {}", e); + } } - } - Err(e) => { - warn!("Failed to parse scraper JSON: {}", e); + } else { + warn!("Scraper returned status: {}", resp.status()); } } - } else { - warn!("Scraper returned status: {}", resp.status()); + Err(e) => { + warn!("Scraper request failed: {}", e); + } } } + Ok(None) => { + info!("No active Facebook accounts available — skipping scrape cycle"); + } Err(e) => { - warn!("Scraper request failed: {}", e); + warn!("Failed to query Facebook accounts: {}", e); } } - let delay = rand::thread_rng().gen_range(120..300); - tokio::time::sleep(Duration::from_secs(delay)).await; + + let jitter = rand::thread_rng().gen_range(16200..19800); // 4.5h – 5.5h + tokio::time::sleep(Duration::from_secs(jitter)).await; } }); diff --git a/src/app/api/dashboard/route.ts b/src/app/api/dashboard/route.ts index c33e3a5..f1a2112 100644 --- a/src/app/api/dashboard/route.ts +++ b/src/app/api/dashboard/route.ts @@ -48,7 +48,7 @@ function stageToStatus(name: string): string { } } -async function fetchLeadsInRange(start: Date, end: Date) { +async function fetchLeadsInRange(start: Date, end: Date, userId?: string, isAdmin?: boolean) { const result = await query( `SELECT l.id, l.created_at, l.company_name, l.contact_name, l.email, l.phone, l.notes, l.assigned_to, l.score, @@ -59,8 +59,11 @@ async function fetchLeadsInRange(start: Date, end: Date) { LEFT JOIN users u ON u.id = l.assigned_to WHERE l.deleted_at IS NULL AND l.created_at >= $1 AND l.created_at <= $2 + ${isAdmin ? "" : "AND l.assigned_to = $3"} ORDER BY l.created_at DESC`, - [start.toISOString(), end.toISOString()] + isAdmin + ? [start.toISOString(), end.toISOString()] + : [start.toISOString(), end.toISOString(), userId] ) return result.rows.map((r: any) => ({ ...r, @@ -118,6 +121,8 @@ export async function GET(request: NextRequest) { const user = await getSessionUser() if (!user) return NextResponse.json({ error: "Unauthorized" }, { status: 401 }) + const isAdmin = user.role === "admin" || user.role === "super_admin" + const { searchParams } = new URL(request.url) const period = searchParams.get("period") || "6months" const yearParam = searchParams.get("year") @@ -134,8 +139,8 @@ export async function GET(request: NextRequest) { } const [currentLeads, prevLeads] = await Promise.all([ - fetchLeadsInRange(start, end), - fetchLeadsInRange(prevRange.start, prevRange.end), + fetchLeadsInRange(start, end, user.id, isAdmin), + fetchLeadsInRange(prevRange.start, prevRange.end, user.id, isAdmin), ]) const currentCounts = countStatuses(currentLeads) diff --git a/src/app/api/leads/[id]/notes/route.ts b/src/app/api/leads/[id]/notes/route.ts index c039b96..06b7b88 100644 --- a/src/app/api/leads/[id]/notes/route.ts +++ b/src/app/api/leads/[id]/notes/route.ts @@ -3,12 +3,28 @@ import { getSessionUser } from "@/lib/auth" import { query } from "@/lib/db" import { avatarSvgUrl } from "@/lib/avatar" +async function checkLeadAccess(leadId: string, userId: string): Promise { + const result = await query( + `SELECT 1 FROM leads WHERE id = $1 AND deleted_at IS NULL + AND (assigned_to = $2 OR EXISTS ( + SELECT 1 FROM user_roles ur JOIN roles r ON r.id = ur.role_id + WHERE ur.user_id = $2 AND r.name IN ('ADMIN', 'SUPER_ADMIN') + ))`, + [leadId, userId] + ) + return result.rows.length > 0 +} + export async function POST(request: NextRequest, { params }: { params: Promise<{ id: string }> }) { try { const user = await getSessionUser() if (!user) return NextResponse.json({ error: "Unauthorized" }, { status: 401 }) const { id } = await params + if (!await checkLeadAccess(id, user.id)) { + return NextResponse.json({ error: "Lead not found" }, { status: 404 }) + } + const { content } = await request.json() if (!content?.trim()) { return NextResponse.json({ error: "Content is required" }, { status: 400 }) @@ -33,6 +49,9 @@ export async function GET(request: NextRequest, { params }: { params: Promise<{ if (!user) return NextResponse.json({ error: "Unauthorized" }, { status: 401 }) const { id } = await params + if (!await checkLeadAccess(id, user.id)) { + return NextResponse.json({ error: "Lead not found" }, { status: 404 }) + } const { searchParams } = new URL(request.url) const limit = parseInt(searchParams.get("limit") || "50", 10) const offset = parseInt(searchParams.get("offset") || "0", 10) diff --git a/src/app/api/leads/[id]/route.ts b/src/app/api/leads/[id]/route.ts index 904cd1d..c3c1a5f 100644 --- a/src/app/api/leads/[id]/route.ts +++ b/src/app/api/leads/[id]/route.ts @@ -111,6 +111,11 @@ export async function PATCH(request: NextRequest, { params }: { params: Promise< if (body.description !== undefined) { fields.push(`notes = $${idx++}`); values.push(body.description) } if (body.source !== undefined) { fields.push(`source_id = $${idx++}`); values.push(body.source) } if (body.assignedUserId !== undefined) { + const isAdmin = user.role === "admin" || user.role === "super_admin" + if (!isAdmin) { + // non-admin cannot reassign + return NextResponse.json({ error: "Only admins can reassign leads" }, { status: 403 }) + } fields.push(`assigned_to = $${idx++}`) values.push(body.assignedUserId === "none" ? null : body.assignedUserId) } diff --git a/src/app/api/leads/route.ts b/src/app/api/leads/route.ts index 4d76162..fa83e33 100644 --- a/src/app/api/leads/route.ts +++ b/src/app/api/leads/route.ts @@ -123,6 +123,15 @@ export async function POST(request: NextRequest) { if (!user) return NextResponse.json({ error: "Unauthorized" }, { status: 401 }) const body = await request.json() + const isAdmin = user.role === "admin" || user.role === "super_admin" + + // Non-admin users can only assign leads to themselves; admin/super_admin can assign to anyone + let assignedUserId = body.assignedUserId + if (!isAdmin) { + assignedUserId = user.id + } else if (assignedUserId === "none" || !assignedUserId) { + assignedUserId = null + } const stageResult = await query( "SELECT id FROM lead_stages WHERE name = $1", @@ -142,7 +151,7 @@ export async function POST(request: NextRequest) { body.description || null, body.source || null, stageId, - body.assignedUserId === "none" ? null : body.assignedUserId, + assignedUserId, ] ) diff --git a/src/app/api/notifications/route.ts b/src/app/api/notifications/route.ts index d8c2c00..7104ccd 100644 --- a/src/app/api/notifications/route.ts +++ b/src/app/api/notifications/route.ts @@ -47,7 +47,8 @@ export async function POST(request: NextRequest) { if (!user) return NextResponse.json({ error: "Unauthorized" }, { status: 401 }) const { type, title, description, link, userId } = await request.json() - const targetUserId = userId || user.id + const isAdmin = user.role === "admin" || user.role === "super_admin" + const targetUserId = (userId && isAdmin) ? userId : user.id const result = await query( `INSERT INTO notifications (user_id, type, title, description, link) diff --git a/src/app/api/settings/facebook/accounts/[id]/route.ts b/src/app/api/settings/facebook/accounts/[id]/route.ts new file mode 100644 index 0000000..a4954ce --- /dev/null +++ b/src/app/api/settings/facebook/accounts/[id]/route.ts @@ -0,0 +1,61 @@ +import { NextRequest, NextResponse } from "next/server" +import { getSessionUser } from "@/lib/auth" +import { query } from "@/lib/db" + +export async function PATCH(request: NextRequest, { params }: { params: Promise<{ id: string }> }) { + try { + const user = await getSessionUser() + if (!user) return NextResponse.json({ error: "Unauthorized" }, { status: 401 }) + if (user.role !== "admin" && user.role !== "super_admin") { + return NextResponse.json({ error: "Forbidden" }, { status: 403 }) + } + + const { id } = await params + const body = await request.json() + const fields: string[] = [] + const values: any[] = [] + let idx = 1 + + if (body.isActive !== undefined) { + fields.push(`is_active = $${idx++}`) + values.push(body.isActive) + } + if (body.label !== undefined) { + fields.push(`label = $${idx++}`) + values.push(body.label.trim()) + } + if (body.profilePath !== undefined) { + fields.push(`profile_path = $${idx++}, cookie_file = $${idx}`) + values.push(body.profilePath.trim()) + values.push(`${body.profilePath.replace(/\\+$/, '')}\\cookies.sqlite`) + idx++ + } + if (body.unflag === true) { + fields.push(`flagged = FALSE, flagged_at = NULL, flagged_reason = NULL, consecutive_failures = 0`) + } + + if (fields.length === 0) { + return NextResponse.json({ error: "No fields to update" }, { status: 400 }) + } + + fields.push(`updated_at = NOW()`) + values.push(id) + + await query( + `UPDATE facebook_accounts SET ${fields.join(", ")} WHERE id = $${idx}`, + values + ) + + const updated = await query( + `SELECT id, label, profile_path, is_active, flagged, flagged_reason, + consecutive_failures, updated_at + FROM facebook_accounts WHERE id = $1`, + [id] + ) + + return NextResponse.json(updated.rows[0] || { success: true }) + } catch (error) { + console.error("Facebook accounts PATCH error:", error) + return NextResponse.json({ error: "Failed to update Facebook account" }, { status: 500 }) + } +} diff --git a/src/app/api/settings/facebook/accounts/route.ts b/src/app/api/settings/facebook/accounts/route.ts new file mode 100644 index 0000000..e5f0005 --- /dev/null +++ b/src/app/api/settings/facebook/accounts/route.ts @@ -0,0 +1,66 @@ +import { NextRequest, NextResponse } from "next/server" +import { getSessionUser } from "@/lib/auth" +import { query } from "@/lib/db" + +export async function GET() { + try { + const user = await getSessionUser() + if (!user) return NextResponse.json({ error: "Unauthorized" }, { status: 401 }) + if (user.role !== "admin" && user.role !== "super_admin") { + return NextResponse.json({ error: "Forbidden" }, { status: 403 }) + } + + const accounts = await query( + `SELECT fa.id, fa.label, fa.profile_path, fa.is_active, + fa.last_scrape_at, fa.last_success_at, fa.last_error_at, + fa.last_error_message, fa.consecutive_failures, + fa.flagged, fa.flagged_at, fa.flagged_reason, + fa.created_at, fa.updated_at, + COALESCE(sl.leads_found, 0) AS last_leads_found, + sl.success AS last_success, + sl.detected_flag AS last_detected_flag + FROM facebook_accounts fa + LEFT JOIN LATERAL ( + SELECT leads_found, success, detected_flag + FROM facebook_scrape_logs + WHERE account_id = fa.id + ORDER BY created_at DESC + LIMIT 1 + ) sl ON TRUE + ORDER BY fa.created_at ASC` + ) + + return NextResponse.json(accounts.rows) + } catch (error) { + console.error("Facebook accounts GET error:", error) + return NextResponse.json({ error: "Failed to load Facebook accounts" }, { status: 500 }) + } +} + +export async function POST(request: NextRequest) { + try { + const user = await getSessionUser() + if (!user) return NextResponse.json({ error: "Unauthorized" }, { status: 401 }) + if (user.role !== "admin" && user.role !== "super_admin") { + return NextResponse.json({ error: "Forbidden" }, { status: 403 }) + } + + const { label, profilePath } = await request.json() + if (!label?.trim() || !profilePath?.trim()) { + return NextResponse.json({ error: "Label and profile path are required" }, { status: 400 }) + } + + const cookieFile = `${profilePath.replace(/\\+$/, '')}\\cookies.sqlite` + const result = await query( + `INSERT INTO facebook_accounts (label, profile_path, cookie_file) + VALUES ($1, $2, $3) + RETURNING id, label, profile_path, is_active, created_at`, + [label.trim(), profilePath.trim(), cookieFile] + ) + + return NextResponse.json(result.rows[0], { status: 201 }) + } catch (error) { + console.error("Facebook accounts POST error:", error) + return NextResponse.json({ error: "Failed to create Facebook account" }, { status: 500 }) + } +} diff --git a/src/app/api/settings/facebook/logs/route.ts b/src/app/api/settings/facebook/logs/route.ts new file mode 100644 index 0000000..7f10470 --- /dev/null +++ b/src/app/api/settings/facebook/logs/route.ts @@ -0,0 +1,41 @@ +import { NextRequest, NextResponse } from "next/server" +import { getSessionUser } from "@/lib/auth" +import { query } from "@/lib/db" + +export async function GET(request: NextRequest) { + try { + const user = await getSessionUser() + if (!user) return NextResponse.json({ error: "Unauthorized" }, { status: 401 }) + if (user.role !== "admin" && user.role !== "super_admin") { + return NextResponse.json({ error: "Forbidden" }, { status: 403 }) + } + + const { searchParams } = new URL(request.url) + const accountId = searchParams.get("accountId") + const limit = parseInt(searchParams.get("limit") || "50", 10) + const offset = parseInt(searchParams.get("offset") || "0", 10) + + let sql = `SELECT sl.id, sl.account_id, fa.label AS account_label, + sl.started_at, sl.completed_at, sl.success, + sl.leads_found, sl.error_message, sl.detected_flag, + sl.created_at + FROM facebook_scrape_logs sl + JOIN facebook_accounts fa ON fa.id = sl.account_id` + const params: any[] = [] + let paramIdx = 1 + + if (accountId) { + sql += ` WHERE sl.account_id = $${paramIdx++}` + params.push(accountId) + } + + sql += ` ORDER BY sl.created_at DESC LIMIT $${paramIdx++} OFFSET $${paramIdx++}` + params.push(limit, offset) + + const result = await query(sql, params) + return NextResponse.json(result.rows) + } catch (error) { + console.error("Facebook scrape logs GET error:", error) + return NextResponse.json({ error: "Failed to load scrape logs" }, { status: 500 }) + } +} diff --git a/src/app/api/system/monitor/route.ts b/src/app/api/system/monitor/route.ts index baecb2e..0051f42 100644 --- a/src/app/api/system/monitor/route.ts +++ b/src/app/api/system/monitor/route.ts @@ -8,6 +8,9 @@ let prevTime = Date.now() export async function GET() { const sessionUser = await getSessionUser() if (!sessionUser) return NextResponse.json({ error: "Unauthorized" }, { status: 401 }) + if (sessionUser.role !== "admin" && sessionUser.role !== "super_admin") { + return NextResponse.json({ error: "Forbidden" }, { status: 403 }) + } const now = Date.now() const elapsed = now - prevTime diff --git a/src/app/api/users/avatar/route.ts b/src/app/api/users/avatar/route.ts index bbcce5b..f05f18b 100644 --- a/src/app/api/users/avatar/route.ts +++ b/src/app/api/users/avatar/route.ts @@ -2,6 +2,9 @@ import { NextRequest, NextResponse } from "next/server" import { getSessionUser } from "@/lib/auth" import { query } from "@/lib/db" +const ALLOWED_PREFIXES = ["data:image/png;base64,", "data:image/jpeg;base64,", "data:image/gif;base64,"] +const MAX_AVATAR_BYTES = 2 * 1024 * 1024 // 2MB + export async function POST(request: NextRequest) { try { const user = await getSessionUser() @@ -12,6 +15,18 @@ export async function POST(request: NextRequest) { return NextResponse.json({ error: "Invalid avatar data" }, { status: 400 }) } + const allowed = ALLOWED_PREFIXES.some((p) => avatar.startsWith(p)) + if (!allowed) { + return NextResponse.json({ error: "Avatar must be a PNG, JPEG, or GIF data URL" }, { status: 400 }) + } + + // Approximate decoded size: base64 is ~4/3 of original + const base64Data = avatar.split(",")[1] || "" + const estimatedBytes = Math.round(base64Data.length * 0.75) + if (estimatedBytes > MAX_AVATAR_BYTES) { + return NextResponse.json({ error: "Avatar exceeds 2MB size limit" }, { status: 400 }) + } + await query( `UPDATE users SET avatar_url = $1 WHERE id = $2`, [avatar, user.id], diff --git a/src/app/api/users/route.ts b/src/app/api/users/route.ts index 2ec3ef7..4f0be95 100644 --- a/src/app/api/users/route.ts +++ b/src/app/api/users/route.ts @@ -7,6 +7,9 @@ export async function GET(request: NextRequest) { try { const sessionUser = await getSessionUser() if (!sessionUser) return NextResponse.json({ error: "Unauthorized" }, { status: 401 }) + if (sessionUser.role !== "admin" && sessionUser.role !== "super_admin") { + return NextResponse.json({ error: "Forbidden" }, { status: 403 }) + } const { searchParams } = new URL(request.url) const limit = parseInt(searchParams.get("limit") || "50", 10) diff --git a/src/app/api/users/search/route.ts b/src/app/api/users/search/route.ts index ca5582b..77fcb87 100644 --- a/src/app/api/users/search/route.ts +++ b/src/app/api/users/search/route.ts @@ -7,6 +7,9 @@ export async function GET(request: NextRequest) { try { const currentUser = await getSessionUser() if (!currentUser) return NextResponse.json({ error: "Unauthorized" }, { status: 401 }) + if (currentUser.role !== "admin" && currentUser.role !== "super_admin") { + return NextResponse.json({ error: "Forbidden" }, { status: 403 }) + } const q = request.nextUrl.searchParams.get("q") || "" diff --git a/src/components/layout/sidebar.tsx b/src/components/layout/sidebar.tsx index f4cdcf6..7be6e4d 100644 --- a/src/components/layout/sidebar.tsx +++ b/src/components/layout/sidebar.tsx @@ -18,10 +18,12 @@ import { PanelLeftClose, MessageSquare, Bot, + Facebook, } from "lucide-react" import { COMPANY_NAME } from "@/lib/constants" import { useUser } from "@/providers/user-provider" import { useNotifications } from "@/providers/notification-provider" +import { FacebookAccountsDialog } from "@/components/settings/facebook-accounts-dialog" const navItems = [ { href: "/dashboard", label: "Dashboard", icon: LayoutDashboard }, @@ -43,7 +45,9 @@ export function Sidebar({ collapsed, onToggle, mobileOpen, onMobileClose }: Side const pathname = usePathname() const { user } = useUser() const { unreadChatCount } = useNotifications() + const [fbDialogOpen, setFbDialogOpen] = useState(false) if (!user) return null + const isAdmin = user.role === "admin" || user.role === "super_admin" const initials = user.name.split(" ").map((n) => n[0]).join("") const sidebarContent = ( @@ -143,6 +147,39 @@ export function Sidebar({ collapsed, onToggle, mobileOpen, onMobileClose }: Side ) })} + {isAdmin && collapsed && ( + + + + + + + Facebook Accounts + + + + )} + {isAdmin && !collapsed && ( + + )} {/* Collapse toggle at bottom (only in collapsed mode) */} @@ -189,6 +226,8 @@ export function Sidebar({ collapsed, onToggle, mobileOpen, onMobileClose }: Side {sidebarContent} + + {/* Mobile sidebar overlay */} {mobileOpen && ( diff --git a/src/components/settings/facebook-accounts-dialog.tsx b/src/components/settings/facebook-accounts-dialog.tsx new file mode 100644 index 0000000..ae116f3 --- /dev/null +++ b/src/components/settings/facebook-accounts-dialog.tsx @@ -0,0 +1,102 @@ +"use client" + +import { useEffect, useState } from "react" +import { Dialog, DialogContent, DialogHeader, DialogTitle, DialogDescription } from "@/components/ui/dialog" +import { Badge } from "@/components/ui/badge" +import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table" +import { ShieldAlert, RefreshCw } from "lucide-react" +import { Button } from "@/components/ui/button" + +interface Account { + id: string + label: string + is_active: boolean + last_scrape_at: string | null + last_success_at: string | null + last_error_at: string | null + consecutive_failures: number + flagged: boolean + flagged_reason: string | null + last_leads_found: number + last_success: boolean | null +} + +export function FacebookAccountsDialog({ open, onOpenChange }: { open: boolean; onOpenChange: (v: boolean) => void }) { + const [accounts, setAccounts] = useState([]) + const [loading, setLoading] = useState(false) + + useEffect(() => { + if (open) load() + }, [open]) + + async function load() { + setLoading(true) + try { + const res = await fetch("/api/settings/facebook/accounts") + if (res.ok) { + setAccounts(await res.json()) + } + } catch {} + setLoading(false) + } + + return ( + + + +
+ Facebook Accounts + + Status of Facebook profiles used for lead scraping + +
+ + + + {loading ? ( +
Loading...
+ ) : accounts.length === 0 ? ( +
No accounts configured
+ ) : ( + + + + Label + Status + Last Scrape + Leads + Failures + + + + {accounts.map((a) => ( + + {a.label} + + {a.flagged ? ( + + + {a.flagged_reason || "Flagged"} + + ) : a.is_active ? ( + Active + ) : ( + Disabled + )} + + + {a.last_scrape_at ? new Date(a.last_scrape_at).toLocaleString() : "Never"} + + {a.last_leads_found} + {a.consecutive_failures} + + ))} + +
+ )} + + + ) +} From d51c84997a7f0251867e98f13151dc70ce55a8d1 Mon Sep 17 00:00:00 2001 From: Ace Date: Tue, 23 Jun 2026 15:29:34 +0200 Subject: [PATCH 3/3] new changes --- rust-ai/src/main.rs | 56 ++++++++++++++++++++++++++++++++------------- 1 file changed, 40 insertions(+), 16 deletions(-) diff --git a/rust-ai/src/main.rs b/rust-ai/src/main.rs index ed07239..ef33b3f 100644 --- a/rust-ai/src/main.rs +++ b/rust-ai/src/main.rs @@ -287,8 +287,8 @@ async fn handle_chat( if has_listing || (has_show && has_job) || (has_show && msg_lower.contains("links")) || msg_lower.contains("recent leads") { let now = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs(); let base_url = "http://localhost:3008/scrape/facebook"; - let mut req_builder = state.http_client.post(base_url).timeout(Duration::from_secs(300)); - + use std::fmt::Write; + let mut service_url = base_url.to_string(); if let Ok(Some((_, path))) = sqlx::query_as::<_, (uuid::Uuid, String)>( "SELECT id, profile_path FROM facebook_accounts \ WHERE is_active = TRUE AND flagged = FALSE \ @@ -297,27 +297,48 @@ async fn handle_chat( .fetch_optional(&state.db) .await { - req_builder = req_builder.query(&[("profile_path", &path), ("force", &"true".to_string())]); + let encoded: String = path.chars().map(|c| match c { + 'A'..='Z' | 'a'..='z' | '0'..='9' | '.' | '-' | '_' | '~' => c.to_string(), + _ => format!("%{:02X}", c as u8), + }).collect(); + write!(service_url, "?profile_path={}&force=true", encoded).unwrap(); + info!("Calling Python scrape at: {}?profile_path=...&force=true", base_url); } else { warn!("No active Facebook account found for on-demand scrape"); } + let req_builder = state.http_client.post(&service_url); match req_builder.send().await { Ok(resp) => { - if let Ok(scrape_resp) = resp.json::().await { - info!("Scraped {} leads from Facebook", scrape_resp.leads.len()); - let mut store = state.leads.lock().await; - for item in &scrape_resp.leads { - store.push(Lead { - title: truncate(&item.title, 120), - url: item.url.clone(), - source: item.source.clone().unwrap_or_else(|| "facebook".to_string()), - found_at: now, - author: truncate(&item.author, 60), - date: truncate(&item.date, 30), - content: truncate(&item.content, 300), - }); + let status = resp.status(); + let body = resp.text().await.unwrap_or_default(); + info!("Python scrape response ({}): {} bytes", status, body.len()); + if body.starts_with('{') { + match serde_json::from_str::(&body) { + Ok(scrape_resp) => { + info!("Scraped {} leads from Facebook", scrape_resp.leads.len()); + if scrape_resp.leads.is_empty() && scrape_resp.error.is_some() { + warn!("Python returned error: {:?}", scrape_resp.error); + } + let mut store = state.leads.lock().await; + for item in &scrape_resp.leads { + store.push(Lead { + title: truncate(&item.title, 120), + url: item.url.clone(), + source: item.source.clone().unwrap_or_else(|| "facebook".to_string()), + found_at: now, + author: truncate(&item.author, 60), + date: truncate(&item.date, 30), + content: truncate(&item.content, 300), + }); + } + } + Err(e) => { + warn!("Failed to parse Python scrape response: {} body: {}", e, &body[..body.len().min(200)]); + } } + } else { + warn!("Python returned non-JSON response: {}", &body[..body.len().min(200)]); } } Err(e) => { @@ -497,6 +518,9 @@ async fn main() { return; } }; + // Initial delay to let user on-demand requests take priority + tokio::time::sleep(Duration::from_secs(300)).await; + loop { // 10% random cycle skip — makes pattern non-periodic if rand::thread_rng().gen_range(0..100) < 10 {