From b245b352e777c45b0b26c5722cc4194936ab7495 Mon Sep 17 00:00:00 2001 From: Ace Date: Wed, 24 Jun 2026 17:17:38 +0200 Subject: [PATCH] Updated Ai to pull from facebook, added a loading SVG to make user wait for server too boot up, and added commands that can be used to start off the AI exstraction --- ai-server/index.mjs | 48 ++++++++++ browser-use-service/main.py | 67 +++++++++----- browser-use-service/test_scrape.py | 21 +++++ package.json | 2 +- src/app/api/ai/chat/route.ts | 29 +++++- src/components/ai/ai-chat.tsx | 143 ++++++++++++++++++++++------- src/lib/ai.ts | 85 +++++++++++------ 7 files changed, 307 insertions(+), 88 deletions(-) create mode 100644 browser-use-service/test_scrape.py diff --git a/ai-server/index.mjs b/ai-server/index.mjs index 9aded49..22f9bed 100644 --- a/ai-server/index.mjs +++ b/ai-server/index.mjs @@ -96,7 +96,55 @@ function appendToImprovementLog(entry) { } // ── Chat handler ──────────────────────────────────────────────── +async function scrapeFacebook() { + const profilePath = process.env.FX_PROFILE || "" + const urlPath = `/scrape/facebook?force=true${profilePath ? `&profile_path=${encodeURIComponent(profilePath)}` : ""}` + const logPath = "C:\\Users\\USER-PC\\AppData\\Local\\Temp\\opencode\\ai-scrape-debug.log" + try { + const body = await new Promise((resolve, reject) => { + const req = http.request({ hostname: "127.0.0.1", port: 3008, path: urlPath, method: "POST", timeout: 360000 }, (res) => { + let data = "" + res.on("data", (c) => data += c) + res.on("end", () => resolve(data)) + res.on("error", reject) + }) + req.on("timeout", () => { req.destroy(); reject(new Error("timeout")) }) + req.on("error", reject) + req.end() + }) + const data = JSON.parse(body) + return data + } catch (e) { + return null + } +} + +function formatLeads(leads) { + if (!leads || leads.length === 0) return "No leads found from the latest scrape." + let output = `**${leads.length} leads found:**\n\n` + for (let i = 0; i < leads.length; i++) { + const l = leads[i] + output += `${i + 1}. ${l.title || "No title"}\n` + if (l.author) output += ` Author: ${l.author}\n` + if (l.date) output += ` Date: ${l.date}\n` + if (l.url) output += ` URL: ${l.url}\n` + output += "\n" + } + return output.trim() +} + async function handleChat(userMessage, userId, userRole) { + const lowerMsg = userMessage.toLowerCase() + const triggerWords = ["lists", "listings", "leads", "recent leads", "pull leads", "show me leads", "show listings"] + + if (triggerWords.some(w => lowerMsg.includes(w))) { + const result = await scrapeFacebook() + if (result && result.success) { + return formatLeads(result.leads) + } + return "Scraper returned no results or encountered an error. Try again later." + } + const jobs = loadedJobs const instructions = readInstructions() diff --git a/browser-use-service/main.py b/browser-use-service/main.py index d088037..5b113b3 100644 --- a/browser-use-service/main.py +++ b/browser-use-service/main.py @@ -369,7 +369,24 @@ async def _get_article_elements(page) -> list[dict]: return results; }''') -async def search_facebook(page, query: str) -> list[dict]: +async def _ensure_page(page, context): + try: + await page.evaluate('1') + return page + except Exception: + logger.warning("Page was closed mid-scrape, creating a fresh page") + page = await context.new_page() + try: + await page.goto('https://www.google.com/', wait_until='domcontentloaded', timeout=15000) + await page.wait_for_timeout(random.randint(1000, 3000)) + except Exception: + logger.warning("Google navigation failed during page recreation") + await page.goto('https://www.facebook.com/', wait_until='domcontentloaded', timeout=30000) + await page.wait_for_timeout(random.randint(3000, 8000)) + return page + +async def search_facebook(page, context, query: str): + page = await _ensure_page(page, context) url = f'https://www.facebook.com/search/posts/?q={urllib.parse.quote(query)}' try: await page.goto(url, wait_until='domcontentloaded', timeout=30000) @@ -400,9 +417,9 @@ async def search_facebook(page, query: str) -> list[dict]: raw = await page.evaluate('document.body.innerText') posts = _extract_posts_from_text(raw, url) except Exception as e: - logger.warning("Facebook search failed: %s", e) - return [] - return posts + logger.warning("Facebook search '%s' failed: %s", query, e) + return page, [] + return page, posts def cleanup_chrome(): import subprocess, signal @@ -425,7 +442,7 @@ async def scrape_facebook(profile_path: str | None = None, force: bool = False) ) viewport = random.choice(VIEWPORTS) context = await browser.new_context( - user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:150.0) Gecko/20100101 Firefox/150.0', + user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36', viewport=viewport, ) @@ -445,8 +462,11 @@ async def scrape_facebook(profile_path: str | None = None, force: bool = False) page = await context.new_page() # Navigate through google first for a legitimate Referer header - await page.goto('https://www.google.com/', wait_until='domcontentloaded', timeout=15000) - await page.wait_for_timeout(random.randint(1000, 3000)) + try: + await page.goto('https://www.google.com/', wait_until='domcontentloaded', timeout=15000) + await page.wait_for_timeout(random.randint(1000, 3000)) + except Exception: + logger.warning("Google navigation failed (IP may be blocked), trying Facebook directly") await page.goto('https://www.facebook.com/', wait_until='domcontentloaded', timeout=30000) await page.wait_for_timeout(random.randint(3000, 8000)) @@ -472,23 +492,22 @@ async def scrape_facebook(profile_path: str | None = None, force: bool = False) all_posts = [] searches = random.sample(FB_SEARCHES, k=random.randint(5, 8)) for i, query in enumerate(searches): - try: - posts = await search_facebook(page, query) - all_posts.extend(posts) - # Between searches: random break with possible small scroll - if random.random() < 0.4: - await page.evaluate(f"window.scrollBy(0, {random.randint(-300, 300)})") - delay = random.uniform(8, 25) - await page.wait_for_timeout(int(delay * 1000)) - # Tab switch: 15% chance after 2nd-3rd search - if i == random.randint(1, 2) and random.random() < 0.15: - new_page = await context.new_page() - await new_page.goto('https://www.messenger.com/', wait_until='domcontentloaded', timeout=15000) - await new_page.wait_for_timeout(random.randint(3000, 8000)) - await new_page.close() - await page.wait_for_timeout(random.randint(1000, 3000)) - except Exception as e: - logger.warning("Facebook search '%s' failed: %s", query, e) + page, posts = await search_facebook(page, context, query) + all_posts.extend(posts) + if not posts: + continue + # Between searches: random break with possible small scroll + if random.random() < 0.4: + await page.evaluate(f"window.scrollBy(0, {random.randint(-300, 300)})") + delay = random.uniform(8, 25) + await page.wait_for_timeout(int(delay * 1000)) + # Tab switch: 15% chance after 2nd-3rd search + if i == random.randint(1, 2) and random.random() < 0.15: + new_page = await context.new_page() + await new_page.goto('https://www.messenger.com/', wait_until='domcontentloaded', timeout=15000) + await new_page.wait_for_timeout(random.randint(3000, 8000)) + await new_page.close() + page = await _ensure_page(page, context) # Idle before closing — human would pause if random.random() < 0.5: diff --git a/browser-use-service/test_scrape.py b/browser-use-service/test_scrape.py new file mode 100644 index 0000000..09e0345 --- /dev/null +++ b/browser-use-service/test_scrape.py @@ -0,0 +1,21 @@ +import asyncio, sys, logging +logging.basicConfig(level=logging.DEBUG, format='%(asctime)s [%(levelname)s] %(message)s') +sys.path.insert(0, '.') +from main import get_fb_cookies, scrape_facebook + +async def test(): + print('Step 1: get cookies') + cookies = await get_fb_cookies() + print(f'Got {len(cookies)} cookies') + if not cookies: + print('No cookies - will still try scrape') + print('Step 2: run scrape_facebook') + result = await scrape_facebook(force=True) + print(f'Result success={result["success"]} leads={len(result["leads"])} error={result.get("error")}') + if result.get('leads'): + for l in result['leads'][:3]: + print(f' - {l["title"][:100]}') + if result.get('error'): + print(f' Error: {result["error"]}') + +asyncio.run(test()) diff --git a/package.json b/package.json index 261ff72..e687c40 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,7 @@ "dev:precheck": "powershell -NoProfile -Command \"Get-NetTCPConnection -State Listen | Where-Object { $_.LocalPort -in 3001,3006,3008 } | ForEach-Object { Stop-Process -Id $_.OwningProcess -Force -ErrorAction SilentlyContinue; Write-Host ('Freed port '+$_.LocalPort) }\"", "dev:ollama": "powershell -NoProfile -Command \"$ollama = if (Get-Command ollama -ErrorAction SilentlyContinue) { 'ollama' } else { Join-Path $env:LOCALAPPDATA 'Programs\\Ollama\\ollama.exe' }; if (-not (Get-Process ollama -ErrorAction SilentlyContinue)) { Start-Process $ollama -ArgumentList 'serve' -WindowStyle Hidden; Start-Sleep 3 }; exit 0\"", "dev:rust": "node ai-server/index.mjs", - "dev:browser-use": "cd browser-use-service && python main.py", + "dev:browser-use": "set FX_PROFILE=C:\\Users\\USER-PC\\AppData\\Roaming\\Mozilla\\Firefox\\Profiles\\h8p11vlj.default-release && cd browser-use-service && python main.py", "build": "next build", "start": "next start -p 3006", "lint": "eslint" diff --git a/src/app/api/ai/chat/route.ts b/src/app/api/ai/chat/route.ts index 6ae4df7..51ad7be 100644 --- a/src/app/api/ai/chat/route.ts +++ b/src/app/api/ai/chat/route.ts @@ -1,8 +1,29 @@ import { NextRequest, NextResponse } from "next/server" +import { chatWithAI } from "@/lib/ai" +import { getSessionUser } from "@/lib/auth" -// This route handler has a known issue with Next.js 15 fetch augmentation -// on this platform. The client-side code calls the AI server directly -// via browser fetch (CORS is open). This handler returns a fallback. export async function POST(request: NextRequest) { - return NextResponse.json({ error: "AI service unavailable on server. Use client-side mode." }, { status: 503 }) + try { + const user = await getSessionUser() + if (!user) { + return NextResponse.json({ error: "Unauthorized" }, { status: 401 }) + } + + const { message } = await request.json() + if (!message || typeof message !== "string") { + return NextResponse.json({ error: "Message is required" }, { status: 400 }) + } + + const sessionCookie = request.cookies.get("session") + const jwtToken = sessionCookie?.value + if (!jwtToken) { + return NextResponse.json({ error: "No session token" }, { status: 401 }) + } + + const response = await chatWithAI(message, jwtToken) + return NextResponse.json({ response }) + } catch (error: any) { + console.error("AI chat error:", error) + return NextResponse.json({ error: error.message || "AI service error" }, { status: 500 }) + } } diff --git a/src/components/ai/ai-chat.tsx b/src/components/ai/ai-chat.tsx index c94e971..14081ea 100644 --- a/src/components/ai/ai-chat.tsx +++ b/src/components/ai/ai-chat.tsx @@ -1,7 +1,7 @@ "use client" import { useState, useRef, useEffect, Fragment } from "react" -import { Send, Loader2, Bot, User, RefreshCw, AlertCircle } from "lucide-react" +import { Send, Bot, User, RefreshCw, AlertCircle, Check, Terminal } from "lucide-react" function linkifyText(text: string) { const urlRegex = /(https?:\/\/[^\s<]+[^\s<.,;:!?)\]}>])/ @@ -24,11 +24,28 @@ export function AIChat() { const [input, setInput] = useState("") const [loading, setLoading] = useState(false) const [error, setError] = useState("") - const [ollamaStatus, setOllamaStatus] = useState(null) + const [bootState, setBootState] = useState<"booting" | "ready" | "error">("booting") const messagesEndRef = useRef(null) + const checkServer = async () => { + try { + const res = await fetch("/api/ai/chat", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ message: "ping" }), + }) + if (res.status !== 503) { + setBootState("ready") + } else { + setTimeout(checkServer, 2000) + } + } catch { + setTimeout(checkServer, 2000) + } + } + useEffect(() => { - fetch(`${AI_API}/ai/jobs`) + fetch("/api/ai/jobs") .then((r) => r.json()) .then((data) => { if (data.jobs?.length) { @@ -56,28 +73,13 @@ export function AIChat() { }, ]) }) + checkServer() }, []) - - useEffect(() => { messagesEndRef.current?.scrollIntoView({ behavior: "smooth" }) }, [messages]) - const AI_API = process.env.NEXT_PUBLIC_AI_URL || "http://127.0.0.1:3001" - - const checkOllama = async () => { - try { - const res = await fetch(`${AI_API}/health`) - const data = await res.json() - setOllamaStatus(data.status === "ok") - } catch { - setOllamaStatus(false) - } - } - - useEffect(() => { checkOllama() }, []) - const sendMessage = async () => { const msg = input.trim() if (!msg || loading) return @@ -88,7 +90,7 @@ export function AIChat() { setLoading(true) try { - const res = await fetch(`${AI_API}/ai/chat`, { + const res = await fetch("/api/ai/chat", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ message: msg }), @@ -120,17 +122,86 @@ export function AIChat() { } } + const bootOverlay = ( +
+ +
+

Servers booting...

+
+
+ + + + + + + + + + + + +
+
+
+
+ ) + + const readyOverlay = ( +
+
+
+ +
+
+

Try these commands:

+
+ lists + leads +
+
+
+
+ ) + + if (bootState === "booting") return bootOverlay + return (
- {ollamaStatus === false && ( -
- - Ollama not responding. Start it with ollama serve - -
- )} + {bootState === "ready" && readyOverlay}
{messages.map((msg, i) => ( @@ -162,7 +233,10 @@ export function AIChat() {
- + + + +
)} @@ -191,10 +265,15 @@ export function AIChat() { disabled={loading || !input.trim()} className="h-9 w-9 rounded-lg bg-[#1BB0CE] hover:bg-[#1BB0CE]/80 disabled:opacity-40 flex items-center justify-center flex-none transition-colors" > - {loading ? : } + {loading ? ( + + + + + ) : } ) -} +} \ No newline at end of file diff --git a/src/lib/ai.ts b/src/lib/ai.ts index 9060296..91cee9c 100644 --- a/src/lib/ai.ts +++ b/src/lib/ai.ts @@ -1,45 +1,76 @@ +import http from "http" + const AI_SERVICE = process.env.AI_SERVICE_URL || "http://localhost:3001" +function parseUrl(url: string) { + const u = new URL(url) + return { hostname: u.hostname, port: parseInt(u.port) || 80, path: u.pathname + u.search } +} + export async function chatWithAI(message: string, jwtToken: string) { - const url = `${AI_SERVICE}/ai/chat` const body = JSON.stringify({ message }) + const { hostname, port, path } = parseUrl(`${AI_SERVICE}/ai/chat`) - const res = await fetch(url, { - method: "POST", - headers: { "Content-Type": "application/json", Authorization: `Bearer ${jwtToken}` }, - body, + const text = await new Promise((resolve, reject) => { + const req = http.request( + { hostname, port, path, method: "POST", headers: { "Content-Type": "application/json", "Authorization": `Bearer ${jwtToken}`, "Content-Length": Buffer.byteLength(body) } }, + (res) => { + let data = "" + res.on("data", (chunk) => data += chunk) + res.on("end", () => { + if (res.statusCode && res.statusCode >= 200 && res.statusCode < 300) { + resolve(data) + } else { + reject(new Error(`AI error ${res.statusCode}: ${data.substring(0, 200)}`)) + } + }) + } + ) + req.on("error", reject) + req.write(body) + req.end() }) - const text = await res.text() - - if (!res.ok) { - throw new Error(`AI error ${res.status}: ${text.substring(0, 200)}`) - } - const data = JSON.parse(text) return data.response || "" } -export async function fetchJobs() { +export async function checkAiServiceStatus() { + const { hostname, port, path } = parseUrl(`${AI_SERVICE}/health`) try { - const res = await fetch(`${AI_SERVICE}/ai/jobs`) - if (!res.ok) return [] - const data = await res.json() + await new Promise((resolve, reject) => { + const req = http.get({ hostname, port, path, timeout: 3000 }, (res) => { + let data = "" + res.on("data", (chunk) => data += chunk) + res.on("end", () => { + try { resolve(JSON.parse(data).status === "ok" ? undefined : reject(new Error("not ok"))) } catch { reject(new Error("bad response")) } + }) + }) + req.on("error", reject) + req.on("timeout", () => { req.destroy(); reject(new Error("timeout")) }) + }) + return true + } catch { + return false + } +} + +export async function fetchJobs() { + const { hostname, port, path } = parseUrl(`${AI_SERVICE}/ai/jobs`) + try { + const text = await new Promise((resolve, reject) => { + const req = http.get({ hostname, port, path, timeout: 5000 }, (res) => { + let data = "" + res.on("data", (chunk) => data += chunk) + res.on("end", () => resolve(data)) + }) + req.on("error", reject) + req.on("timeout", () => { req.destroy(); reject(new Error("timeout")) }) + }) + const data = JSON.parse(text) return data.jobs || [] } catch { console.warn("Failed to fetch AI jobs") return [] } } - -export async function checkAiServiceStatus() { - try { - const res = await fetch(`${AI_SERVICE}/health`) - if (!res.ok) return false - const data = await res.json() - return data.status === "ok" - } catch { - console.warn("Failed to check AI service status") - return false - } -}