Updated Ai to pull from facebook, added a loading SVG to make user wait for server too boot up, and added commands that can be used to start off the AI exstraction

ai-server/index.mjs

@@ -96,7 +96,55 @@ function appendToImprovementLog(entry) {
 }
 
 // ── Chat handler ────────────────────────────────────────────────
+async function scrapeFacebook() {
+  const profilePath = process.env.FX_PROFILE || ""
+  const urlPath = `/scrape/facebook?force=true${profilePath ? `&profile_path=${encodeURIComponent(profilePath)}` : ""}`
+  const logPath = "C:\\Users\\USER-PC\\AppData\\Local\\Temp\\opencode\\ai-scrape-debug.log"
+  try {
+    const body = await new Promise((resolve, reject) => {
+      const req = http.request({ hostname: "127.0.0.1", port: 3008, path: urlPath, method: "POST", timeout: 360000 }, (res) => {
+        let data = ""
+        res.on("data", (c) => data += c)
+        res.on("end", () => resolve(data))
+        res.on("error", reject)
+      })
+      req.on("timeout", () => { req.destroy(); reject(new Error("timeout")) })
+      req.on("error", reject)
+      req.end()
+    })
+    const data = JSON.parse(body)
+    return data
+  } catch (e) {
+    return null
+  }
+}
+
+function formatLeads(leads) {
+  if (!leads || leads.length === 0) return "No leads found from the latest scrape."
+  let output = `**${leads.length} leads found:**\n\n`
+  for (let i = 0; i < leads.length; i++) {
+    const l = leads[i]
+    output += `${i + 1}. ${l.title || "No title"}\n`
+    if (l.author) output += `   Author: ${l.author}\n`
+    if (l.date) output += `   Date: ${l.date}\n`
+    if (l.url) output += `   URL: ${l.url}\n`
+    output += "\n"
+  }
+  return output.trim()
+}
+
 async function handleChat(userMessage, userId, userRole) {
+  const lowerMsg = userMessage.toLowerCase()
+  const triggerWords = ["lists", "listings", "leads", "recent leads", "pull leads", "show me leads", "show listings"]
+
+  if (triggerWords.some(w => lowerMsg.includes(w))) {
+    const result = await scrapeFacebook()
+    if (result && result.success) {
+      return formatLeads(result.leads)
+    }
+    return "Scraper returned no results or encountered an error. Try again later."
+  }
+
   const jobs = loadedJobs
   const instructions = readInstructions()
 

browser-use-service/main.py

@@ -369,7 +369,24 @@ async def _get_article_elements(page) -> list[dict]:
         return results;
     }''')
 
-async def search_facebook(page, query: str) -> list[dict]:
+async def _ensure_page(page, context):
+    try:
+        await page.evaluate('1')
+        return page
+    except Exception:
+        logger.warning("Page was closed mid-scrape, creating a fresh page")
+        page = await context.new_page()
+        try:
+            await page.goto('https://www.google.com/', wait_until='domcontentloaded', timeout=15000)
+            await page.wait_for_timeout(random.randint(1000, 3000))
+        except Exception:
+            logger.warning("Google navigation failed during page recreation")
+        await page.goto('https://www.facebook.com/', wait_until='domcontentloaded', timeout=30000)
+        await page.wait_for_timeout(random.randint(3000, 8000))
+        return page
+
+async def search_facebook(page, context, query: str):
+    page = await _ensure_page(page, context)
     url = f'https://www.facebook.com/search/posts/?q={urllib.parse.quote(query)}'
     try:
         await page.goto(url, wait_until='domcontentloaded', timeout=30000)
@@ -400,9 +417,9 @@ async def search_facebook(page, query: str) -> list[dict]:
             raw = await page.evaluate('document.body.innerText')
             posts = _extract_posts_from_text(raw, url)
     except Exception as e:
-        logger.warning("Facebook search failed: %s", e)
+        logger.warning("Facebook search '%s' failed: %s", query, e)
-        return []
+        return page, []
-    return posts
+    return page, posts
 
 def cleanup_chrome():
     import subprocess, signal
@@ -425,7 +442,7 @@ async def scrape_facebook(profile_path: str | None = None, force: bool = False)
             )
             viewport = random.choice(VIEWPORTS)
             context = await browser.new_context(
-                user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:150.0) Gecko/20100101 Firefox/150.0',
+                user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36',
                 viewport=viewport,
             )
 
@@ -445,8 +462,11 @@ async def scrape_facebook(profile_path: str | None = None, force: bool = False)
 
             page = await context.new_page()
             # Navigate through google first for a legitimate Referer header
-            await page.goto('https://www.google.com/', wait_until='domcontentloaded', timeout=15000)
+            try:
-            await page.wait_for_timeout(random.randint(1000, 3000))
+                await page.goto('https://www.google.com/', wait_until='domcontentloaded', timeout=15000)
+                await page.wait_for_timeout(random.randint(1000, 3000))
+            except Exception:
+                logger.warning("Google navigation failed (IP may be blocked), trying Facebook directly")
             await page.goto('https://www.facebook.com/', wait_until='domcontentloaded', timeout=30000)
             await page.wait_for_timeout(random.randint(3000, 8000))
 
@@ -472,23 +492,22 @@ async def scrape_facebook(profile_path: str | None = None, force: bool = False)
             all_posts = []
             searches = random.sample(FB_SEARCHES, k=random.randint(5, 8))
             for i, query in enumerate(searches):
-                try:
+                page, posts = await search_facebook(page, context, query)
-                    posts = await search_facebook(page, query)
+                all_posts.extend(posts)
-                    all_posts.extend(posts)
+                if not posts:
-                    # Between searches: random break with possible small scroll
+                    continue
-                    if random.random() < 0.4:
+                # Between searches: random break with possible small scroll
-                        await page.evaluate(f"window.scrollBy(0, {random.randint(-300, 300)})")
+                if random.random() < 0.4:
-                    delay = random.uniform(8, 25)
+                    await page.evaluate(f"window.scrollBy(0, {random.randint(-300, 300)})")
-                    await page.wait_for_timeout(int(delay * 1000))
+                delay = random.uniform(8, 25)
-                    # Tab switch: 15% chance after 2nd-3rd search
+                await page.wait_for_timeout(int(delay * 1000))
-                    if i == random.randint(1, 2) and random.random() < 0.15:
+                # Tab switch: 15% chance after 2nd-3rd search
-                        new_page = await context.new_page()
+                if i == random.randint(1, 2) and random.random() < 0.15:
-                        await new_page.goto('https://www.messenger.com/', wait_until='domcontentloaded', timeout=15000)
+                    new_page = await context.new_page()
-                        await new_page.wait_for_timeout(random.randint(3000, 8000))
+                    await new_page.goto('https://www.messenger.com/', wait_until='domcontentloaded', timeout=15000)
-                        await new_page.close()
+                    await new_page.wait_for_timeout(random.randint(3000, 8000))
-                        await page.wait_for_timeout(random.randint(1000, 3000))
+                    await new_page.close()
-                except Exception as e:
+                    page = await _ensure_page(page, context)
-                    logger.warning("Facebook search '%s' failed: %s", query, e)
 
             # Idle before closing — human would pause
             if random.random() < 0.5:

browser-use-service/test_scrape.py

@@ -0,0 +1,21 @@
+import asyncio, sys, logging
+logging.basicConfig(level=logging.DEBUG, format='%(asctime)s [%(levelname)s] %(message)s')
+sys.path.insert(0, '.')
+from main import get_fb_cookies, scrape_facebook
+
+async def test():
+    print('Step 1: get cookies')
+    cookies = await get_fb_cookies()
+    print(f'Got {len(cookies)} cookies')
+    if not cookies:
+        print('No cookies - will still try scrape')
+    print('Step 2: run scrape_facebook')
+    result = await scrape_facebook(force=True)
+    print(f'Result success={result["success"]} leads={len(result["leads"])} error={result.get("error")}')
+    if result.get('leads'):
+        for l in result['leads'][:3]:
+            print(f'  - {l["title"][:100]}')
+    if result.get('error'):
+        print(f'  Error: {result["error"]}')
+
+asyncio.run(test())

package.json

@@ -9,7 +9,7 @@
     "dev:precheck": "powershell -NoProfile -Command \"Get-NetTCPConnection -State Listen | Where-Object { $_.LocalPort -in 3001,3006,3008 } | ForEach-Object { Stop-Process -Id $_.OwningProcess -Force -ErrorAction SilentlyContinue; Write-Host ('Freed port '+$_.LocalPort) }\"",
     "dev:ollama": "powershell -NoProfile -Command \"$ollama = if (Get-Command ollama -ErrorAction SilentlyContinue) { 'ollama' } else { Join-Path $env:LOCALAPPDATA 'Programs\\Ollama\\ollama.exe' }; if (-not (Get-Process ollama -ErrorAction SilentlyContinue)) { Start-Process $ollama -ArgumentList 'serve' -WindowStyle Hidden; Start-Sleep 3 }; exit 0\"",
     "dev:rust": "node ai-server/index.mjs",
-    "dev:browser-use": "cd browser-use-service && python main.py",
+    "dev:browser-use": "set FX_PROFILE=C:\\Users\\USER-PC\\AppData\\Roaming\\Mozilla\\Firefox\\Profiles\\h8p11vlj.default-release && cd browser-use-service && python main.py",
     "build": "next build",
     "start": "next start -p 3006",
     "lint": "eslint"

src/app/api/ai/chat/route.ts

@@ -1,8 +1,29 @@
 import { NextRequest, NextResponse } from "next/server"
+import { chatWithAI } from "@/lib/ai"
+import { getSessionUser } from "@/lib/auth"
 
-// This route handler has a known issue with Next.js 15 fetch augmentation
-// on this platform. The client-side code calls the AI server directly
-// via browser fetch (CORS is open). This handler returns a fallback.
 export async function POST(request: NextRequest) {
-  return NextResponse.json({ error: "AI service unavailable on server. Use client-side mode." }, { status: 503 })
+  try {
+    const user = await getSessionUser()
+    if (!user) {
+      return NextResponse.json({ error: "Unauthorized" }, { status: 401 })
+    }
+
+    const { message } = await request.json()
+    if (!message || typeof message !== "string") {
+      return NextResponse.json({ error: "Message is required" }, { status: 400 })
+    }
+
+    const sessionCookie = request.cookies.get("session")
+    const jwtToken = sessionCookie?.value
+    if (!jwtToken) {
+      return NextResponse.json({ error: "No session token" }, { status: 401 })
+    }
+
+    const response = await chatWithAI(message, jwtToken)
+    return NextResponse.json({ response })
+  } catch (error: any) {
+    console.error("AI chat error:", error)
+    return NextResponse.json({ error: error.message || "AI service error" }, { status: 500 })
+  }
 }

src/components/ai/ai-chat.tsx

@@ -1,7 +1,7 @@
 "use client"
 
 import { useState, useRef, useEffect, Fragment } from "react"
-import { Send, Loader2, Bot, User, RefreshCw, AlertCircle } from "lucide-react"
+import { Send, Bot, User, RefreshCw, AlertCircle, Check, Terminal } from "lucide-react"
 
 function linkifyText(text: string) {
   const urlRegex = /(https?:\/\/[^\s<]+[^\s<.,;:!?)\]}>])/
@@ -24,11 +24,28 @@ export function AIChat() {
   const [input, setInput] = useState("")
   const [loading, setLoading] = useState(false)
   const [error, setError] = useState("")
-  const [ollamaStatus, setOllamaStatus] = useState<boolean | null>(null)
+  const [bootState, setBootState] = useState<"booting" | "ready" | "error">("booting")
   const messagesEndRef = useRef<HTMLDivElement>(null)
 
+  const checkServer = async () => {
+    try {
+      const res = await fetch("/api/ai/chat", {
+        method: "POST",
+        headers: { "Content-Type": "application/json" },
+        body: JSON.stringify({ message: "ping" }),
+      })
+      if (res.status !== 503) {
+        setBootState("ready")
+      } else {
+        setTimeout(checkServer, 2000)
+      }
+    } catch {
+      setTimeout(checkServer, 2000)
+    }
+  }
+
   useEffect(() => {
-    fetch(`${AI_API}/ai/jobs`)
+    fetch("/api/ai/jobs")
       .then((r) => r.json())
       .then((data) => {
         if (data.jobs?.length) {
@@ -56,28 +73,13 @@ export function AIChat() {
           },
         ])
       })
+    checkServer()
   }, [])
 
-
-
   useEffect(() => {
     messagesEndRef.current?.scrollIntoView({ behavior: "smooth" })
   }, [messages])
 
-  const AI_API = process.env.NEXT_PUBLIC_AI_URL || "http://127.0.0.1:3001"
-
-  const checkOllama = async () => {
-    try {
-      const res = await fetch(`${AI_API}/health`)
-      const data = await res.json()
-      setOllamaStatus(data.status === "ok")
-    } catch {
-      setOllamaStatus(false)
-    }
-  }
-
-  useEffect(() => { checkOllama() }, [])
-
   const sendMessage = async () => {
     const msg = input.trim()
     if (!msg || loading) return
@@ -88,7 +90,7 @@ export function AIChat() {
     setLoading(true)
 
     try {
-      const res = await fetch(`${AI_API}/ai/chat`, {
+      const res = await fetch("/api/ai/chat", {
         method: "POST",
         headers: { "Content-Type": "application/json" },
         body: JSON.stringify({ message: msg }),
@@ -120,17 +122,86 @@ export function AIChat() {
     }
   }
 
+  const bootOverlay = (
+    <div className="flex-1 flex items-center justify-center">
+      <style>{`
+        @keyframes walk {
+          0%, 100% { transform: translateX(0) translateY(0); }
+          25% { transform: translateX(12px) translateY(-4px); }
+          50% { transform: translateX(24px) translateY(0); }
+          75% { transform: translateX(12px) translateY(-4px); }
+        }
+        @keyframes legLeft {
+          0%, 100% { transform: rotate(-10deg); }
+          50% { transform: rotate(10deg); }
+        }
+        @keyframes legRight {
+          0%, 100% { transform: rotate(10deg); }
+          50% { transform: rotate(-10deg); }
+        }
+        @keyframes armLeft {
+          0%, 100% { transform: rotate(15deg); }
+          50% { transform: rotate(-15deg); }
+        }
+        @keyframes armRight {
+          0%, 100% { transform: rotate(-15deg); }
+          50% { transform: rotate(15deg); }
+        }
+        @keyframes blink {
+          0%, 45%, 55%, 100% { height: 4px; }
+          50% { height: 1px; }
+        }
+        .robot-walk { animation: walk 0.6s ease-in-out infinite; }
+        .robot-leg-l { transform-origin: top center; animation: legLeft 0.3s ease-in-out infinite; }
+        .robot-leg-r { transform-origin: top center; animation: legRight 0.3s ease-in-out infinite; }
+        .robot-arm-l { transform-origin: top center; animation: armLeft 0.3s ease-in-out infinite; }
+        .robot-arm-r { transform-origin: top center; animation: armRight 0.3s ease-in-out infinite; }
+        .robot-eye { animation: blink 2s ease-in-out infinite; }
+      `}</style>
+      <div className="flex flex-col items-center gap-4">
+        <p className="text-sm text-[#6a6a75]">Servers booting...</p>
+        <div className="h-[50px] w-[100px] bg-[#1a1a24] border border-[#2a2a35] rounded-lg flex items-center justify-center overflow-hidden">
+          <div className="robot-walk relative">
+            <svg width="40" height="36" viewBox="0 0 40 36" fill="none">
+              <rect x="10" y="2" width="20" height="16" rx="3" fill="#1BB0CE" opacity="0.9"/>
+              <rect x="6" y="6" width="6" height="2" rx="1" className="robot-arm-l" fill="#1BB0CE" opacity="0.7"/>
+              <rect x="28" y="6" width="6" height="2" rx="1" className="robot-arm-r" fill="#1BB0CE" opacity="0.7"/>
+              <rect x="14" y="5" width="4" height="4" rx="1" fill="#0d1117"/>
+              <rect x="22" y="5" width="4" height="4" rx="1" fill="#0d1117"/>
+              <circle cx="16" cy="7" r="1.5" className="robot-eye" fill="#1BB0CE"/>
+              <circle cx="24" cy="7" r="1.5" className="robot-eye" fill="#1BB0CE"/>
+              <rect x="17" y="10" width="6" height="2" rx="1" fill="#0d1117"/>
+              <rect x="12" y="18" width="5" height="10" rx="2" className="robot-leg-l" fill="#1BB0CE" opacity="0.8"/>
+              <rect x="23" y="18" width="5" height="10" rx="2" className="robot-leg-r" fill="#1BB0CE" opacity="0.8"/>
+            </svg>
+          </div>
+        </div>
+      </div>
+    </div>
+  )
+
+  const readyOverlay = (
+    <div className="flex-1 flex items-center justify-center">
+      <div className="flex flex-col items-center gap-4">
+        <div className="h-[50px] w-[100px] bg-[#1a1a24] border border-[#2a2a35] rounded-lg flex items-center justify-center">
+          <Check className="h-6 w-6 text-green-400" />
+        </div>
+        <div className="text-center space-y-1">
+          <p className="text-xs text-[#6a6a75]">Try these commands:</p>
+          <div className="flex gap-2">
+            <code className="text-xs px-2 py-1 rounded bg-[#2a2a35] text-[#1BB0CE] flex items-center gap-1"><Terminal className="h-3 w-3" /> lists</code>
+            <code className="text-xs px-2 py-1 rounded bg-[#2a2a35] text-[#1BB0CE] flex items-center gap-1"><Terminal className="h-3 w-3" /> leads</code>
+          </div>
+        </div>
+      </div>
+    </div>
+  )
+
+  if (bootState === "booting") return bootOverlay
+
   return (
     <div className="flex flex-col h-full">
-      {ollamaStatus === false && (
+      {bootState === "ready" && readyOverlay}
-        <div className="flex items-center gap-2 px-4 py-2 bg-amber-500/10 border-b border-amber-500/20 text-amber-400 text-xs">
-          <AlertCircle className="h-3.5 w-3.5 flex-none" />
-          <span className="flex-1">Ollama not responding. Start it with <code className="bg-amber-500/20 px-1 rounded">ollama serve</code></span>
-          <button type="button" onClick={checkOllama} className="hover:text-amber-300">
-            <RefreshCw className="h-3.5 w-3.5" />
-          </button>
-        </div>
-      )}
 
       <div className="flex-1 overflow-y-auto p-4 space-y-4 scrollbar-thin">
         {messages.map((msg, i) => (
@@ -162,7 +233,10 @@ export function AIChat() {
               <Bot className="h-4 w-4 text-[#1BB0CE]" />
             </div>
             <div className="max-w-[75%] rounded-lg px-4 py-2.5 bg-[#1a1a24] border border-[#2a2a35]">
-              <Loader2 className="h-4 w-4 animate-spin text-[#1BB0CE]" />
+              <svg className="h-4 w-4 animate-spin text-[#1BB0CE]" viewBox="0 0 24 24" fill="none">
+                <circle cx="12" cy="12" r="10" stroke="currentColor" strokeWidth="4" opacity="0.25"/>
+                <path d="M12 2a10 10 0 0 1 10 10" stroke="currentColor" strokeWidth="4" strokeLinecap="round"/>
+              </svg>
             </div>
           </div>
         )}
@@ -191,7 +265,12 @@ export function AIChat() {
             disabled={loading || !input.trim()}
             className="h-9 w-9 rounded-lg bg-[#1BB0CE] hover:bg-[#1BB0CE]/80 disabled:opacity-40 flex items-center justify-center flex-none transition-colors"
           >
-            {loading ? <Loader2 className="h-4 w-4 animate-spin" /> : <Send className="h-4 w-4" />}
+            {loading ? (
+              <svg className="h-4 w-4 animate-spin" viewBox="0 0 24 24" fill="none">
+                <circle cx="12" cy="12" r="10" stroke="currentColor" strokeWidth="4" opacity="0.25"/>
+                <path d="M12 2a10 10 0 0 1 10 10" stroke="currentColor" strokeWidth="4" strokeLinecap="round"/>
+              </svg>
+            ) : <Send className="h-4 w-4" />}
           </button>
         </div>
       </div>

src/lib/ai.ts

@@ -1,45 +1,76 @@
+import http from "http"
+
 const AI_SERVICE = process.env.AI_SERVICE_URL || "http://localhost:3001"
 
+function parseUrl(url: string) {
+  const u = new URL(url)
+  return { hostname: u.hostname, port: parseInt(u.port) || 80, path: u.pathname + u.search }
+}
+
 export async function chatWithAI(message: string, jwtToken: string) {
-  const url = `${AI_SERVICE}/ai/chat`
   const body = JSON.stringify({ message })
+  const { hostname, port, path } = parseUrl(`${AI_SERVICE}/ai/chat`)
 
-  const res = await fetch(url, {
+  const text = await new Promise<string>((resolve, reject) => {
-    method: "POST",
+    const req = http.request(
-    headers: { "Content-Type": "application/json", Authorization: `Bearer ${jwtToken}` },
+      { hostname, port, path, method: "POST", headers: { "Content-Type": "application/json", "Authorization": `Bearer ${jwtToken}`, "Content-Length": Buffer.byteLength(body) } },
-    body,
+      (res) => {
+        let data = ""
+        res.on("data", (chunk) => data += chunk)
+        res.on("end", () => {
+          if (res.statusCode && res.statusCode >= 200 && res.statusCode < 300) {
+            resolve(data)
+          } else {
+            reject(new Error(`AI error ${res.statusCode}: ${data.substring(0, 200)}`))
+          }
+        })
+      }
+    )
+    req.on("error", reject)
+    req.write(body)
+    req.end()
   })
 
-  const text = await res.text()
-
-  if (!res.ok) {
-    throw new Error(`AI error ${res.status}: ${text.substring(0, 200)}`)
-  }
-
   const data = JSON.parse(text)
   return data.response || ""
 }
 
-export async function fetchJobs() {
+export async function checkAiServiceStatus() {
+  const { hostname, port, path } = parseUrl(`${AI_SERVICE}/health`)
   try {
-    const res = await fetch(`${AI_SERVICE}/ai/jobs`)
+    await new Promise<void>((resolve, reject) => {
-    if (!res.ok) return []
+      const req = http.get({ hostname, port, path, timeout: 3000 }, (res) => {
-    const data = await res.json()
+        let data = ""
+        res.on("data", (chunk) => data += chunk)
+        res.on("end", () => {
+          try { resolve(JSON.parse(data).status === "ok" ? undefined : reject(new Error("not ok"))) } catch { reject(new Error("bad response")) }
+        })
+      })
+      req.on("error", reject)
+      req.on("timeout", () => { req.destroy(); reject(new Error("timeout")) })
+    })
+    return true
+  } catch {
+    return false
+  }
+}
+
+export async function fetchJobs() {
+  const { hostname, port, path } = parseUrl(`${AI_SERVICE}/ai/jobs`)
+  try {
+    const text = await new Promise<string>((resolve, reject) => {
+      const req = http.get({ hostname, port, path, timeout: 5000 }, (res) => {
+        let data = ""
+        res.on("data", (chunk) => data += chunk)
+        res.on("end", () => resolve(data))
+      })
+      req.on("error", reject)
+      req.on("timeout", () => { req.destroy(); reject(new Error("timeout")) })
+    })
+    const data = JSON.parse(text)
     return data.jobs || []
   } catch {
     console.warn("Failed to fetch AI jobs")
     return []
   }
 }
-
-export async function checkAiServiceStatus() {
-  try {
-    const res = await fetch(`${AI_SERVICE}/health`)
-    if (!res.ok) return false
-    const data = await res.json()
-    return data.status === "ok"
-  } catch {
-    console.warn("Failed to check AI service status")
-    return false
-  }
-}