feat: add Facebook account tracking and management

- Introduced new migration for `facebook_accounts` and `facebook_scrape_logs` tables.
- Updated the scraping logic to handle Facebook accounts, including success and failure tracking.
- Implemented API endpoints for managing Facebook accounts and their scrape logs.
- Enhanced user permissions to restrict access to Facebook account management to admins and super admins.
- Added a dialog component for displaying and managing Facebook accounts in the UI.
- Updated lead fetching logic to include user role checks for assignment and access.
This commit is contained in:
Ace
2026-06-23 14:18:18 +02:00
parent 1adc4806fa
commit ff56cea4b8
25 changed files with 778 additions and 216 deletions
Binary file not shown.
+169 -134
View File
@@ -1,9 +1,7 @@
import os, json, asyncio, re, shutil, sqlite3, traceback, urllib.parse, random, time, logging import os, json, asyncio, re, shutil, sqlite3, urllib.parse, random, logging
from datetime import datetime, timedelta from datetime import datetime, timedelta
from bs4 import BeautifulSoup from fastapi import FastAPI, Query
import httpx from fastapi.middleware.cors import CORSMiddleware
from fastapi import FastAPI
from pydantic import BaseModel
import uvicorn import uvicorn
from playwright.async_api import async_playwright from playwright.async_api import async_playwright
@@ -11,22 +9,20 @@ logging.basicConfig(level=logging.INFO, format='%(asctime)s [%(levelname)s] %(me
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
app = FastAPI() app = FastAPI()
app.add_middleware(
CORSMiddleware,
allow_origins=["http://localhost:3006", "http://127.0.0.1:3006"],
allow_methods=["POST"],
allow_headers=["*"],
)
PORT = int(os.getenv("PORT", "3008")) PORT = int(os.getenv("PORT", "3008"))
OLLAMA_URL = os.getenv("OLLAMA_URL", "http://localhost:11434") OLLAMA_URL = os.getenv("OLLAMA_URL", "http://localhost:11434")
CLASSIFY_MODEL = os.getenv("CLASSIFY_MODEL", "dolphin-llama3:8b") CLASSIFY_MODEL = os.getenv("CLASSIFY_MODEL", "dolphin-llama3:8b")
FX_PROFILE = os.getenv('FX_PROFILE', '') FX_PROFILE = os.getenv('FX_PROFILE', '')
FX_COOKIE_DB = os.path.join(FX_PROFILE, 'cookies.sqlite') if FX_PROFILE else ''
HEADERS = { BROAD_KEYWORDS = [
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
}
last_scrape_time: datetime | None = None
STRICT_KEYWORDS = [
"website", "web design", "web develop", "web dev", "website", "web design", "web develop", "web dev",
"build my website", "build a website", "create a website", "build my website", "build a website", "create a website",
"need web", "looking for web", "new website", "need web", "looking for web", "new website",
@@ -34,9 +30,6 @@ STRICT_KEYWORDS = [
"need a website", "my website", "business website", "need a website", "my website", "business website",
"need a designer", "help with my website", "need a designer", "help with my website",
"redesign", "update my website", "redesign", "update my website",
]
BROAD_KEYWORDS = STRICT_KEYWORDS + [
"looking for", "need a", "need an", "looking to", "looking for", "need a", "need an", "looking to",
"help me build", "create my", "build me", "help me build", "create my", "build me",
"design my", "make me a", "would like", "design my", "make me a", "would like",
@@ -44,10 +37,6 @@ BROAD_KEYWORDS = STRICT_KEYWORDS + [
"want someone", "need help with", "want someone", "need help with",
] ]
def kw_match_strict(text: str) -> bool:
t = text.lower()
return any(kw in t for kw in STRICT_KEYWORDS)
def kw_match(text: str) -> bool: def kw_match(text: str) -> bool:
t = text.lower() t = text.lower()
return any(kw in t for kw in BROAD_KEYWORDS) return any(kw in t for kw in BROAD_KEYWORDS)
@@ -67,14 +56,27 @@ FB_SEARCHES = [
"need a site for my business", "need a site for my business",
] ]
async def get_fb_cookies(): VIEWPORTS = [
if not FX_COOKIE_DB or not os.path.exists(FX_COOKIE_DB): {'width': 1280, 'height': 800},
logger.warning("FX_COOKIE_DB not found or FX_PROFILE not set") {'width': 1366, 'height': 768},
{'width': 1440, 'height': 900},
{'width': 1536, 'height': 864},
{'width': 1920, 'height': 1080},
]
async def get_fb_cookies(profile_path: str | None = None):
cookie_db_path = profile_path or FX_PROFILE
if not cookie_db_path:
logger.warning("No profile path provided")
return [] return []
tmp = os.path.join(os.path.dirname(FX_COOKIE_DB), f'cookies_tmp_{os.getpid()}.sqlite') cookie_db = os.path.join(cookie_db_path, 'cookies.sqlite')
if not os.path.exists(cookie_db):
logger.warning("Cookie DB not found at %s", cookie_db)
return []
tmp = os.path.join(os.path.dirname(cookie_db), f'cookies_tmp_{os.getpid()}.sqlite')
for attempt in range(3): for attempt in range(3):
try: try:
shutil.copy2(FX_COOKIE_DB, tmp) shutil.copy2(cookie_db, tmp)
break break
except PermissionError: except PermissionError:
if attempt < 2: if attempt < 2:
@@ -182,35 +184,95 @@ def _extract_posts_from_text(raw: str, url: str) -> list[dict]:
}) })
return posts return posts
async def human_scroll(page, steps: int = None, total_delay: float = None):
steps = steps or random.randint(2, 5)
total_delay = total_delay or random.uniform(6, 18)
step_delay = total_delay / steps
for i in range(steps):
scroll_dist = random.randint(200, 600)
await page.evaluate(f"window.scrollBy(0, {scroll_dist})")
await page.wait_for_timeout(int(step_delay * 1000))
if random.random() < 0.3:
await page.evaluate(f"window.scrollBy(0, -{random.randint(100, 300)})")
await page.wait_for_timeout(random.randint(1000, 3000))
async def random_idle(page):
action = random.random()
if action < 0.15:
try:
elems = await page.query_selector_all('a, span, div[role="button"]')
if elems:
el = random.choice(elems[:20])
box = await el.bounding_box()
if box:
x = box['x'] + box['width'] / 2 + random.randint(-20, 20)
y = box['y'] + box['height'] / 2 + random.randint(-10, 10)
await page.mouse.move(x, y, steps=random.randint(20, 40))
await page.wait_for_timeout(random.randint(500, 2000))
except Exception:
pass
async def search_facebook(page, query: str) -> list[dict]: async def search_facebook(page, query: str) -> list[dict]:
url = f'https://www.facebook.com/search/posts/?q={urllib.parse.quote(query)}' url = f'https://www.facebook.com/search/posts/?q={urllib.parse.quote(query)}'
try: try:
await page.goto(url, wait_until='domcontentloaded', timeout=30000) await page.goto(url, wait_until='domcontentloaded', timeout=30000)
await page.wait_for_timeout(6000) await page.wait_for_timeout(random.randint(3000, 8000))
except Exception as e:
logger.warning("Facebook search navigation failed for '%s': %s", query, e) await human_scroll(page, steps=random.randint(2, 4), total_delay=random.uniform(6, 15))
return []
# Accidental overscroll: 10% chance to jump to bottom and back
if random.random() < 0.1:
await page.evaluate("window.scrollTo(0, document.body.scrollHeight)")
await page.wait_for_timeout(random.randint(1000, 3000))
await page.evaluate("window.scrollBy(0, -300)")
await page.wait_for_timeout(random.randint(1000, 3000))
if random.random() < 0.2:
await page.evaluate("window.scrollTo(0, 0)")
await page.wait_for_timeout(random.randint(2000, 5000))
if random.random() < 0.3:
await random_idle(page)
try:
raw = await page.evaluate('document.body.innerText') raw = await page.evaluate('document.body.innerText')
except Exception as e: except Exception as e:
logger.warning("Failed to evaluate page text: %s", e) logger.warning("Facebook search failed: %s", e)
return [] return []
return _extract_posts_from_text(raw, url) return _extract_posts_from_text(raw, url)
async def scrape_facebook() -> list[dict]: def cleanup_chrome():
all_posts = [] import subprocess, signal
fb_cookies = await get_fb_cookies() try:
subprocess.run(["taskkill", "/F", "/IM", "chrome-headless-shell.exe"], capture_output=True, timeout=5)
except Exception:
pass
async def scrape_facebook(profile_path: str | None = None, force: bool = False) -> dict:
cleanup_chrome()
fb_cookies = await get_fb_cookies(profile_path)
if not fb_cookies: if not fb_cookies:
logger.warning("No Facebook cookies available") logger.warning("No Facebook cookies available")
return [] return {"success": False, "leads": [], "flagged": False, "flag_reason": None, "error": "No cookies available"}
try: try:
async with async_playwright() as pw: async with async_playwright() as pw:
browser = await pw.chromium.launch(headless=True) browser = await pw.chromium.launch(
headless=True,
args=['--disable-blink-features=AutomationControlled']
)
viewport = random.choice(VIEWPORTS)
context = await browser.new_context( context = await browser.new_context(
user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:150.0) Gecko/20100101 Firefox/150.0', user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:150.0) Gecko/20100101 Firefox/150.0',
viewport={'width': 1280, 'height': 800}, viewport=viewport,
) )
await context.add_init_script("""
Object.defineProperty(navigator, 'webdriver', { get: () => false });
Object.defineProperty(navigator, 'languages', { get: () => ['en-US', 'en'] });
Object.defineProperty(navigator, 'platform', { get: () => 'Win32' });
window.chrome = { runtime: {} };
Object.defineProperty(navigator, 'hardwareConcurrency', { get: () => 8 });
Object.defineProperty(navigator, 'deviceMemory', { get: () => 8 });
""")
for c in fb_cookies: for c in fb_cookies:
try: try:
await context.add_cookies([c]) await context.add_cookies([c])
@@ -218,36 +280,77 @@ async def scrape_facebook() -> list[dict]:
logger.warning("Failed to inject cookie %s: %s", c.get('name'), e) logger.warning("Failed to inject cookie %s: %s", c.get('name'), e)
page = await context.new_page() page = await context.new_page()
# Navigate through google first for a legitimate Referer header
await page.goto('https://www.google.com/', wait_until='domcontentloaded', timeout=15000)
await page.wait_for_timeout(random.randint(1000, 3000))
await page.goto('https://www.facebook.com/', wait_until='domcontentloaded', timeout=30000) await page.goto('https://www.facebook.com/', wait_until='domcontentloaded', timeout=30000)
await page.wait_for_timeout(5000) await page.wait_for_timeout(random.randint(3000, 8000))
url = page.url url = page.url
if '/login' in url.lower(): if '/login' in url.lower():
logger.warning("Facebook login page detected — cookies may be expired") logger.warning("Facebook login page detected — account flagged")
return [] await browser.close()
return {"success": False, "leads": [], "flagged": True, "flag_reason": "login_page", "error": "Facebook login page detected"}
for query in FB_SEARCHES[:6]: # Browse feed like a human before searching
await human_scroll(page, steps=random.randint(2, 4), total_delay=random.uniform(8, 20))
if random.random() < 0.25:
await page.evaluate("window.scrollTo(0, 0)")
await page.wait_for_timeout(random.randint(2000, 5000))
await human_scroll(page, steps=random.randint(1, 2))
# False start: 30% chance to idle and leave without scraping (skipped when force=true)
if not force and random.random() < 0.3:
await page.wait_for_timeout(random.randint(8000, 20000))
await browser.close()
return {"success": True, "leads": [], "flagged": False, "flag_reason": None, "error": None}
all_posts = []
searches = random.sample(FB_SEARCHES, k=random.randint(3, 6))
for i, query in enumerate(searches):
try: try:
posts = await search_facebook(page, query) posts = await search_facebook(page, query)
all_posts.extend(posts) all_posts.extend(posts)
delay = random.uniform(3, 7) # Between searches: random break with possible small scroll
if random.random() < 0.4:
await page.evaluate(f"window.scrollBy(0, {random.randint(-300, 300)})")
delay = random.uniform(8, 25)
await page.wait_for_timeout(int(delay * 1000)) await page.wait_for_timeout(int(delay * 1000))
# Tab switch: 15% chance after 2nd-3rd search
if i == random.randint(1, 2) and random.random() < 0.15:
new_page = await context.new_page()
await new_page.goto('https://www.messenger.com/', wait_until='domcontentloaded', timeout=15000)
await new_page.wait_for_timeout(random.randint(3000, 8000))
await new_page.close()
await page.wait_for_timeout(random.randint(1000, 3000))
except Exception as e: except Exception as e:
logger.warning("Facebook search '%s' failed: %s", query, e) logger.warning("Facebook search '%s' failed: %s", query, e)
# Idle before closing — human would pause
if random.random() < 0.5:
await page.wait_for_timeout(random.randint(3000, 10000))
await browser.close()
seen = set()
deduped = []
for p in all_posts:
key = p.get('content', '')[:100]
if key not in seen:
seen.add(key)
deduped.append(p)
leads = deduped[:20]
if leads:
leads = await classify_leads(leads)
return {"success": True, "leads": leads[:15], "flagged": False, "flag_reason": None, "error": None}
except Exception as e: except Exception as e:
logger.error("Facebook scrape failed: %s", e) logger.error("Facebook scrape failed: %s", e)
return [] return {"success": False, "leads": [], "flagged": False, "flag_reason": None, "error": str(e)}
seen = set()
deduped = []
for p in all_posts:
key = p.get('content', '')[:100]
if key not in seen:
seen.add(key)
deduped.append(p)
return deduped[:20]
async def ask_ollama(prompt: str) -> str: async def ask_ollama(prompt: str) -> str:
import httpx
async with httpx.AsyncClient(timeout=120) as c: async with httpx.AsyncClient(timeout=120) as c:
r = await c.post(f"{OLLAMA_URL}/api/chat", json={ r = await c.post(f"{OLLAMA_URL}/api/chat", json={
"model": CLASSIFY_MODEL, "model": CLASSIFY_MODEL,
@@ -262,42 +365,6 @@ async def ask_ollama(prompt: str) -> str:
data = r.json() data = r.json()
return data["message"]["content"] return data["message"]["content"]
async def scrape_warriorforum() -> list[dict]:
results = []
try:
async with httpx.AsyncClient(headers=HEADERS, timeout=15, follow_redirects=True) as c:
r = await c.get('https://www.warriorforum.com/wanted-members-looking-hire-you/')
soup = BeautifulSoup(r.text, 'lxml')
for row in soup.find_all('td', class_='FlexTable-item--title'):
a = row.find('a', href=True)
if not a:
continue
href = a['href']
title = a.text.strip()
if not title or len(title) < 15 or not kw_match_strict(title):
continue
author_div = row.find('div', class_='FlexTable-item-author')
author = author_div.get_text(strip=True).lstrip('by') if author_div else ''
date_str = ''
date_div = row.find('div', class_=lambda c: c and 'media--available' in c)
if date_div:
txt = date_div.get_text(strip=True)
m = re.search(r'(\d{10})', txt)
if m:
date_str = datetime.utcfromtimestamp(int(m.group(1))).strftime('%Y-%m-%d')
if not href.startswith('http'):
href = 'https://www.warriorforum.com' + href
results.append({
"title": title, "url": href,
"author": author,
"content": title[:300],
"source": "warriorforum",
"date": date_str
})
except Exception:
logger.error("WarriorForum scrape failed", exc_info=True)
return results
async def classify_leads(results: list[dict]) -> list[dict]: async def classify_leads(results: list[dict]) -> list[dict]:
if not results: if not results:
return [] return []
@@ -320,13 +387,10 @@ Return a JSON array like ["yes","no","yes"] matching the order above."""
try: try:
raw = await ask_ollama(prompt) raw = await ask_ollama(prompt)
raw = raw.strip() raw = raw.strip()
# Strip markdown code fences
if raw.startswith("```"): if raw.startswith("```"):
# Remove opening fence
first_nl = raw.find('\n') first_nl = raw.find('\n')
if first_nl != -1: if first_nl != -1:
raw = raw[first_nl + 1:] raw = raw[first_nl + 1:]
# Remove closing fence
if raw.endswith("```"): if raw.endswith("```"):
raw = raw[:-3] raw = raw[:-3]
raw = raw.strip() raw = raw.strip()
@@ -339,18 +403,25 @@ Return a JSON array like ["yes","no","yes"] matching the order above."""
filtered.append(results[i]) filtered.append(results[i])
if filtered: if filtered:
return filtered[:10] return filtered[:10]
# AI successfully classified but returned all "no" — respect that decision
logger.info("AI classified all %d items as NOT LEAD — returning empty", len(results)) logger.info("AI classified all %d items as NOT LEAD — returning empty", len(results))
return [] return []
except Exception as e: except Exception as e:
logger.warning("AI classification failed, falling back to keyword filter: %s", e) logger.warning("AI classification failed, falling back to keyword filter: %s", e)
# Fallback: only use keyword filter when AI failed
if not ai_succeeded: if not ai_succeeded:
strict_keywords = [
"website", "web design", "web develop", "web dev",
"build my website", "build a website", "create a website",
"need web", "looking for web", "new website",
"landing page", "wordpress",
"need a website", "my website", "business website",
"need a designer", "help with my website",
"redesign", "update my website",
]
filtered = [] filtered = []
for r in results: for r in results:
t = r['title'].lower() t = r['title'].lower()
if not kw_match_strict(t): if not any(kw in t for kw in strict_keywords):
continue continue
if any(kw in t for kw in ['i build', 'i offer', 'i create', 'i am a', 'web developer available', if any(kw in t for kw in ['i build', 'i offer', 'i create', 'i am a', 'web developer available',
'affordable web', 'web hosting', 'free website', 'affordable web', 'web hosting', 'free website',
@@ -365,46 +436,10 @@ Return a JSON array like ["yes","no","yes"] matching the order above."""
async def health(): async def health():
return {"status": "ok"} return {"status": "ok"}
class ScrapeRequest(BaseModel):
query: str = ""
@app.post("/scrape/requests")
async def scrape_requests(req: ScrapeRequest):
global last_scrape_time
now = datetime.now()
if last_scrape_time and (now - last_scrape_time) < timedelta(seconds=15):
return {"error": "rate_limited", "retry_after": 15}
last_scrape_time = now
results = await scrape_warriorforum()
if results:
results = await classify_leads(results)
return results[:10]
@app.post("/scrape/facebook") @app.post("/scrape/facebook")
async def scrape_facebook_endpoint(): async def scrape_facebook_endpoint(profile_path: str | None = Query(None), force: bool = Query(False)):
results = await scrape_facebook() result = await scrape_facebook(profile_path, force)
if results: return result
results = await classify_leads(results)
return results[:15]
@app.post("/scrape/all")
async def scrape_all():
results = []
try:
wf = await scrape_warriorforum()
if wf:
wf = await classify_leads(wf)
results.extend(wf[:5])
except Exception as e:
logger.error("WarriorForum scrape in /scrape/all failed: %s", e)
try:
fb = await scrape_facebook()
if fb:
fb = await classify_leads(fb)
results.extend(fb[:8])
except Exception as e:
logger.error("Facebook scrape in /scrape/all failed: %s", e)
return results[:12]
if __name__ == "__main__": if __name__ == "__main__":
uvicorn.run(app, host="0.0.0.0", port=PORT) uvicorn.run(app, host="0.0.0.0", port=PORT)
View File
-7
View File
@@ -1,7 +0,0 @@
python : INFO: Started server process [20044]
+ CategoryInfo : NotSpecified: (INFO: Started server process [20044]:String) [], RemoteException
+ FullyQualifiedErrorId : NativeCommandError
INFO: Waiting for application startup.
INFO: Application startup complete.
INFO: Uvicorn running on http://0.0.0.0:3008 (Press CTRL+C to quit)
-6
View File
@@ -1,6 +0,0 @@
[2026-06-22T19:24:48.030482] Browser Use service starting on port 3008
[2026-06-22T19:27:19.231701] Browser Use service starting on port 3008
[2026-06-22T19:28:28.335765] Browser Use service starting on port 3008
[2026-06-22T19:29:05.796265] Browser Use service starting on port 3008
[2026-06-22T19:29:17.042807] Scraping WarriorForum...
[2026-06-22T19:29:19.075166] WarriorForum: 55 results
-4
View File
@@ -1,4 +0,0 @@
INFO: Started server process [24268]
INFO: Waiting for application startup.
INFO: Application startup complete.
INFO: Uvicorn running on http://0.0.0.0:3008 (Press CTRL+C to quit)
-5
View File
@@ -1,5 +0,0 @@
Browser Use service starting on port 3008
INFO: 127.0.0.1:65003 - "GET /health HTTP/1.1" 200 OK
Scraping WarriorForum...
WarriorForum: 55 results
INFO: 127.0.0.1:54587 - "POST /scrape/all HTTP/1.1" 200 OK
+4 -4
View File
@@ -126,10 +126,10 @@ separate 1:N child tables.
| Username | Password | Role | | Username | Password | Role |
|----------|----------|------| |----------|----------|------|
| `superadmin_demo` | `SuperAdmin@2026` | SUPER_ADMIN | | `superadmin_demo` | `[REDACTED]` | SUPER_ADMIN |
| `admin_demo` | `AdminAccess@2026` | ADMIN | | `admin_demo` | `[REDACTED]` | ADMIN |
| `sales_demo` | `SalesAccess@2026` | SALES_USER | | `sales_demo` | `[REDACTED]` | SALES_USER |
| `dev_demo` | `DevTesting@2026` | DEVELOPER | | `dev_demo` | `[REDACTED]` | DEVELOPER |
--- ---
@@ -0,0 +1,34 @@
CREATE TABLE IF NOT EXISTS facebook_accounts (
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
label VARCHAR(100) NOT NULL,
profile_path TEXT NOT NULL,
cookie_file TEXT NOT NULL,
is_active BOOLEAN NOT NULL DEFAULT TRUE,
last_scrape_at TIMESTAMPTZ,
last_success_at TIMESTAMPTZ,
last_error_at TIMESTAMPTZ,
last_error_message TEXT,
consecutive_failures INT NOT NULL DEFAULT 0,
flagged BOOLEAN NOT NULL DEFAULT FALSE,
flagged_at TIMESTAMPTZ,
flagged_reason TEXT,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
);
CREATE INDEX IF NOT EXISTS idx_fb_accounts_active ON facebook_accounts(is_active);
CREATE INDEX IF NOT EXISTS idx_fb_accounts_flagged ON facebook_accounts(flagged);
CREATE TABLE IF NOT EXISTS facebook_scrape_logs (
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
account_id UUID NOT NULL REFERENCES facebook_accounts(id) ON DELETE CASCADE,
started_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
completed_at TIMESTAMPTZ,
success BOOLEAN NOT NULL DEFAULT FALSE,
leads_found INT NOT NULL DEFAULT 0,
error_message TEXT,
detected_flag VARCHAR(50),
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
);
CREATE INDEX IF NOT EXISTS idx_fb_scrape_logs_account ON facebook_scrape_logs(account_id, created_at DESC);
+3
View File
@@ -34,5 +34,8 @@ BEGIN;
\echo '=== Running 009_settings.sql (Company Settings + User Preferences) ===' \echo '=== Running 009_settings.sql (Company Settings + User Preferences) ==='
\i 009_settings.sql \i 009_settings.sql
\echo '=== Running 010_facebook_accounts.sql (Facebook Account Tracking) ==='
\i 010_facebook_accounts.sql
\echo '=== Migration Complete ===' \echo '=== Migration Complete ==='
COMMIT; COMMIT;
+190 -50
View File
@@ -15,6 +15,7 @@ use tokio::sync::Mutex;
use tracing::{error, info, warn}; use tracing::{error, info, warn};
use uuid::Uuid; use uuid::Uuid;
use rand::Rng; use rand::Rng;
use chrono::Timelike;
use std::time::Duration; use std::time::Duration;
use std::time::{SystemTime, UNIX_EPOCH}; use std::time::{SystemTime, UNIX_EPOCH};
@@ -93,6 +94,25 @@ struct LeadStore {
max_size: usize, max_size: usize,
} }
#[derive(Debug, Deserialize)]
struct ScrapeResponse {
success: bool,
leads: Vec<ScrapeLead>,
flagged: bool,
flag_reason: Option<String>,
error: Option<String>,
}
#[derive(Debug, Deserialize, Clone)]
struct ScrapeLead {
title: String,
url: String,
author: String,
date: String,
content: String,
source: Option<String>,
}
impl LeadStore { impl LeadStore {
fn new(max_size: usize) -> Self { fn new(max_size: usize) -> Self {
Self { leads: Vec::new(), max_size } Self { leads: Vec::new(), max_size }
@@ -188,7 +208,7 @@ fn extract_claims(headers: &HeaderMap, state: &AppState) -> Result<Claims, (Stat
let claims = verify_jwt(token, &state.jwt_secret).ok_or_else(|| { let claims = verify_jwt(token, &state.jwt_secret).ok_or_else(|| {
(StatusCode::UNAUTHORIZED, "Unauthorized".to_string()) (StatusCode::UNAUTHORIZED, "Unauthorized".to_string())
})?; })?;
match claims.role.as_str() { match claims.role.to_lowercase().as_str() {
"sales" | "admin" | "super_admin" => Ok(claims), "sales" | "admin" | "super_admin" => Ok(claims),
_ => Err((StatusCode::FORBIDDEN, "Forbidden".to_string())), _ => Err((StatusCode::FORBIDDEN, "Forbidden".to_string())),
} }
@@ -266,30 +286,43 @@ async fn handle_chat(
let has_job = msg_words.contains(&"jobs") || msg_words.contains(&"job"); let has_job = msg_words.contains(&"jobs") || msg_words.contains(&"job");
if has_listing || (has_show && has_job) || (has_show && msg_lower.contains("links")) || msg_lower.contains("recent leads") { if has_listing || (has_show && has_job) || (has_show && msg_lower.contains("links")) || msg_lower.contains("recent leads") {
let now = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs(); let now = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs();
let service_url = "http://localhost:3008/scrape/all".to_string(); let base_url = "http://localhost:3008/scrape/facebook";
if let Ok(resp) = state.http_client let mut req_builder = state.http_client.post(base_url).timeout(Duration::from_secs(300));
.post(&service_url)
.timeout(Duration::from_secs(120)) if let Ok(Some((_, path))) = sqlx::query_as::<_, (uuid::Uuid, String)>(
.send() "SELECT id, profile_path FROM facebook_accounts \
.await WHERE is_active = TRUE AND flagged = FALSE \
ORDER BY last_scrape_at ASC NULLS FIRST LIMIT 1"
)
.fetch_optional(&state.db)
.await
{ {
if let Ok(leads_data) = resp.json::<Vec<serde_json::Value>>().await { req_builder = req_builder.query(&[("profile_path", &path), ("force", &"true".to_string())]);
info!("Scraped {} leads from {}", leads_data.len(), service_url); } else {
let mut store = state.leads.lock().await; warn!("No active Facebook account found for on-demand scrape");
for item in &leads_data { }
store.push(Lead {
title: truncate(item["title"].as_str().unwrap_or(""), 120), match req_builder.send().await {
url: item["url"].as_str().unwrap_or("").to_string(), Ok(resp) => {
source: item["source"].as_str().unwrap_or("unknown").to_string(), if let Ok(scrape_resp) = resp.json::<ScrapeResponse>().await {
found_at: now, info!("Scraped {} leads from Facebook", scrape_resp.leads.len());
author: truncate(item["author"].as_str().unwrap_or(""), 60), let mut store = state.leads.lock().await;
date: truncate(item["date"].as_str().unwrap_or(""), 30), for item in &scrape_resp.leads {
content: truncate(item["content"].as_str().unwrap_or(""), 300), store.push(Lead {
}); title: truncate(&item.title, 120),
url: item.url.clone(),
source: item.source.clone().unwrap_or_else(|| "facebook".to_string()),
found_at: now,
author: truncate(&item.author, 60),
date: truncate(&item.date, 30),
content: truncate(&item.content, 300),
});
}
} }
} }
} else { Err(e) => {
error!("Scraper service unreachable: {}", service_url); error!("Scraper request error: {} - URL: {}", e, base_url);
}
} }
let recent_leads = state.leads.lock().await.recent(604800, 20); let recent_leads = state.leads.lock().await.recent(604800, 20);
@@ -411,7 +444,7 @@ async fn main() {
info!("Connected to PostgreSQL"); info!("Connected to PostgreSQL");
let http_client = reqwest::Client::builder() let http_client = reqwest::Client::builder()
.timeout(Duration::from_secs(120)) .timeout(Duration::from_secs(300))
.build() .build()
.expect("Failed to build HTTP client"); .expect("Failed to build HTTP client");
@@ -441,7 +474,7 @@ async fn main() {
.route("/ai/jobs", get(handle_jobs)) .route("/ai/jobs", get(handle_jobs))
.route("/health", get(handle_health)) .route("/health", get(handle_health))
.layer(cors) .layer(cors)
.with_state(state); .with_state(state.clone());
let addr = format!("{}:{}", host, port); let addr = format!("{}:{}", host, port);
info!("CRM AI server listening on {}", addr); info!("CRM AI server listening on {}", addr);
@@ -451,10 +484,11 @@ async fn main() {
.expect("Failed to bind address"); .expect("Failed to bind address");
let bg_leads = lead_store.clone(); let bg_leads = lead_store.clone();
let bg_url = "http://localhost:3008/scrape/all".to_string(); let bg_db = state.db.clone();
let bg_url = "http://localhost:3008/scrape/facebook".to_string();
tokio::spawn(async move { tokio::spawn(async move {
let client = match reqwest::Client::builder() let client = match reqwest::Client::builder()
.timeout(Duration::from_secs(120)) .timeout(Duration::from_secs(300))
.build() .build()
{ {
Ok(c) => c, Ok(c) => c,
@@ -464,39 +498,145 @@ async fn main() {
} }
}; };
loop { loop {
// 10% random cycle skip — makes pattern non-periodic
if rand::thread_rng().gen_range(0..100) < 10 {
info!("Skipping this scrape cycle (random 10% skip)");
let jitter = rand::thread_rng().gen_range(16200..19800);
tokio::time::sleep(Duration::from_secs(jitter)).await;
continue;
}
// Skip night hours (23:00 06:00)
let hour = chrono::Local::now().hour();
if hour < 6 || hour >= 23 {
info!("Night hours ({}) — skipping scrape", hour);
let jitter = rand::thread_rng().gen_range(16200..19800);
tokio::time::sleep(Duration::from_secs(jitter)).await;
continue;
}
let now = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs(); let now = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs();
match client.post(&bg_url).send().await {
Ok(resp) => { // Pick next active un-flagged account
if resp.status().is_success() { let account = sqlx::query_as::<_, (uuid::Uuid, String)>(
match resp.json::<Vec<serde_json::Value>>().await { "SELECT id, profile_path FROM facebook_accounts \
Ok(data) => { WHERE is_active = TRUE AND flagged = FALSE \
let mut store = bg_leads.lock().await; ORDER BY last_scrape_at ASC NULLS FIRST LIMIT 1"
for item in &data { )
store.push(Lead { .fetch_optional(&bg_db)
title: truncate(item["title"].as_str().unwrap_or(""), 120), .await;
url: item["url"].as_str().unwrap_or("").to_string(),
source: item["source"].as_str().unwrap_or("unknown").to_string(), match account {
found_at: now, Ok(Some((account_id, profile_path))) => {
author: truncate(item["author"].as_str().unwrap_or(""), 60), match client.post(&bg_url).query(&[("profile_path", &profile_path)]).send().await {
date: truncate(item["date"].as_str().unwrap_or(""), 30), Ok(resp) => {
content: truncate(item["content"].as_str().unwrap_or(""), 300), if resp.status().is_success() {
}); match resp.json::<ScrapeResponse>().await {
Ok(data) => {
let leads_count = data.leads.len() as i32;
if data.flagged {
let _ = sqlx::query(
"UPDATE facebook_accounts SET flagged = TRUE, flagged_at = NOW(), \
flagged_reason = $2, last_error_at = NOW(), \
last_error_message = $3, consecutive_failures = consecutive_failures + 1 \
WHERE id = $1"
)
.bind(account_id)
.bind(&data.flag_reason)
.bind(&data.error)
.execute(&bg_db)
.await;
warn!("Facebook account {} flagged: {:?}", account_id, data.flag_reason);
let reason = data.flag_reason.as_deref().unwrap_or("unknown");
let _ = sqlx::query(
"INSERT INTO notifications (user_id, type, title, description, link) \
SELECT id, 'warning', 'Facebook Account Flagged', \
$1 || ' - ' || COALESCE($2, 'unknown reason'), \
NULL \
FROM users u JOIN user_roles ur ON ur.user_id = u.id \
JOIN roles r ON r.id = ur.role_id \
WHERE r.name IN ('ADMIN', 'SUPER_ADMIN')"
)
.bind(&account_id.to_string())
.bind(reason)
.execute(&bg_db)
.await;
} else if data.success {
let _ = sqlx::query(
"UPDATE facebook_accounts SET last_scrape_at = NOW(), \
last_success_at = NOW(), consecutive_failures = 0, \
updated_at = NOW() WHERE id = $1"
)
.bind(account_id)
.execute(&bg_db)
.await;
let mut store = bg_leads.lock().await;
for item in &data.leads {
store.push(Lead {
title: truncate(&item.title, 120),
url: item.url.clone(),
source: item.source.clone().unwrap_or_else(|| "facebook".to_string()),
found_at: now,
author: truncate(&item.author, 60),
date: truncate(&item.date, 30),
content: truncate(&item.content, 300),
});
}
info!("Scraped {} leads from Facebook account {}", leads_count, account_id);
} else {
// Increment failures; auto-flag if >= 3 consecutive
let _ = sqlx::query(
"UPDATE facebook_accounts SET last_error_at = NOW(), \
last_error_message = $2, consecutive_failures = consecutive_failures + 1, \
flagged = CASE WHEN consecutive_failures + 1 >= 3 THEN TRUE ELSE flagged END, \
flagged_reason = CASE WHEN consecutive_failures + 1 >= 3 THEN 'too_many_failures' ELSE flagged_reason END, \
flagged_at = CASE WHEN consecutive_failures + 1 >= 3 THEN NOW() ELSE flagged_at END, \
updated_at = NOW() WHERE id = $1"
)
.bind(account_id)
.bind(&data.error)
.execute(&bg_db)
.await;
warn!("Facebook scrape failed for account {}: {:?}", account_id, data.error);
}
let _ = sqlx::query(
"INSERT INTO facebook_scrape_logs \
(account_id, started_at, completed_at, success, leads_found, error_message, detected_flag) \
VALUES ($1, NOW() - interval '5 hours', NOW(), $2, $3, $4, $5)"
)
.bind(account_id)
.bind(data.success && !data.flagged)
.bind(leads_count)
.bind(&data.error)
.bind(&data.flag_reason)
.execute(&bg_db)
.await;
}
Err(e) => {
warn!("Failed to parse scraper JSON: {}", e);
}
} }
} } else {
Err(e) => { warn!("Scraper returned status: {}", resp.status());
warn!("Failed to parse scraper JSON: {}", e);
} }
} }
} else { Err(e) => {
warn!("Scraper returned status: {}", resp.status()); warn!("Scraper request failed: {}", e);
}
} }
} }
Ok(None) => {
info!("No active Facebook accounts available — skipping scrape cycle");
}
Err(e) => { Err(e) => {
warn!("Scraper request failed: {}", e); warn!("Failed to query Facebook accounts: {}", e);
} }
} }
let delay = rand::thread_rng().gen_range(120..300);
tokio::time::sleep(Duration::from_secs(delay)).await; let jitter = rand::thread_rng().gen_range(16200..19800); // 4.5h 5.5h
tokio::time::sleep(Duration::from_secs(jitter)).await;
} }
}); });
+9 -4
View File
@@ -48,7 +48,7 @@ function stageToStatus(name: string): string {
} }
} }
async function fetchLeadsInRange(start: Date, end: Date) { async function fetchLeadsInRange(start: Date, end: Date, userId?: string, isAdmin?: boolean) {
const result = await query( const result = await query(
`SELECT l.id, l.created_at, l.company_name, l.contact_name, l.email, l.phone, `SELECT l.id, l.created_at, l.company_name, l.contact_name, l.email, l.phone,
l.notes, l.assigned_to, l.score, l.notes, l.assigned_to, l.score,
@@ -59,8 +59,11 @@ async function fetchLeadsInRange(start: Date, end: Date) {
LEFT JOIN users u ON u.id = l.assigned_to LEFT JOIN users u ON u.id = l.assigned_to
WHERE l.deleted_at IS NULL WHERE l.deleted_at IS NULL
AND l.created_at >= $1 AND l.created_at <= $2 AND l.created_at >= $1 AND l.created_at <= $2
${isAdmin ? "" : "AND l.assigned_to = $3"}
ORDER BY l.created_at DESC`, ORDER BY l.created_at DESC`,
[start.toISOString(), end.toISOString()] isAdmin
? [start.toISOString(), end.toISOString()]
: [start.toISOString(), end.toISOString(), userId]
) )
return result.rows.map((r: any) => ({ return result.rows.map((r: any) => ({
...r, ...r,
@@ -118,6 +121,8 @@ export async function GET(request: NextRequest) {
const user = await getSessionUser() const user = await getSessionUser()
if (!user) return NextResponse.json({ error: "Unauthorized" }, { status: 401 }) if (!user) return NextResponse.json({ error: "Unauthorized" }, { status: 401 })
const isAdmin = user.role === "admin" || user.role === "super_admin"
const { searchParams } = new URL(request.url) const { searchParams } = new URL(request.url)
const period = searchParams.get("period") || "6months" const period = searchParams.get("period") || "6months"
const yearParam = searchParams.get("year") const yearParam = searchParams.get("year")
@@ -134,8 +139,8 @@ export async function GET(request: NextRequest) {
} }
const [currentLeads, prevLeads] = await Promise.all([ const [currentLeads, prevLeads] = await Promise.all([
fetchLeadsInRange(start, end), fetchLeadsInRange(start, end, user.id, isAdmin),
fetchLeadsInRange(prevRange.start, prevRange.end), fetchLeadsInRange(prevRange.start, prevRange.end, user.id, isAdmin),
]) ])
const currentCounts = countStatuses(currentLeads) const currentCounts = countStatuses(currentLeads)
+19
View File
@@ -3,12 +3,28 @@ import { getSessionUser } from "@/lib/auth"
import { query } from "@/lib/db" import { query } from "@/lib/db"
import { avatarSvgUrl } from "@/lib/avatar" import { avatarSvgUrl } from "@/lib/avatar"
async function checkLeadAccess(leadId: string, userId: string): Promise<boolean> {
const result = await query(
`SELECT 1 FROM leads WHERE id = $1 AND deleted_at IS NULL
AND (assigned_to = $2 OR EXISTS (
SELECT 1 FROM user_roles ur JOIN roles r ON r.id = ur.role_id
WHERE ur.user_id = $2 AND r.name IN ('ADMIN', 'SUPER_ADMIN')
))`,
[leadId, userId]
)
return result.rows.length > 0
}
export async function POST(request: NextRequest, { params }: { params: Promise<{ id: string }> }) { export async function POST(request: NextRequest, { params }: { params: Promise<{ id: string }> }) {
try { try {
const user = await getSessionUser() const user = await getSessionUser()
if (!user) return NextResponse.json({ error: "Unauthorized" }, { status: 401 }) if (!user) return NextResponse.json({ error: "Unauthorized" }, { status: 401 })
const { id } = await params const { id } = await params
if (!await checkLeadAccess(id, user.id)) {
return NextResponse.json({ error: "Lead not found" }, { status: 404 })
}
const { content } = await request.json() const { content } = await request.json()
if (!content?.trim()) { if (!content?.trim()) {
return NextResponse.json({ error: "Content is required" }, { status: 400 }) return NextResponse.json({ error: "Content is required" }, { status: 400 })
@@ -33,6 +49,9 @@ export async function GET(request: NextRequest, { params }: { params: Promise<{
if (!user) return NextResponse.json({ error: "Unauthorized" }, { status: 401 }) if (!user) return NextResponse.json({ error: "Unauthorized" }, { status: 401 })
const { id } = await params const { id } = await params
if (!await checkLeadAccess(id, user.id)) {
return NextResponse.json({ error: "Lead not found" }, { status: 404 })
}
const { searchParams } = new URL(request.url) const { searchParams } = new URL(request.url)
const limit = parseInt(searchParams.get("limit") || "50", 10) const limit = parseInt(searchParams.get("limit") || "50", 10)
const offset = parseInt(searchParams.get("offset") || "0", 10) const offset = parseInt(searchParams.get("offset") || "0", 10)
+5
View File
@@ -111,6 +111,11 @@ export async function PATCH(request: NextRequest, { params }: { params: Promise<
if (body.description !== undefined) { fields.push(`notes = $${idx++}`); values.push(body.description) } if (body.description !== undefined) { fields.push(`notes = $${idx++}`); values.push(body.description) }
if (body.source !== undefined) { fields.push(`source_id = $${idx++}`); values.push(body.source) } if (body.source !== undefined) { fields.push(`source_id = $${idx++}`); values.push(body.source) }
if (body.assignedUserId !== undefined) { if (body.assignedUserId !== undefined) {
const isAdmin = user.role === "admin" || user.role === "super_admin"
if (!isAdmin) {
// non-admin cannot reassign
return NextResponse.json({ error: "Only admins can reassign leads" }, { status: 403 })
}
fields.push(`assigned_to = $${idx++}`) fields.push(`assigned_to = $${idx++}`)
values.push(body.assignedUserId === "none" ? null : body.assignedUserId) values.push(body.assignedUserId === "none" ? null : body.assignedUserId)
} }
+10 -1
View File
@@ -123,6 +123,15 @@ export async function POST(request: NextRequest) {
if (!user) return NextResponse.json({ error: "Unauthorized" }, { status: 401 }) if (!user) return NextResponse.json({ error: "Unauthorized" }, { status: 401 })
const body = await request.json() const body = await request.json()
const isAdmin = user.role === "admin" || user.role === "super_admin"
// Non-admin users can only assign leads to themselves; admin/super_admin can assign to anyone
let assignedUserId = body.assignedUserId
if (!isAdmin) {
assignedUserId = user.id
} else if (assignedUserId === "none" || !assignedUserId) {
assignedUserId = null
}
const stageResult = await query( const stageResult = await query(
"SELECT id FROM lead_stages WHERE name = $1", "SELECT id FROM lead_stages WHERE name = $1",
@@ -142,7 +151,7 @@ export async function POST(request: NextRequest) {
body.description || null, body.description || null,
body.source || null, body.source || null,
stageId, stageId,
body.assignedUserId === "none" ? null : body.assignedUserId, assignedUserId,
] ]
) )
+2 -1
View File
@@ -47,7 +47,8 @@ export async function POST(request: NextRequest) {
if (!user) return NextResponse.json({ error: "Unauthorized" }, { status: 401 }) if (!user) return NextResponse.json({ error: "Unauthorized" }, { status: 401 })
const { type, title, description, link, userId } = await request.json() const { type, title, description, link, userId } = await request.json()
const targetUserId = userId || user.id const isAdmin = user.role === "admin" || user.role === "super_admin"
const targetUserId = (userId && isAdmin) ? userId : user.id
const result = await query( const result = await query(
`INSERT INTO notifications (user_id, type, title, description, link) `INSERT INTO notifications (user_id, type, title, description, link)
@@ -0,0 +1,61 @@
import { NextRequest, NextResponse } from "next/server"
import { getSessionUser } from "@/lib/auth"
import { query } from "@/lib/db"
export async function PATCH(request: NextRequest, { params }: { params: Promise<{ id: string }> }) {
try {
const user = await getSessionUser()
if (!user) return NextResponse.json({ error: "Unauthorized" }, { status: 401 })
if (user.role !== "admin" && user.role !== "super_admin") {
return NextResponse.json({ error: "Forbidden" }, { status: 403 })
}
const { id } = await params
const body = await request.json()
const fields: string[] = []
const values: any[] = []
let idx = 1
if (body.isActive !== undefined) {
fields.push(`is_active = $${idx++}`)
values.push(body.isActive)
}
if (body.label !== undefined) {
fields.push(`label = $${idx++}`)
values.push(body.label.trim())
}
if (body.profilePath !== undefined) {
fields.push(`profile_path = $${idx++}, cookie_file = $${idx}`)
values.push(body.profilePath.trim())
values.push(`${body.profilePath.replace(/\\+$/, '')}\\cookies.sqlite`)
idx++
}
if (body.unflag === true) {
fields.push(`flagged = FALSE, flagged_at = NULL, flagged_reason = NULL, consecutive_failures = 0`)
}
if (fields.length === 0) {
return NextResponse.json({ error: "No fields to update" }, { status: 400 })
}
fields.push(`updated_at = NOW()`)
values.push(id)
await query(
`UPDATE facebook_accounts SET ${fields.join(", ")} WHERE id = $${idx}`,
values
)
const updated = await query(
`SELECT id, label, profile_path, is_active, flagged, flagged_reason,
consecutive_failures, updated_at
FROM facebook_accounts WHERE id = $1`,
[id]
)
return NextResponse.json(updated.rows[0] || { success: true })
} catch (error) {
console.error("Facebook accounts PATCH error:", error)
return NextResponse.json({ error: "Failed to update Facebook account" }, { status: 500 })
}
}
@@ -0,0 +1,66 @@
import { NextRequest, NextResponse } from "next/server"
import { getSessionUser } from "@/lib/auth"
import { query } from "@/lib/db"
export async function GET() {
try {
const user = await getSessionUser()
if (!user) return NextResponse.json({ error: "Unauthorized" }, { status: 401 })
if (user.role !== "admin" && user.role !== "super_admin") {
return NextResponse.json({ error: "Forbidden" }, { status: 403 })
}
const accounts = await query(
`SELECT fa.id, fa.label, fa.profile_path, fa.is_active,
fa.last_scrape_at, fa.last_success_at, fa.last_error_at,
fa.last_error_message, fa.consecutive_failures,
fa.flagged, fa.flagged_at, fa.flagged_reason,
fa.created_at, fa.updated_at,
COALESCE(sl.leads_found, 0) AS last_leads_found,
sl.success AS last_success,
sl.detected_flag AS last_detected_flag
FROM facebook_accounts fa
LEFT JOIN LATERAL (
SELECT leads_found, success, detected_flag
FROM facebook_scrape_logs
WHERE account_id = fa.id
ORDER BY created_at DESC
LIMIT 1
) sl ON TRUE
ORDER BY fa.created_at ASC`
)
return NextResponse.json(accounts.rows)
} catch (error) {
console.error("Facebook accounts GET error:", error)
return NextResponse.json({ error: "Failed to load Facebook accounts" }, { status: 500 })
}
}
export async function POST(request: NextRequest) {
try {
const user = await getSessionUser()
if (!user) return NextResponse.json({ error: "Unauthorized" }, { status: 401 })
if (user.role !== "admin" && user.role !== "super_admin") {
return NextResponse.json({ error: "Forbidden" }, { status: 403 })
}
const { label, profilePath } = await request.json()
if (!label?.trim() || !profilePath?.trim()) {
return NextResponse.json({ error: "Label and profile path are required" }, { status: 400 })
}
const cookieFile = `${profilePath.replace(/\\+$/, '')}\\cookies.sqlite`
const result = await query(
`INSERT INTO facebook_accounts (label, profile_path, cookie_file)
VALUES ($1, $2, $3)
RETURNING id, label, profile_path, is_active, created_at`,
[label.trim(), profilePath.trim(), cookieFile]
)
return NextResponse.json(result.rows[0], { status: 201 })
} catch (error) {
console.error("Facebook accounts POST error:", error)
return NextResponse.json({ error: "Failed to create Facebook account" }, { status: 500 })
}
}
@@ -0,0 +1,41 @@
import { NextRequest, NextResponse } from "next/server"
import { getSessionUser } from "@/lib/auth"
import { query } from "@/lib/db"
export async function GET(request: NextRequest) {
try {
const user = await getSessionUser()
if (!user) return NextResponse.json({ error: "Unauthorized" }, { status: 401 })
if (user.role !== "admin" && user.role !== "super_admin") {
return NextResponse.json({ error: "Forbidden" }, { status: 403 })
}
const { searchParams } = new URL(request.url)
const accountId = searchParams.get("accountId")
const limit = parseInt(searchParams.get("limit") || "50", 10)
const offset = parseInt(searchParams.get("offset") || "0", 10)
let sql = `SELECT sl.id, sl.account_id, fa.label AS account_label,
sl.started_at, sl.completed_at, sl.success,
sl.leads_found, sl.error_message, sl.detected_flag,
sl.created_at
FROM facebook_scrape_logs sl
JOIN facebook_accounts fa ON fa.id = sl.account_id`
const params: any[] = []
let paramIdx = 1
if (accountId) {
sql += ` WHERE sl.account_id = $${paramIdx++}`
params.push(accountId)
}
sql += ` ORDER BY sl.created_at DESC LIMIT $${paramIdx++} OFFSET $${paramIdx++}`
params.push(limit, offset)
const result = await query(sql, params)
return NextResponse.json(result.rows)
} catch (error) {
console.error("Facebook scrape logs GET error:", error)
return NextResponse.json({ error: "Failed to load scrape logs" }, { status: 500 })
}
}
+3
View File
@@ -8,6 +8,9 @@ let prevTime = Date.now()
export async function GET() { export async function GET() {
const sessionUser = await getSessionUser() const sessionUser = await getSessionUser()
if (!sessionUser) return NextResponse.json({ error: "Unauthorized" }, { status: 401 }) if (!sessionUser) return NextResponse.json({ error: "Unauthorized" }, { status: 401 })
if (sessionUser.role !== "admin" && sessionUser.role !== "super_admin") {
return NextResponse.json({ error: "Forbidden" }, { status: 403 })
}
const now = Date.now() const now = Date.now()
const elapsed = now - prevTime const elapsed = now - prevTime
+15
View File
@@ -2,6 +2,9 @@ import { NextRequest, NextResponse } from "next/server"
import { getSessionUser } from "@/lib/auth" import { getSessionUser } from "@/lib/auth"
import { query } from "@/lib/db" import { query } from "@/lib/db"
const ALLOWED_PREFIXES = ["data:image/png;base64,", "data:image/jpeg;base64,", "data:image/gif;base64,"]
const MAX_AVATAR_BYTES = 2 * 1024 * 1024 // 2MB
export async function POST(request: NextRequest) { export async function POST(request: NextRequest) {
try { try {
const user = await getSessionUser() const user = await getSessionUser()
@@ -12,6 +15,18 @@ export async function POST(request: NextRequest) {
return NextResponse.json({ error: "Invalid avatar data" }, { status: 400 }) return NextResponse.json({ error: "Invalid avatar data" }, { status: 400 })
} }
const allowed = ALLOWED_PREFIXES.some((p) => avatar.startsWith(p))
if (!allowed) {
return NextResponse.json({ error: "Avatar must be a PNG, JPEG, or GIF data URL" }, { status: 400 })
}
// Approximate decoded size: base64 is ~4/3 of original
const base64Data = avatar.split(",")[1] || ""
const estimatedBytes = Math.round(base64Data.length * 0.75)
if (estimatedBytes > MAX_AVATAR_BYTES) {
return NextResponse.json({ error: "Avatar exceeds 2MB size limit" }, { status: 400 })
}
await query( await query(
`UPDATE users SET avatar_url = $1 WHERE id = $2`, `UPDATE users SET avatar_url = $1 WHERE id = $2`,
[avatar, user.id], [avatar, user.id],
+3
View File
@@ -7,6 +7,9 @@ export async function GET(request: NextRequest) {
try { try {
const sessionUser = await getSessionUser() const sessionUser = await getSessionUser()
if (!sessionUser) return NextResponse.json({ error: "Unauthorized" }, { status: 401 }) if (!sessionUser) return NextResponse.json({ error: "Unauthorized" }, { status: 401 })
if (sessionUser.role !== "admin" && sessionUser.role !== "super_admin") {
return NextResponse.json({ error: "Forbidden" }, { status: 403 })
}
const { searchParams } = new URL(request.url) const { searchParams } = new URL(request.url)
const limit = parseInt(searchParams.get("limit") || "50", 10) const limit = parseInt(searchParams.get("limit") || "50", 10)
+3
View File
@@ -7,6 +7,9 @@ export async function GET(request: NextRequest) {
try { try {
const currentUser = await getSessionUser() const currentUser = await getSessionUser()
if (!currentUser) return NextResponse.json({ error: "Unauthorized" }, { status: 401 }) if (!currentUser) return NextResponse.json({ error: "Unauthorized" }, { status: 401 })
if (currentUser.role !== "admin" && currentUser.role !== "super_admin") {
return NextResponse.json({ error: "Forbidden" }, { status: 403 })
}
const q = request.nextUrl.searchParams.get("q") || "" const q = request.nextUrl.searchParams.get("q") || ""
+39
View File
@@ -18,10 +18,12 @@ import {
PanelLeftClose, PanelLeftClose,
MessageSquare, MessageSquare,
Bot, Bot,
Facebook,
} from "lucide-react" } from "lucide-react"
import { COMPANY_NAME } from "@/lib/constants" import { COMPANY_NAME } from "@/lib/constants"
import { useUser } from "@/providers/user-provider" import { useUser } from "@/providers/user-provider"
import { useNotifications } from "@/providers/notification-provider" import { useNotifications } from "@/providers/notification-provider"
import { FacebookAccountsDialog } from "@/components/settings/facebook-accounts-dialog"
const navItems = [ const navItems = [
{ href: "/dashboard", label: "Dashboard", icon: LayoutDashboard }, { href: "/dashboard", label: "Dashboard", icon: LayoutDashboard },
@@ -43,7 +45,9 @@ export function Sidebar({ collapsed, onToggle, mobileOpen, onMobileClose }: Side
const pathname = usePathname() const pathname = usePathname()
const { user } = useUser() const { user } = useUser()
const { unreadChatCount } = useNotifications() const { unreadChatCount } = useNotifications()
const [fbDialogOpen, setFbDialogOpen] = useState(false)
if (!user) return null if (!user) return null
const isAdmin = user.role === "admin" || user.role === "super_admin"
const initials = user.name.split(" ").map((n) => n[0]).join("") const initials = user.name.split(" ").map((n) => n[0]).join("")
const sidebarContent = ( const sidebarContent = (
@@ -143,6 +147,39 @@ export function Sidebar({ collapsed, onToggle, mobileOpen, onMobileClose }: Side
</Link> </Link>
) )
})} })}
{isAdmin && collapsed && (
<TooltipProvider delayDuration={0}>
<Tooltip>
<TooltipTrigger asChild>
<button
onClick={() => setFbDialogOpen(true)}
className="relative flex h-10 w-10 items-center justify-center rounded-lg transition-colors text-sidebar-foreground/60 hover:bg-sidebar-accent hover:text-sidebar-accent-foreground"
>
<Facebook className="h-5 w-5" />
</button>
</TooltipTrigger>
<TooltipContent side="right" className="ml-2">
Facebook Accounts
</TooltipContent>
</Tooltip>
</TooltipProvider>
)}
{isAdmin && !collapsed && (
<button
onClick={() => setFbDialogOpen(true)}
className="flex w-full items-center gap-3 rounded-lg px-3 py-2.5 text-sm font-medium transition-colors text-sidebar-foreground/60 hover:bg-sidebar-accent hover:text-sidebar-accent-foreground"
>
<Facebook className="h-5 w-5 shrink-0" />
<motion.span
initial={{ opacity: 0 }}
animate={{ opacity: 1 }}
exit={{ opacity: 0 }}
className="truncate"
>
Facebook Accounts
</motion.span>
</button>
)}
</nav> </nav>
{/* Collapse toggle at bottom (only in collapsed mode) */} {/* Collapse toggle at bottom (only in collapsed mode) */}
@@ -189,6 +226,8 @@ export function Sidebar({ collapsed, onToggle, mobileOpen, onMobileClose }: Side
{sidebarContent} {sidebarContent}
</aside> </aside>
<FacebookAccountsDialog open={fbDialogOpen} onOpenChange={setFbDialogOpen} />
{/* Mobile sidebar overlay */} {/* Mobile sidebar overlay */}
<AnimatePresence> <AnimatePresence>
{mobileOpen && ( {mobileOpen && (
@@ -0,0 +1,102 @@
"use client"
import { useEffect, useState } from "react"
import { Dialog, DialogContent, DialogHeader, DialogTitle, DialogDescription } from "@/components/ui/dialog"
import { Badge } from "@/components/ui/badge"
import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table"
import { ShieldAlert, RefreshCw } from "lucide-react"
import { Button } from "@/components/ui/button"
interface Account {
id: string
label: string
is_active: boolean
last_scrape_at: string | null
last_success_at: string | null
last_error_at: string | null
consecutive_failures: number
flagged: boolean
flagged_reason: string | null
last_leads_found: number
last_success: boolean | null
}
export function FacebookAccountsDialog({ open, onOpenChange }: { open: boolean; onOpenChange: (v: boolean) => void }) {
const [accounts, setAccounts] = useState<Account[]>([])
const [loading, setLoading] = useState(false)
useEffect(() => {
if (open) load()
}, [open])
async function load() {
setLoading(true)
try {
const res = await fetch("/api/settings/facebook/accounts")
if (res.ok) {
setAccounts(await res.json())
}
} catch {}
setLoading(false)
}
return (
<Dialog open={open} onOpenChange={onOpenChange}>
<DialogContent className="max-w-2xl">
<DialogHeader className="flex flex-row items-center justify-between">
<div>
<DialogTitle>Facebook Accounts</DialogTitle>
<DialogDescription>
Status of Facebook profiles used for lead scraping
</DialogDescription>
</div>
<Button variant="ghost" size="icon" onClick={load} disabled={loading}>
<RefreshCw className={`h-4 w-4 ${loading ? "animate-spin" : ""}`} />
</Button>
</DialogHeader>
{loading ? (
<div className="text-center py-8 text-muted-foreground">Loading...</div>
) : accounts.length === 0 ? (
<div className="text-center py-8 text-muted-foreground">No accounts configured</div>
) : (
<Table>
<TableHeader>
<TableRow>
<TableHead>Label</TableHead>
<TableHead>Status</TableHead>
<TableHead>Last Scrape</TableHead>
<TableHead>Leads</TableHead>
<TableHead>Failures</TableHead>
</TableRow>
</TableHeader>
<TableBody>
{accounts.map((a) => (
<TableRow key={a.id}>
<TableCell className="font-medium">{a.label}</TableCell>
<TableCell>
{a.flagged ? (
<Badge variant="destructive" className="gap-1">
<ShieldAlert className="h-3 w-3" />
{a.flagged_reason || "Flagged"}
</Badge>
) : a.is_active ? (
<Badge variant="default" className="bg-green-600">Active</Badge>
) : (
<Badge variant="secondary">Disabled</Badge>
)}
</TableCell>
<TableCell className="text-sm text-muted-foreground">
{a.last_scrape_at ? new Date(a.last_scrape_at).toLocaleString() : "Never"}
</TableCell>
<TableCell>{a.last_leads_found}</TableCell>
<TableCell>{a.consecutive_failures}</TableCell>
</TableRow>
))}
</TableBody>
</Table>
)}
</DialogContent>
</Dialog>
)
}