From 8d5bad87bc1b972f1bad27f7bcc5178462416e12 Mon Sep 17 00:00:00 2001 From: caitz13 Date: Tue, 23 Jun 2026 11:21:24 +0200 Subject: [PATCH] Push all current state --- frontend_old/.gitignore | 15 + frontend_old/browser-use-service/err.txt | Bin 0 -> 1092 bytes frontend_old/browser-use-service/main.py | 410 ++++++++ frontend_old/browser-use-service/out.txt | 0 frontend_old/browser-use-service/run_log.txt | 7 + .../browser-use-service/scrape_log.txt | 6 + frontend_old/browser-use-service/stderr.txt | 4 + frontend_old/browser-use-service/stdout.txt | 5 + .../database/migrations/001_schema.sql | 87 +- frontend_old/database/migrations/003_chat.sql | 2 + .../database/migrations/009_settings.sql | 22 + .../migrations/010_chat_notifications.sql | 4 + frontend_old/database/migrations/run_all.sql | 16 +- frontend_old/fix.md | 87 +- frontend_old/next.config.ts | 2 +- frontend_old/package.json | 8 +- frontend_old/rust-ai/Cargo.lock | 619 +++---------- frontend_old/rust-ai/Cargo.toml | 6 +- frontend_old/rust-ai/src/instructions.rs | 85 -- frontend_old/rust-ai/src/main.rs | 444 +++++---- frontend_old/scraper-service/index.js | 100 ++ .../scraper-service/package-lock.json | 872 ++++++++++++++++++ frontend_old/scraper-service/package.json | 13 + .../src/app/(dashboard)/chats/page.tsx | 84 +- .../src/app/(dashboard)/dashboard/page.tsx | 2 +- frontend_old/src/app/(dashboard)/layout.tsx | 5 +- .../src/app/(dashboard)/leads/[id]/page.tsx | 31 +- .../src/app/(dashboard)/leads/new/page.tsx | 60 +- .../src/app/(dashboard)/profile/page.tsx | 1 + .../src/app/(dashboard)/users/page.tsx | 2 +- frontend_old/src/app/api/ai/chat/route.ts | 6 +- frontend_old/src/app/api/ai/jobs/route.ts | 1 + frontend_old/src/app/api/auth/login/route.ts | 2 +- .../api/conversations/[id]/messages/route.ts | 75 +- .../app/api/conversations/[id]/read/route.ts | 6 + .../src/app/api/conversations/route.ts | 8 +- frontend_old/src/app/api/dashboard/route.ts | 3 +- .../src/app/api/leads/[id]/notes/route.ts | 11 +- frontend_old/src/app/api/leads/[id]/route.ts | 91 +- frontend_old/src/app/api/leads/route.ts | 78 +- .../src/app/api/notifications/route.ts | 4 +- .../src/app/api/settings/company/route.ts | 68 ++ .../src/app/api/settings/preferences/route.ts | 60 ++ .../src/app/api/system/monitor/route.ts | 4 + frontend_old/src/app/api/users/[id]/route.ts | 7 + frontend_old/src/app/api/users/route.ts | 26 +- .../src/app/api/users/search/route.ts | 3 +- frontend_old/src/app/globals.css | 266 +++++- frontend_old/src/app/login/page.tsx | 2 +- frontend_old/src/components/ai/ai-chat.tsx | 53 +- .../dashboard/leads-per-month-chart.tsx | 2 +- .../src/components/layout/system-monitor.tsx | 2 +- .../src/components/leads/lead-form-dialog.tsx | 60 +- .../src/components/notes/note-form.tsx | 63 +- .../settings/company-settings-form.tsx | 82 +- .../settings/notification-settings.tsx | 15 +- .../components/settings/theme-settings.tsx | 140 ++- .../settings/user-preferences-form.tsx | 69 +- .../src/components/shared/error-boundary.tsx | 50 + .../src/components/users/user-form-dialog.tsx | 2 +- .../src/components/users/users-table.tsx | 1 + frontend_old/src/lib/ai.ts | 28 +- frontend_old/src/lib/auth.ts | 18 +- frontend_old/src/lib/avatar.ts | 26 + frontend_old/src/lib/db.ts | 7 +- frontend_old/src/middleware.ts | 19 +- .../src/providers/notification-provider.tsx | 12 +- frontend_old/src/providers/theme-provider.tsx | 9 +- frontend_old/src/providers/user-provider.tsx | 3 +- 69 files changed, 3282 insertions(+), 1099 deletions(-) create mode 100644 frontend_old/browser-use-service/err.txt create mode 100644 frontend_old/browser-use-service/main.py create mode 100644 frontend_old/browser-use-service/out.txt create mode 100644 frontend_old/browser-use-service/run_log.txt create mode 100644 frontend_old/browser-use-service/scrape_log.txt create mode 100644 frontend_old/browser-use-service/stderr.txt create mode 100644 frontend_old/browser-use-service/stdout.txt create mode 100644 frontend_old/database/migrations/009_settings.sql create mode 100644 frontend_old/database/migrations/010_chat_notifications.sql delete mode 100644 frontend_old/rust-ai/src/instructions.rs create mode 100644 frontend_old/scraper-service/index.js create mode 100644 frontend_old/scraper-service/package-lock.json create mode 100644 frontend_old/scraper-service/package.json create mode 100644 frontend_old/src/app/api/settings/company/route.ts create mode 100644 frontend_old/src/app/api/settings/preferences/route.ts create mode 100644 frontend_old/src/components/shared/error-boundary.tsx create mode 100644 frontend_old/src/lib/avatar.ts diff --git a/frontend_old/.gitignore b/frontend_old/.gitignore index c1b3cd5..91ec548 100644 --- a/frontend_old/.gitignore +++ b/frontend_old/.gitignore @@ -34,9 +34,24 @@ yarn-error.log* # env files (can opt-in for committing if needed) .env* +# runtime data +data/ +data/ai/ +data/ai/jobs.jsonl + +# logs +*.log +*.out +error-log + # vercel .vercel +# python +browser-use-service/venv/ +browser-use-service/__pycache__/ +browser-use-service/.env + # typescript *.tsbuildinfo next-env.d.ts diff --git a/frontend_old/browser-use-service/err.txt b/frontend_old/browser-use-service/err.txt new file mode 100644 index 0000000000000000000000000000000000000000..a1f2d3d461a83da4457c6c43854eceb1c767d8a7 GIT binary patch literal 1092 zcmd6m%TB^z5QWd$#CK?1K@`PHBI%A0Nl3inZ2`t0R;iIv3!-d%boHC54K9dr;lfP% zU#Dlze`e<6J=IJT4Kz}sKp|gChidA`*8zL8Q+#^L7$e=XA2Vl~`WPr>JyM_D7z@W7 zSbfgM?$OqPuC)jI3$+)X-1uKF_#69*o_$sM+D~C;Z|G`BtU&8-b7pV)A7+e-yqIPU z*jirh(I`W`78>8NE_}Yvmfu6}CS){q^tyQdscgee}=;a3e*494V+2C#h NR-h;FO_{R4@e|+6x6}Xt literal 0 HcmV?d00001 diff --git a/frontend_old/browser-use-service/main.py b/frontend_old/browser-use-service/main.py new file mode 100644 index 0000000..726a364 --- /dev/null +++ b/frontend_old/browser-use-service/main.py @@ -0,0 +1,410 @@ +import os, json, asyncio, re, shutil, sqlite3, traceback, urllib.parse, random, time, logging +from datetime import datetime, timedelta +from bs4 import BeautifulSoup +import httpx +from fastapi import FastAPI +from pydantic import BaseModel +import uvicorn +from playwright.async_api import async_playwright + +logging.basicConfig(level=logging.INFO, format='%(asctime)s [%(levelname)s] %(message)s') +logger = logging.getLogger(__name__) + +app = FastAPI() +PORT = int(os.getenv("PORT", "3008")) + +OLLAMA_URL = os.getenv("OLLAMA_URL", "http://localhost:11434") +CLASSIFY_MODEL = os.getenv("CLASSIFY_MODEL", "dolphin-llama3:8b") + +FX_PROFILE = os.getenv('FX_PROFILE', '') +FX_COOKIE_DB = os.path.join(FX_PROFILE, 'cookies.sqlite') if FX_PROFILE else '' + +HEADERS = { + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36', + 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', +} + +last_scrape_time: datetime | None = None + +STRICT_KEYWORDS = [ + "website", "web design", "web develop", "web dev", + "build my website", "build a website", "create a website", + "need web", "looking for web", "new website", + "landing page", "wordpress", + "need a website", "my website", "business website", + "need a designer", "help with my website", + "redesign", "update my website", +] + +BROAD_KEYWORDS = STRICT_KEYWORDS + [ + "looking for", "need a", "need an", "looking to", + "help me build", "create my", "build me", + "design my", "make me a", "would like", + "need someone", "hire a", "looking to hire", + "want someone", "need help with", +] + +def kw_match_strict(text: str) -> bool: + t = text.lower() + return any(kw in t for kw in STRICT_KEYWORDS) + +def kw_match(text: str) -> bool: + t = text.lower() + return any(kw in t for kw in BROAD_KEYWORDS) + +FB_SEARCHES = [ + "looking for web developer", + "need a website designed", + "need website built South Africa", + "need someone to build my website", + "need web designer", + "looking for someone to create website", + "need ecommerce website built", + "want to hire web developer", + "website quote please", + "need wordpress website", + "I need a website for my business", + "need a site for my business", +] + +async def get_fb_cookies(): + if not FX_COOKIE_DB or not os.path.exists(FX_COOKIE_DB): + logger.warning("FX_COOKIE_DB not found or FX_PROFILE not set") + return [] + tmp = os.path.join(os.path.dirname(FX_COOKIE_DB), f'cookies_tmp_{os.getpid()}.sqlite') + for attempt in range(3): + try: + shutil.copy2(FX_COOKIE_DB, tmp) + break + except PermissionError: + if attempt < 2: + await asyncio.sleep(0.5) + else: + logger.error("Failed to copy cookie DB after 3 attempts") + return [] + try: + conn = sqlite3.connect(tmp) + c = conn.cursor() + c.execute("SELECT name, value, host, path FROM moz_cookies WHERE host LIKE '%facebook.com'") + rows = c.fetchall() + conn.close() + try: + os.remove(tmp) + except Exception: + pass + return [{ + "name": name, "value": value, + "domain": host if host.startswith('.') else f'.{host}', + "path": path, + "httpOnly": True, "secure": True, "sameSite": "Lax", + } for name, value, host, path in rows] + except Exception as e: + logger.error("Cookie DB read error: %s", e) + try: + os.remove(tmp) + except Exception: + pass + return [] + +WEEKDAY_ORDER = ['monday', 'tuesday', 'wednesday', 'thursday', 'friday', 'saturday', 'sunday'] + +def _parse_fb_date(block: list[str]) -> str: + for l in block: + l = l.strip() + m = re.match(r'(\d+)\s*(h|hr|hrs|hour|hours)\s*ago', l) + if m: + hours = int(m.group(1)) + d = datetime.now() - timedelta(hours=hours) + return d.strftime('%Y-%m-%d') + if l.lower() == 'yesterday': + d = datetime.now() - timedelta(days=1) + return d.strftime('%Y-%m-%d') + low = l.lower() + if low in WEEKDAY_ORDER: + day_idx = WEEKDAY_ORDER.index(low) + today_idx = datetime.now().weekday() + days_ago = (today_idx - day_idx) % 7 + d = datetime.now() - timedelta(days=days_ago) + return d.strftime('%Y-%m-%d') + for fmt in ['%Y-%m-%d', '%d %B %Y', '%B %d %Y', '%d %b %Y', '%b %d %Y', '%d %b', '%b %d']: + try: + dt = datetime.strptime(l, fmt) + return dt.strftime('%Y-%m-%d') + except ValueError: + pass + return datetime.now().strftime('%Y-%m-%d') + +def _extract_posts_from_text(raw: str, url: str) -> list[dict]: + lines = [l.strip() for l in raw.split('\n')] + blocks = [] + cur = [] + fb_run = 0 + for l in lines: + if 'facebook' in l.lower(): + fb_run += 1 + if cur and fb_run >= 2: + blocks.append(cur) + cur = [] + else: + fb_run = 0 + if l and len(l) >= 15: + words = re.findall(r'[A-Za-z]{2,}', l) + if len(words) >= 2: + cur.append(l) + if cur: + blocks.append(cur) + + posts = [] + seen_texts = set() + for block in blocks: + if not block: + continue + kw_indices = [i for i, l in enumerate(block) if kw_match(l)] + if not kw_indices: + continue + i = kw_indices[0] + start = max(0, i - 2) + end = min(len(block), i + 5) + snippet = ' '.join(block[start:end]) + if len(snippet) < 40: + continue + dekey = snippet[:80] + if dekey in seen_texts: + continue + seen_texts.add(dekey) + posts.append({ + "title": snippet[:300], + "content": snippet[:1000], + "author": block[start] if start < i else '', + "url": url, + "source": "facebook", + "date": _parse_fb_date(block), + }) + return posts + +async def search_facebook(page, query: str) -> list[dict]: + url = f'https://www.facebook.com/search/posts/?q={urllib.parse.quote(query)}' + try: + await page.goto(url, wait_until='domcontentloaded', timeout=30000) + await page.wait_for_timeout(6000) + except Exception as e: + logger.warning("Facebook search navigation failed for '%s': %s", query, e) + return [] + + try: + raw = await page.evaluate('document.body.innerText') + except Exception as e: + logger.warning("Failed to evaluate page text: %s", e) + return [] + return _extract_posts_from_text(raw, url) + +async def scrape_facebook() -> list[dict]: + all_posts = [] + fb_cookies = await get_fb_cookies() + if not fb_cookies: + logger.warning("No Facebook cookies available") + return [] + try: + async with async_playwright() as pw: + browser = await pw.chromium.launch(headless=True) + context = await browser.new_context( + user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:150.0) Gecko/20100101 Firefox/150.0', + viewport={'width': 1280, 'height': 800}, + ) + for c in fb_cookies: + try: + await context.add_cookies([c]) + except Exception as e: + logger.warning("Failed to inject cookie %s: %s", c.get('name'), e) + + page = await context.new_page() + await page.goto('https://www.facebook.com/', wait_until='domcontentloaded', timeout=30000) + await page.wait_for_timeout(5000) + + url = page.url + if '/login' in url.lower(): + logger.warning("Facebook login page detected — cookies may be expired") + return [] + + for query in FB_SEARCHES[:6]: + try: + posts = await search_facebook(page, query) + all_posts.extend(posts) + delay = random.uniform(3, 7) + await page.wait_for_timeout(int(delay * 1000)) + except Exception as e: + logger.warning("Facebook search '%s' failed: %s", query, e) + except Exception as e: + logger.error("Facebook scrape failed: %s", e) + return [] + + seen = set() + deduped = [] + for p in all_posts: + key = p.get('content', '')[:100] + if key not in seen: + seen.add(key) + deduped.append(p) + return deduped[:20] + +async def ask_ollama(prompt: str) -> str: + async with httpx.AsyncClient(timeout=120) as c: + r = await c.post(f"{OLLAMA_URL}/api/chat", json={ + "model": CLASSIFY_MODEL, + "messages": [ + {"role": "system", "content": "You classify forum posts. Return only valid JSON."}, + {"role": "user", "content": prompt} + ], + "stream": False, + "options": {"temperature": 0.05, "num_predict": 1024}, + }) + r.raise_for_status() + data = r.json() + return data["message"]["content"] + +async def scrape_warriorforum() -> list[dict]: + results = [] + try: + async with httpx.AsyncClient(headers=HEADERS, timeout=15, follow_redirects=True) as c: + r = await c.get('https://www.warriorforum.com/wanted-members-looking-hire-you/') + soup = BeautifulSoup(r.text, 'lxml') + for row in soup.find_all('td', class_='FlexTable-item--title'): + a = row.find('a', href=True) + if not a: + continue + href = a['href'] + title = a.text.strip() + if not title or len(title) < 15 or not kw_match_strict(title): + continue + author_div = row.find('div', class_='FlexTable-item-author') + author = author_div.get_text(strip=True).lstrip('by') if author_div else '' + date_str = '' + date_div = row.find('div', class_=lambda c: c and 'media--available' in c) + if date_div: + txt = date_div.get_text(strip=True) + m = re.search(r'(\d{10})', txt) + if m: + date_str = datetime.utcfromtimestamp(int(m.group(1))).strftime('%Y-%m-%d') + if not href.startswith('http'): + href = 'https://www.warriorforum.com' + href + results.append({ + "title": title, "url": href, + "author": author, + "content": title[:300], + "source": "warriorforum", + "date": date_str + }) + except Exception: + logger.error("WarriorForum scrape failed", exc_info=True) + return results + +async def classify_leads(results: list[dict]) -> list[dict]: + if not results: + return [] + briefs = [r["title"][:200] for r in results] + prompt = f"""Classify each post as LEAD or NOT. +LEAD = someone REQUESTING/POSTING/WANTING a website built, designed, or created for them. +LEAD examples: "Need a website for my business", "Looking for web developer to build my site", "I need someone to create my website", "Want a new website for my company", "Looking for someone to design my WordPress site" + +NOT LEAD: +- Offering web design services: "I build websites", "I offer web design", "Affordable web design packages" +- Already have a website and need marketing, SEO, content, video, link building, email marketing, affiliates +- Recruiting employees, hiring staff, looking for business partners +- Selling products, promoting services, affiliate offers +- "Need web hosting", "Looking for a partner", "Looking for content writer", "Video spokesperson" + +For each numbered post, answer ONLY "yes" (LEAD) or "no" (NOT LEAD): +{chr(10).join(f'{i+1}. {t}' for i, t in enumerate(briefs))} +Return a JSON array like ["yes","no","yes"] matching the order above.""" + ai_succeeded = False + try: + raw = await ask_ollama(prompt) + raw = raw.strip() + # Strip markdown code fences + if raw.startswith("```"): + # Remove opening fence + first_nl = raw.find('\n') + if first_nl != -1: + raw = raw[first_nl + 1:] + # Remove closing fence + if raw.endswith("```"): + raw = raw[:-3] + raw = raw.strip() + answers = json.loads(raw) + if isinstance(answers, list) and len(answers) == len(results): + ai_succeeded = True + filtered = [] + for i, ans in enumerate(answers): + if isinstance(ans, str) and ans.lower() == 'yes': + filtered.append(results[i]) + if filtered: + return filtered[:10] + # AI successfully classified but returned all "no" — respect that decision + logger.info("AI classified all %d items as NOT LEAD — returning empty", len(results)) + return [] + except Exception as e: + logger.warning("AI classification failed, falling back to keyword filter: %s", e) + + # Fallback: only use keyword filter when AI failed + if not ai_succeeded: + filtered = [] + for r in results: + t = r['title'].lower() + if not kw_match_strict(t): + continue + if any(kw in t for kw in ['i build', 'i offer', 'i create', 'i am a', 'web developer available', + 'affordable web', 'web hosting', 'free website', + 'limited time', 'special offer', 'sign up now', + 'offer']): + continue + filtered.append(r) + return filtered[:10] + return [] + +@app.get("/health") +async def health(): + return {"status": "ok"} + +class ScrapeRequest(BaseModel): + query: str = "" + +@app.post("/scrape/requests") +async def scrape_requests(req: ScrapeRequest): + global last_scrape_time + now = datetime.now() + if last_scrape_time and (now - last_scrape_time) < timedelta(seconds=15): + return {"error": "rate_limited", "retry_after": 15} + last_scrape_time = now + results = await scrape_warriorforum() + if results: + results = await classify_leads(results) + return results[:10] + +@app.post("/scrape/facebook") +async def scrape_facebook_endpoint(): + results = await scrape_facebook() + if results: + results = await classify_leads(results) + return results[:15] + +@app.post("/scrape/all") +async def scrape_all(): + results = [] + try: + wf = await scrape_warriorforum() + if wf: + wf = await classify_leads(wf) + results.extend(wf[:5]) + except Exception as e: + logger.error("WarriorForum scrape in /scrape/all failed: %s", e) + try: + fb = await scrape_facebook() + if fb: + fb = await classify_leads(fb) + results.extend(fb[:8]) + except Exception as e: + logger.error("Facebook scrape in /scrape/all failed: %s", e) + return results[:12] + +if __name__ == "__main__": + uvicorn.run(app, host="0.0.0.0", port=PORT) diff --git a/frontend_old/browser-use-service/out.txt b/frontend_old/browser-use-service/out.txt new file mode 100644 index 0000000..e69de29 diff --git a/frontend_old/browser-use-service/run_log.txt b/frontend_old/browser-use-service/run_log.txt new file mode 100644 index 0000000..df792e6 --- /dev/null +++ b/frontend_old/browser-use-service/run_log.txt @@ -0,0 +1,7 @@ +python : INFO: Started server process [20044] + + CategoryInfo : NotSpecified: (INFO: Started server process [20044]:String) [], RemoteException + + FullyQualifiedErrorId : NativeCommandError + +INFO: Waiting for application startup. +INFO: Application startup complete. +INFO: Uvicorn running on http://0.0.0.0:3008 (Press CTRL+C to quit) diff --git a/frontend_old/browser-use-service/scrape_log.txt b/frontend_old/browser-use-service/scrape_log.txt new file mode 100644 index 0000000..a356eb3 --- /dev/null +++ b/frontend_old/browser-use-service/scrape_log.txt @@ -0,0 +1,6 @@ +[2026-06-22T19:24:48.030482] Browser Use service starting on port 3008 +[2026-06-22T19:27:19.231701] Browser Use service starting on port 3008 +[2026-06-22T19:28:28.335765] Browser Use service starting on port 3008 +[2026-06-22T19:29:05.796265] Browser Use service starting on port 3008 +[2026-06-22T19:29:17.042807] Scraping WarriorForum... +[2026-06-22T19:29:19.075166] WarriorForum: 55 results diff --git a/frontend_old/browser-use-service/stderr.txt b/frontend_old/browser-use-service/stderr.txt new file mode 100644 index 0000000..85fb7e5 --- /dev/null +++ b/frontend_old/browser-use-service/stderr.txt @@ -0,0 +1,4 @@ +INFO: Started server process [24268] +INFO: Waiting for application startup. +INFO: Application startup complete. +INFO: Uvicorn running on http://0.0.0.0:3008 (Press CTRL+C to quit) diff --git a/frontend_old/browser-use-service/stdout.txt b/frontend_old/browser-use-service/stdout.txt new file mode 100644 index 0000000..66ec1d4 --- /dev/null +++ b/frontend_old/browser-use-service/stdout.txt @@ -0,0 +1,5 @@ +Browser Use service starting on port 3008 +INFO: 127.0.0.1:65003 - "GET /health HTTP/1.1" 200 OK +Scraping WarriorForum... +WarriorForum: 55 results +INFO: 127.0.0.1:54587 - "POST /scrape/all HTTP/1.1" 200 OK diff --git a/frontend_old/database/migrations/001_schema.sql b/frontend_old/database/migrations/001_schema.sql index 231cd74..9ae5d1c 100644 --- a/frontend_old/database/migrations/001_schema.sql +++ b/frontend_old/database/migrations/001_schema.sql @@ -22,7 +22,7 @@ $$ LANGUAGE plpgsql; -- 1. AUTHENTICATION & ROLE MANAGEMENT -- ============================================================================ -CREATE TABLE roles ( +CREATE TABLE IF NOT EXISTS roles ( id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), name VARCHAR(50) NOT NULL, display_name VARCHAR(100), @@ -35,7 +35,7 @@ CREATE TABLE roles ( CREATE UNIQUE INDEX uq_roles_name ON roles(name); -CREATE TABLE permissions ( +CREATE TABLE IF NOT EXISTS permissions ( id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), resource VARCHAR(100) NOT NULL, action VARCHAR(50) NOT NULL, @@ -46,7 +46,7 @@ CREATE TABLE permissions ( CREATE UNIQUE INDEX uq_permissions ON permissions(resource, action); -CREATE TABLE role_permissions ( +CREATE TABLE IF NOT EXISTS role_permissions ( role_id UUID NOT NULL REFERENCES roles(id) ON DELETE CASCADE, permission_id UUID NOT NULL REFERENCES permissions(id) ON DELETE CASCADE, granted_by UUID, @@ -54,7 +54,7 @@ CREATE TABLE role_permissions ( PRIMARY KEY (role_id, permission_id) ); -CREATE TABLE users ( +CREATE TABLE IF NOT EXISTS users ( id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), username VARCHAR(100) NOT NULL, email VARCHAR(255) NOT NULL, @@ -83,7 +83,7 @@ CREATE INDEX idx_users_is_active ON users(is_active) WHERE deleted_at IS NULL; CREATE INDEX idx_users_is_locked ON users(is_locked) WHERE is_locked = TRUE AND deleted_at IS NULL; CREATE INDEX idx_users_created_by ON users(created_by); -CREATE TABLE user_roles ( +CREATE TABLE IF NOT EXISTS user_roles ( user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, role_id UUID NOT NULL REFERENCES roles(id) ON DELETE CASCADE, assigned_by UUID REFERENCES users(id), @@ -98,7 +98,7 @@ CREATE INDEX idx_user_roles_role ON user_roles(role_id); -- 2. BAN & SUSPENSION SYSTEM -- ============================================================================ -CREATE TABLE banned_users ( +CREATE TABLE IF NOT EXISTS banned_users ( id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), banned_user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, banned_by UUID NOT NULL REFERENCES users(id), @@ -122,7 +122,7 @@ CREATE INDEX idx_banned_users_active ON banned_users(banned_user_id) WHERE is_reversed = FALSE; CREATE INDEX idx_banned_users_banned_by ON banned_users(banned_by); -CREATE TABLE suspended_users ( +CREATE TABLE IF NOT EXISTS suspended_users ( id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), suspended_user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, suspended_by UUID NOT NULL REFERENCES users(id), @@ -145,7 +145,7 @@ CREATE INDEX idx_suspended_users_suspended_by ON suspended_users(suspended_by); -- 3. SESSION & LOGIN MANAGEMENT -- ============================================================================ -CREATE TABLE sessions ( +CREATE TABLE IF NOT EXISTS sessions ( id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, token_hash VARCHAR(255) NOT NULL, @@ -161,7 +161,7 @@ CREATE INDEX idx_sessions_user ON sessions(user_id) WHERE is_active = TRUE; CREATE UNIQUE INDEX uq_sessions_token ON sessions(token_hash); CREATE INDEX idx_sessions_expires ON sessions(expires_at) WHERE is_active = TRUE; -CREATE TABLE login_attempts ( +CREATE TABLE IF NOT EXISTS login_attempts ( id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), user_id UUID REFERENCES users(id), username_attempted VARCHAR(100) NOT NULL, @@ -181,7 +181,7 @@ CREATE INDEX idx_login_attempts_time ON login_attempts(attempted_at DESC); -- 4. CUSTOMER MANAGEMENT -- ============================================================================ -CREATE TABLE customer_statuses ( +CREATE TABLE IF NOT EXISTS customer_statuses ( id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), name VARCHAR(100) NOT NULL, description TEXT, @@ -196,7 +196,7 @@ CREATE TABLE customer_statuses ( CREATE UNIQUE INDEX uq_customer_statuses_name ON customer_statuses(name) WHERE deleted_at IS NULL; -CREATE TABLE customers ( +CREATE TABLE IF NOT EXISTS customers ( id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), customer_type VARCHAR(20) NOT NULL, status_id UUID NOT NULL REFERENCES customer_statuses(id), @@ -218,7 +218,7 @@ CREATE INDEX idx_customers_score ON customers(score DESC) WHERE deleted_at IS NU CREATE INDEX idx_customers_tags ON customers USING GIN(tags) WHERE deleted_at IS NULL; CREATE INDEX idx_customers_created ON customers(created_at DESC) WHERE deleted_at IS NULL; -CREATE TABLE individual_customers ( +CREATE TABLE IF NOT EXISTS individual_customers ( customer_id UUID PRIMARY KEY REFERENCES customers(id) ON DELETE CASCADE, first_name VARCHAR(100) NOT NULL, last_name VARCHAR(100) NOT NULL, @@ -233,7 +233,7 @@ CREATE TABLE individual_customers ( CREATE INDEX idx_individual_names ON individual_customers(last_name, first_name); -CREATE TABLE company_customers ( +CREATE TABLE IF NOT EXISTS company_customers ( customer_id UUID PRIMARY KEY REFERENCES customers(id) ON DELETE CASCADE, company_name VARCHAR(255) NOT NULL, registration_number VARCHAR(100), @@ -250,7 +250,7 @@ CREATE TABLE company_customers ( CREATE INDEX idx_company_name ON company_customers(company_name); CREATE INDEX idx_company_industry ON company_customers(industry); -CREATE TABLE contact_information ( +CREATE TABLE IF NOT EXISTS contact_information ( id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), customer_id UUID NOT NULL REFERENCES customers(id) ON DELETE CASCADE, type VARCHAR(20) NOT NULL, @@ -270,7 +270,7 @@ CREATE INDEX idx_contacts_type ON contact_information(type) WHERE deleted_at IS CREATE INDEX idx_contacts_value ON contact_information(value) WHERE deleted_at IS NULL; CREATE UNIQUE INDEX uq_contacts_primary ON contact_information(customer_id, type) WHERE is_primary = TRUE AND deleted_at IS NULL; -CREATE TABLE addresses ( +CREATE TABLE IF NOT EXISTS addresses ( id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), customer_id UUID NOT NULL REFERENCES customers(id) ON DELETE CASCADE, address_type VARCHAR(20) NOT NULL, @@ -294,7 +294,7 @@ CREATE INDEX idx_addresses_country ON addresses(country) WHERE deleted_at IS NUL CREATE INDEX idx_addresses_city ON addresses(city) WHERE deleted_at IS NULL; CREATE UNIQUE INDEX uq_addresses_primary ON addresses(customer_id, address_type) WHERE is_primary = TRUE AND deleted_at IS NULL; -CREATE TABLE customer_notes ( +CREATE TABLE IF NOT EXISTS customer_notes ( id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), customer_id UUID NOT NULL REFERENCES customers(id) ON DELETE CASCADE, author_id UUID NOT NULL REFERENCES users(id), @@ -315,7 +315,7 @@ CREATE INDEX idx_notes_pinned ON customer_notes(is_pinned) WHERE is_pinned = TRU -- 5. CUSTOMER & INCIDENT REPORTS -- ============================================================================ -CREATE TABLE customer_reports ( +CREATE TABLE IF NOT EXISTS customer_reports ( id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), customer_id UUID NOT NULL REFERENCES customers(id), reported_by UUID NOT NULL REFERENCES users(id), @@ -339,7 +339,7 @@ CREATE INDEX idx_customer_reports_reported_by ON customer_reports(reported_by); CREATE INDEX idx_customer_reports_status ON customer_reports(status); CREATE INDEX idx_customer_reports_severity ON customer_reports(severity); -CREATE TABLE incident_reports ( +CREATE TABLE IF NOT EXISTS incident_reports ( id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), reported_by UUID NOT NULL REFERENCES users(id), target_user_id UUID REFERENCES users(id), @@ -368,7 +368,7 @@ CREATE INDEX idx_incident_reports_severity ON incident_reports(severity); -- 6. LEAD MANAGEMENT -- ============================================================================ -CREATE TABLE lead_sources ( +CREATE TABLE IF NOT EXISTS lead_sources ( id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), name VARCHAR(100) NOT NULL, description TEXT, @@ -380,7 +380,7 @@ CREATE TABLE lead_sources ( CREATE UNIQUE INDEX uq_lead_sources_name ON lead_sources(name) WHERE deleted_at IS NULL; -CREATE TABLE lead_stages ( +CREATE TABLE IF NOT EXISTS lead_stages ( id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), name VARCHAR(100) NOT NULL, description TEXT, @@ -395,7 +395,7 @@ CREATE TABLE lead_stages ( CREATE UNIQUE INDEX uq_lead_stages_name ON lead_stages(name) WHERE deleted_at IS NULL; CREATE INDEX idx_lead_stages_sort ON lead_stages(sort_order); -CREATE TABLE leads ( +CREATE TABLE IF NOT EXISTS leads ( id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), source_id UUID REFERENCES lead_sources(id), stage_id UUID NOT NULL REFERENCES lead_stages(id), @@ -431,7 +431,7 @@ CREATE INDEX idx_leads_converted ON leads(converted_customer_id) WHERE converted CREATE INDEX idx_leads_email ON leads(email) WHERE deleted_at IS NULL; CREATE INDEX idx_leads_active ON leads(stage_id, assigned_to) WHERE deleted_at IS NULL AND converted_customer_id IS NULL; -CREATE TABLE lead_conversions ( +CREATE TABLE IF NOT EXISTS lead_conversions ( id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), lead_id UUID NOT NULL REFERENCES leads(id) ON DELETE CASCADE, customer_id UUID NOT NULL REFERENCES customers(id) ON DELETE CASCADE, @@ -447,7 +447,7 @@ CREATE INDEX idx_lead_conversions_customer ON lead_conversions(customer_id); -- 7. PRODUCT CATALOG -- ============================================================================ -CREATE TABLE product_categories ( +CREATE TABLE IF NOT EXISTS product_categories ( id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), name VARCHAR(255) NOT NULL, description TEXT, @@ -462,7 +462,7 @@ CREATE TABLE product_categories ( CREATE UNIQUE INDEX uq_product_categories_name ON product_categories(name) WHERE deleted_at IS NULL; CREATE INDEX idx_product_categories_parent ON product_categories(parent_id); -CREATE TABLE products ( +CREATE TABLE IF NOT EXISTS products ( id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), category_id UUID REFERENCES product_categories(id), name VARCHAR(255) NOT NULL, @@ -487,7 +487,7 @@ CREATE INDEX idx_products_active ON products(is_active) WHERE deleted_at IS NULL -- 8. SALES PIPELINE -- ============================================================================ -CREATE TABLE deal_stages ( +CREATE TABLE IF NOT EXISTS deal_stages ( id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), name VARCHAR(100) NOT NULL, description TEXT, @@ -502,7 +502,7 @@ CREATE TABLE deal_stages ( CREATE UNIQUE INDEX uq_deal_stages_name ON deal_stages(name) WHERE deleted_at IS NULL; CREATE INDEX idx_deal_stages_sort ON deal_stages(sort_order); -CREATE TABLE opportunities ( +CREATE TABLE IF NOT EXISTS opportunities ( id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), customer_id UUID NOT NULL REFERENCES customers(id), lead_id UUID REFERENCES leads(id), @@ -530,7 +530,7 @@ CREATE INDEX idx_opportunities_won ON opportunities(is_won) WHERE deleted_at IS CREATE INDEX idx_opportunities_created ON opportunities(created_at DESC) WHERE deleted_at IS NULL; CREATE INDEX idx_opportunities_owner_stage ON opportunities(owner_id, stage_id) WHERE deleted_at IS NULL AND is_won IS NULL; -CREATE TABLE opportunity_products ( +CREATE TABLE IF NOT EXISTS opportunity_products ( id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), opportunity_id UUID NOT NULL REFERENCES opportunities(id) ON DELETE CASCADE, product_id UUID NOT NULL REFERENCES products(id), @@ -549,7 +549,7 @@ CREATE INDEX idx_opp_products_product ON opportunity_products(product_id); -- 9. COMMUNICATION TRACKING -- ============================================================================ -CREATE TABLE communication_types ( +CREATE TABLE IF NOT EXISTS communication_types ( id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), name VARCHAR(50) NOT NULL, description TEXT, @@ -561,7 +561,7 @@ CREATE TABLE communication_types ( CREATE UNIQUE INDEX uq_comm_types_name ON communication_types(name); -CREATE TABLE communications ( +CREATE TABLE IF NOT EXISTS communications ( id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), customer_id UUID NOT NULL REFERENCES customers(id), opportunity_id UUID REFERENCES opportunities(id), @@ -585,7 +585,7 @@ CREATE INDEX idx_communications_type ON communications(type_id) WHERE deleted_at CREATE INDEX idx_communications_created_by ON communications(created_by) WHERE deleted_at IS NULL; CREATE INDEX idx_communications_started ON communications(started_at) WHERE deleted_at IS NULL; -CREATE TABLE communication_participants ( +CREATE TABLE IF NOT EXISTS communication_participants ( id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), communication_id UUID NOT NULL REFERENCES communications(id) ON DELETE CASCADE, user_id UUID REFERENCES users(id), @@ -602,7 +602,7 @@ CREATE INDEX idx_comm_participants_user ON communication_participants(user_id); -- 10. TASKS AND ACTIVITIES -- ============================================================================ -CREATE TABLE task_priorities ( +CREATE TABLE IF NOT EXISTS task_priorities ( id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), name VARCHAR(50) NOT NULL, color VARCHAR(7), @@ -613,7 +613,7 @@ CREATE TABLE task_priorities ( CREATE UNIQUE INDEX uq_task_priorities_name ON task_priorities(name); -CREATE TABLE tasks ( +CREATE TABLE IF NOT EXISTS tasks ( id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), customer_id UUID REFERENCES customers(id), opportunity_id UUID REFERENCES opportunities(id), @@ -645,7 +645,7 @@ CREATE INDEX idx_tasks_reminder ON tasks(reminder_at) WHERE reminder_sent = FALS -- 11. INVOICE SUPPORT -- ============================================================================ -CREATE TABLE invoice_statuses ( +CREATE TABLE IF NOT EXISTS invoice_statuses ( id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), name VARCHAR(50) NOT NULL, description TEXT, @@ -657,7 +657,7 @@ CREATE TABLE invoice_statuses ( CREATE UNIQUE INDEX uq_invoice_statuses_name ON invoice_statuses(name); -CREATE TABLE invoices ( +CREATE TABLE IF NOT EXISTS invoices ( id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), customer_id UUID NOT NULL REFERENCES customers(id), opportunity_id UUID REFERENCES opportunities(id), @@ -689,7 +689,7 @@ CREATE INDEX idx_invoices_issued ON invoices(issued_date DESC) WHERE deleted_at CREATE INDEX idx_invoices_due ON invoices(due_date) WHERE deleted_at IS NULL; -- Skipped: cannot use NOW() in partial index predicate (not IMMUTABLE) -CREATE TABLE invoice_items ( +CREATE TABLE IF NOT EXISTS invoice_items ( id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), invoice_id UUID NOT NULL REFERENCES invoices(id) ON DELETE CASCADE, product_id UUID REFERENCES products(id), @@ -705,7 +705,7 @@ CREATE TABLE invoice_items ( CREATE INDEX idx_invoice_items_invoice ON invoice_items(invoice_id); CREATE INDEX idx_invoice_items_product ON invoice_items(product_id); -CREATE TABLE payments ( +CREATE TABLE IF NOT EXISTS payments ( id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), invoice_id UUID NOT NULL REFERENCES invoices(id), amount DECIMAL(15,2) NOT NULL CHECK (amount > 0), @@ -729,7 +729,7 @@ CREATE INDEX idx_payments_transaction ON payments(transaction_id) WHERE transact -- 12. AUDIT LOGGING -- ============================================================================ -CREATE TABLE audit_logs ( +CREATE TABLE IF NOT EXISTS audit_logs ( id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), table_name VARCHAR(100) NOT NULL, record_id UUID NOT NULL, @@ -755,7 +755,7 @@ CREATE INDEX idx_audit_session ON audit_logs(session_id); -- 13. DUPLICATE DETECTION -- ============================================================================ -CREATE TABLE customer_duplicates ( +CREATE TABLE IF NOT EXISTS customer_duplicates ( id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), customer_id UUID NOT NULL REFERENCES customers(id) ON DELETE CASCADE, duplicate_customer_id UUID NOT NULL REFERENCES customers(id) ON DELETE CASCADE, @@ -810,11 +810,11 @@ BEGIN ) LOOP EXECUTE format( - 'CREATE TRIGGER trg_%I_updated_at + 'DROP TRIGGER IF EXISTS trg_%I_updated_at ON %I; CREATE TRIGGER trg_%I_updated_at BEFORE UPDATE ON %I FOR EACH ROW EXECUTE FUNCTION update_updated_at_column()', - t, t + t, t, t, t ); END LOOP; END; @@ -885,6 +885,7 @@ BEGIN END; $$ LANGUAGE plpgsql SECURITY DEFINER; +DROP TRIGGER IF EXISTS trg_enforce_create_user ON users; CREATE TRIGGER trg_enforce_create_user BEFORE INSERT ON users FOR EACH ROW @@ -935,6 +936,7 @@ BEGIN END; $$ LANGUAGE plpgsql SECURITY DEFINER; +DROP TRIGGER IF EXISTS trg_enforce_role_assignment ON user_roles; CREATE TRIGGER trg_enforce_role_assignment BEFORE INSERT ON user_roles FOR EACH ROW @@ -973,7 +975,7 @@ BEGIN audit_action, CASE WHEN audit_action IN ('UPDATE', 'DELETE') THEN old_row ELSE NULL END, CASE WHEN audit_action IN ('CREATE', 'UPDATE') THEN new_row ELSE NULL END, - NULL + NULLIF(current_setting('app.current_user_id', true), '') ); RETURN COALESCE(NEW, OLD); @@ -998,6 +1000,7 @@ BEGIN END; $$ LANGUAGE plpgsql SECURITY DEFINER; +DROP TRIGGER IF EXISTS trg_audit_ban ON banned_users; CREATE TRIGGER trg_audit_ban AFTER INSERT OR UPDATE ON banned_users FOR EACH ROW @@ -1021,6 +1024,7 @@ BEGIN END; $$ LANGUAGE plpgsql SECURITY DEFINER; +DROP TRIGGER IF EXISTS trg_audit_suspension ON suspended_users; CREATE TRIGGER trg_audit_suspension AFTER INSERT OR UPDATE ON suspended_users FOR EACH ROW @@ -1051,6 +1055,7 @@ BEGIN END; $$ LANGUAGE plpgsql SECURITY DEFINER; +DROP TRIGGER IF EXISTS trg_audit_login ON login_attempts; CREATE TRIGGER trg_audit_login AFTER INSERT ON login_attempts FOR EACH ROW diff --git a/frontend_old/database/migrations/003_chat.sql b/frontend_old/database/migrations/003_chat.sql index 9221490..68cefa2 100644 --- a/frontend_old/database/migrations/003_chat.sql +++ b/frontend_old/database/migrations/003_chat.sql @@ -29,6 +29,8 @@ CREATE INDEX IF NOT EXISTS idx_messages_conversation_id ON messages(conversation CREATE INDEX IF NOT EXISTS idx_messages_created_at ON messages(created_at); CREATE INDEX IF NOT EXISTS idx_conversation_participants_user_id ON conversation_participants(user_id); +CREATE INDEX IF NOT EXISTS idx_messages_conversation_created ON messages(conversation_id, created_at DESC); + -- Seed conversations between superadmin and other users INSERT INTO conversations (id, created_at) VALUES ('c0000000-0000-0000-0000-000000000001', NOW() - INTERVAL '2 hours'), diff --git a/frontend_old/database/migrations/009_settings.sql b/frontend_old/database/migrations/009_settings.sql new file mode 100644 index 0000000..5874ba0 --- /dev/null +++ b/frontend_old/database/migrations/009_settings.sql @@ -0,0 +1,22 @@ +CREATE TABLE IF NOT EXISTS company_settings ( + id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), + company_name VARCHAR(255) NOT NULL DEFAULT '', + company_email VARCHAR(255) NOT NULL DEFAULT '', + company_phone VARCHAR(50) NOT NULL DEFAULT '', + company_website VARCHAR(255) NOT NULL DEFAULT '', + company_address TEXT NOT NULL DEFAULT '', + updated_by UUID REFERENCES users(id), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +INSERT INTO company_settings (company_name, company_email, company_phone, company_website, company_address) +VALUES ('Coastal IT Solutions', 'info@coastalit.com', '(555) 123-4567', 'https://coastalit.com', '123 Business Ave, Suite 100, San Francisco, CA 94105') +ON CONFLICT DO NOTHING; + +CREATE TABLE IF NOT EXISTS user_preferences ( + user_id UUID PRIMARY KEY REFERENCES users(id) ON DELETE CASCADE, + timezone VARCHAR(100) NOT NULL DEFAULT 'america-los_angeles', + date_format VARCHAR(10) NOT NULL DEFAULT 'mdy', + items_per_page INTEGER NOT NULL DEFAULT 20, + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); diff --git a/frontend_old/database/migrations/010_chat_notifications.sql b/frontend_old/database/migrations/010_chat_notifications.sql new file mode 100644 index 0000000..b96c3fd --- /dev/null +++ b/frontend_old/database/migrations/010_chat_notifications.sql @@ -0,0 +1,4 @@ +ALTER TABLE notifications ADD COLUMN IF NOT EXISTS context_id UUID; +ALTER TABLE notifications ADD COLUMN IF NOT EXISTS context_type VARCHAR(50); + +CREATE INDEX IF NOT EXISTS idx_notifications_context ON notifications(context_type, context_id) WHERE context_type IS NOT NULL; diff --git a/frontend_old/database/migrations/run_all.sql b/frontend_old/database/migrations/run_all.sql index c5d75d0..a7c3b72 100644 --- a/frontend_old/database/migrations/run_all.sql +++ b/frontend_old/database/migrations/run_all.sql @@ -4,6 +4,9 @@ -- Usage: psql -U postgres -d crm -f run_all.sql -- ============================================================================ +BEGIN; +\set ON_ERROR_STOP on + \echo '=== Running 001_schema.sql (Tables + Constraints + Functions + Views) ===' \i 001_schema.sql @@ -28,10 +31,11 @@ \echo '=== Running 008_notifications.sql (Notifications + Preferences) ===' \i 008_notifications.sql +\echo '=== Running 009_settings.sql (Company Settings + User Preferences) ===' +\i 009_settings.sql + +\echo '=== Running 010_chat_notifications.sql (Chat Message Notifications) ===' +\i 010_chat_notifications.sql + \echo '=== Migration Complete ===' -\echo '' -\echo 'Test accounts created:' -\echo ' superadmin_demo / SuperAdmin@2026' -\echo ' admin_demo / AdminAccess@2026' -\echo ' sales_demo / SalesAccess@2026' -\echo ' dev_demo / DevTesting@2026' +COMMIT; diff --git a/frontend_old/fix.md b/frontend_old/fix.md index 1c6a94c..88b2025 100644 --- a/frontend_old/fix.md +++ b/frontend_old/fix.md @@ -1,60 +1,71 @@ -# Facebook Scraper - Configuration Needed +# Scrapers - Configuration & Status -## Proxy Configuration +## Facebook Scraper **File:** `rust-ai/src/main.rs` -**Lines:** 25-27 - -The scraper needs real proxy URLs. The dummy placeholder `"http://0.0.0.0:0"` will fail to connect (expected — no crash, just error logs): +**Lines:** ~70-85 +Target URL (line 72): ```rust -const PROXIES: &[&str] = &[ - "http://0.0.0.0:0", // <- Replace with real proxy(es) -]; +let url = "https://www.facebook.com/search/top/?q=need%20website%20create"; ``` -Replace with your actual proxies, e.g.: -```rust -const PROXIES: &[&str] = &[ - "http://user:pass@192.168.1.1:8080", - "http://user:pass@192.168.1.2:8080", -]; +**Status:** Uses direct connection (no proxy) — your home IP. Facebook blocks datacenter IPs. May work from a residential connection. + +**Test with curl:** +``` +curl.exe -s "https://www.facebook.com/search/top/?q=need%20website%20create" -H "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36" --max-time 10 2>$null ``` -## User Agents (Optional) +**Expected log output when blocked:** +``` +ERROR crm_ai: Facebook scraper error: error sending request for url (https://www.facebook.com/search/top/?q=need%20website%20create) +``` + +## Reddit Scraper **File:** `rust-ai/src/main.rs` -**Lines:** 32-36 +**Lines:** ~90-120 -Add more realistic user agents here if needed. +Uses `old.reddit.com` (older design that allows scraping without captchas). -## Scraper Target URL +Search queries (currently 6, lines 93-100): +- `r/southafrica` — "need website", "web developer" +- Global — '"need a website"', "website quote" +- `r/forhire` — "website" +- `r/smallbusiness` — "website" + +**Status:** Working. Reddit results appear as `INFO LEAD:` entries in the server log. + +**Test with curl:** +``` +curl.exe -s "https://old.reddit.com/r/southafrica/search?q=need+website&sort=new&restrict_sr=on&t=week" -H "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36" +``` + +**Expected log output when working:** +``` +INFO crm_ai: LEAD: [Post Title] -> https://old.reddit.com/r/.../... +``` + +## Background Loop **File:** `rust-ai/src/main.rs` -**Line:** 68 +**Lines:** ~370-383 -Currently fetches `https://www.facebook.com/search/top/?q=need%20website%20create`. Change if needed. +Both scrapers run together every 60-180 seconds on a `spawn_blocking` thread. -## Background Thread +## What You Need to Do -**File:** `rust-ai/src/main.rs` -**Lines:** 355-371 +### Facebook +- Try running from your home PC instead — your residential IP may not be blocked +- If still blocked, you need residential proxies (BrightData, IPRoyal, Oxylabs) +- Configure them in the `PROXIES` array at line 25 -Runs in a `tokio::task::spawn_blocking` thread (changed from `tokio::spawn` because the scraper uses `reqwest::blocking::Client` + `thread::sleep`). +### Reddit +- Works out of the box via `old.reddit.com` +- If it stops working, Reddit IP-blocked you — use proxies or switch to Playwright -## Expected Behavior Until Configured +### To add more search queries +Edit the `searches` array in `run_reddit_scraper()` (line 93). -Until real proxies are set, the Rust server will log this every 1-5 seconds (not a crash): -``` -ERROR crm_ai: Facebook scraper error: error sending request for url (https://www.facebook.com/messages) -``` -Once you add working proxies, these errors stop and actual scraping begins. - -## How it works - -- Runs every 1-5 seconds on a background blocking thread (line 355-371) -- Rotates through proxies and user agents for each request -- Parses conversation HTML via `scraper` crate -- Designed to detect job posts like "need a website" and notify sales reps -- Error handling added to prevent server crash (uses `match` instead of `.unwrap()` at line 69) diff --git a/frontend_old/next.config.ts b/frontend_old/next.config.ts index fc53880..2e27cc2 100644 --- a/frontend_old/next.config.ts +++ b/frontend_old/next.config.ts @@ -2,7 +2,7 @@ import type { NextConfig } from "next" const nextConfig: NextConfig = { eslint: { - ignoreDuringBuilds: true, + ignoreDuringBuilds: false, }, images: { remotePatterns: [ diff --git a/frontend_old/package.json b/frontend_old/package.json index 57b42d6..8aee1d3 100644 --- a/frontend_old/package.json +++ b/frontend_old/package.json @@ -4,13 +4,14 @@ "private": true, "scripts": { "dev": "npm run dev:precheck & npm run dev:ollama & npm run dev:start", - "dev:start": "concurrently -n AI,NEXT -c cyan,green \"npm run dev:rust\" \"npm run dev:next\"", + "dev:start": "concurrently -n AI,BROWSE,NEXT -c cyan,magenta,green \"npm run dev:rust\" \"npm run dev:browser-use\" \"npm run dev:next\"", "dev:next": "next dev -p 3006", - "dev:precheck": "powershell -NoProfile -Command \"$targetPorts=3001,3006; netstat -ano | Select-String LISTENING | ForEach-Object { $line=$_.ToString(); foreach($p in $targetPorts){ if($line -match ('[:]'+$p+'\\s')){ $foundPid=($line -split '\\s+')[-1]; try{ Stop-Process -Id $foundPid -Force -ErrorAction SilentlyContinue; Write-Host ('Freed port '+$p) }catch{} } } }; exit 0\"", + "dev:precheck": "powershell -NoProfile -Command \"Get-NetTCPConnection -State Listen | Where-Object { $_.LocalPort -in 3001,3006,3008 } | ForEach-Object { Stop-Process -Id $_.OwningProcess -Force -ErrorAction SilentlyContinue; Write-Host ('Freed port '+$_.LocalPort) }\"", "dev:ollama": "powershell -NoProfile -Command \"if (-not (Get-Process ollama -ErrorAction SilentlyContinue)) { Start-Process ollama -ArgumentList 'serve' -WindowStyle Hidden; Start-Sleep 3 }; exit 0\"", "dev:rust": "cd rust-ai && cargo run", + "dev:browser-use": "cd browser-use-service && python main.py", "build": "next build", - "start": "npm run dev:next", + "start": "next start -p 3006", "lint": "eslint" }, "dependencies": { @@ -36,7 +37,6 @@ "bcryptjs": "^3.0.3", "class-variance-authority": "^0.7.1", "clsx": "^2.1.1", - "emoji-mart": "^5.6.0", "framer-motion": "^11.15.0", "jose": "^6.2.3", "lucide-react": "^0.468.0", diff --git a/frontend_old/rust-ai/Cargo.lock b/frontend_old/rust-ai/Cargo.lock index 6d3a2fb..1778fb6 100644 --- a/frontend_old/rust-ai/Cargo.lock +++ b/frontend_old/rust-ai/Cargo.lock @@ -34,7 +34,7 @@ checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn", ] [[package]] @@ -73,7 +73,7 @@ dependencies = [ "http-body-util", "hyper", "hyper-util", - "itoa 1.0.18", + "itoa", "matchit", "memchr", "mime", @@ -119,12 +119,6 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" -[[package]] -name = "bitflags" -version = "1.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" - [[package]] name = "bitflags" version = "2.13.0" @@ -226,12 +220,6 @@ dependencies = [ "crossbeam-utils", ] -[[package]] -name = "convert_case" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e" - [[package]] name = "core-foundation" version = "0.9.4" @@ -298,9 +286,9 @@ dependencies = [ "axum", "chrono", "dotenvy", + "jsonwebtoken", "rand 0.8.6", "reqwest", - "scraper", "serde", "serde_json", "sqlx", @@ -345,33 +333,6 @@ dependencies = [ "hybrid-array", ] -[[package]] -name = "cssparser" -version = "0.27.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "754b69d351cdc2d8ee09ae203db831e005560fc6030da058f86ad60c92a9cb0a" -dependencies = [ - "cssparser-macros", - "dtoa-short", - "itoa 0.4.8", - "matches", - "phf", - "proc-macro2", - "quote", - "smallvec", - "syn 1.0.109", -] - -[[package]] -name = "cssparser-macros" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13b588ba4ac1a99f7f2964d24b3d896ddc6bf847ee3855dbd4366f058cfcd331" -dependencies = [ - "quote", - "syn 2.0.118", -] - [[package]] name = "ctutils" version = "0.4.2" @@ -382,17 +343,10 @@ dependencies = [ ] [[package]] -name = "derive_more" -version = "0.99.20" +name = "deranged" +version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6edb4b64a43d977b8e99788fe3a04d483834fba1215a7e02caa415b626497f7f" -dependencies = [ - "convert_case", - "proc-macro2", - "quote", - "rustc_version", - "syn 2.0.118", -] +checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" [[package]] name = "digest" @@ -423,7 +377,7 @@ checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn", ] [[package]] @@ -432,27 +386,6 @@ version = "0.15.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" -[[package]] -name = "dtoa" -version = "1.0.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c3cf4824e2d5f025c7b531afcb2325364084a16806f6d47fbc1f5fbd9960590" - -[[package]] -name = "dtoa-short" -version = "0.3.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd1511a7b6a56299bd043a9c167a6d2bfb37bf84a6dfceaba651168adfb43c87" -dependencies = [ - "dtoa", -] - -[[package]] -name = "ego-tree" -version = "0.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12a0bb14ac04a9fcf170d0bbbef949b44cc492f4452bd20c095636956f653642" - [[package]] name = "either" version = "1.16.0" @@ -567,16 +500,6 @@ dependencies = [ "percent-encoding", ] -[[package]] -name = "futf" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df420e2e84819663797d1ec6544b13c5be84629e7bb00dc960d6917db2987843" -dependencies = [ - "mac", - "new_debug_unreachable", -] - [[package]] name = "futures-channel" version = "0.3.32" @@ -648,15 +571,6 @@ dependencies = [ "slab", ] -[[package]] -name = "fxhash" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c" -dependencies = [ - "byteorder", -] - [[package]] name = "generic-array" version = "0.14.7" @@ -667,26 +581,6 @@ dependencies = [ "version_check", ] -[[package]] -name = "getopts" -version = "0.2.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfe4fbac503b8d1f88e6676011885f34b7174f46e59956bba534ba83abded4df" -dependencies = [ - "unicode-width", -] - -[[package]] -name = "getrandom" -version = "0.1.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce" -dependencies = [ - "cfg-if", - "libc", - "wasi 0.9.0+wasi-snapshot-preview1", -] - [[package]] name = "getrandom" version = "0.2.17" @@ -694,8 +588,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" dependencies = [ "cfg-if", + "js-sys", "libc", - "wasi 0.11.1+wasi-snapshot-preview1", + "wasi", + "wasm-bindgen", ] [[package]] @@ -785,20 +681,6 @@ dependencies = [ "digest 0.11.3", ] -[[package]] -name = "html5ever" -version = "0.25.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5c13fb08e5d4dfc151ee5e88bae63f7773d61852f3bdc73c9f4b9e1bde03148" -dependencies = [ - "log", - "mac", - "markup5ever", - "proc-macro2", - "quote", - "syn 1.0.109", -] - [[package]] name = "http" version = "1.4.2" @@ -806,7 +688,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6970f50e31d6fc17d3fa27329444bfa74e196cf62e95052a3f6fee181dba6425" dependencies = [ "bytes", - "itoa 1.0.18", + "itoa", ] [[package]] @@ -868,7 +750,7 @@ dependencies = [ "http-body", "httparse", "httpdate", - "itoa 1.0.18", + "itoa", "pin-project-lite", "smallvec", "tokio", @@ -1074,12 +956,6 @@ version = "2.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" -[[package]] -name = "itoa" -version = "0.4.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4" - [[package]] name = "itoa" version = "1.0.18" @@ -1097,6 +973,21 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "jsonwebtoken" +version = "9.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a87cc7a48537badeae96744432de36f4be2b4a34a05a5ef32e9dd8a1c169dde" +dependencies = [ + "base64", + "js-sys", + "pem", + "ring", + "serde", + "serde_json", + "simple_asn1", +] + [[package]] name = "lazy_static" version = "1.5.0" @@ -1146,26 +1037,6 @@ version = "0.4.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" -[[package]] -name = "mac" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4" - -[[package]] -name = "markup5ever" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a24f40fb03852d1cdd84330cddcaf98e9ec08a7b7768e952fad3b4cf048ec8fd" -dependencies = [ - "log", - "phf", - "phf_codegen", - "string_cache", - "string_cache_codegen", - "tendril", -] - [[package]] name = "matchers" version = "0.2.0" @@ -1175,12 +1046,6 @@ dependencies = [ "regex-automata", ] -[[package]] -name = "matches" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2532096657941c2fea9c289d370a250971c689d4f143798ff67113ec042024a5" - [[package]] name = "matchit" version = "0.7.3" @@ -1216,7 +1081,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "02bd0af71c67b473010cbbc60715ee815645a4dc942899111f494b4b737d6fda" dependencies = [ "libc", - "wasi 0.11.1+wasi-snapshot-preview1", + "wasi", "windows-sys 0.61.2", ] @@ -1237,18 +1102,6 @@ dependencies = [ "tempfile", ] -[[package]] -name = "new_debug_unreachable" -version = "1.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086" - -[[package]] -name = "nodrop" -version = "0.1.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72ef4a56884ca558e5ddb05a1d1e7e1bfd9a68d9ed024c21704cc98872dae1bb" - [[package]] name = "nu-ansi-term" version = "0.50.3" @@ -1258,6 +1111,31 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "num-bigint" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-conv" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" + +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -1279,7 +1157,7 @@ version = "0.10.81" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77823a27f0babb03091cb9ed9ef80af3b39dbc82f97e8fa530374b7dafd87a45" dependencies = [ - "bitflags 2.13.0", + "bitflags", "cfg-if", "foreign-types", "libc", @@ -1295,7 +1173,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn", ] [[package]] @@ -1345,85 +1223,22 @@ dependencies = [ "windows-link", ] +[[package]] +name = "pem" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d30c53c26bc5b31a98cd02d20f25a7c8567146caf63ed593a9d87b2775291be" +dependencies = [ + "base64", + "serde_core", +] + [[package]] name = "percent-encoding" version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" -[[package]] -name = "phf" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3dfb61232e34fcb633f43d12c58f83c1df82962dcdfa565a4e866ffc17dafe12" -dependencies = [ - "phf_macros", - "phf_shared 0.8.0", - "proc-macro-hack", -] - -[[package]] -name = "phf_codegen" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cbffee61585b0411840d3ece935cce9cb6321f01c45477d30066498cd5e1a815" -dependencies = [ - "phf_generator 0.8.0", - "phf_shared 0.8.0", -] - -[[package]] -name = "phf_generator" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17367f0cc86f2d25802b2c26ee58a7b23faeccf78a396094c13dced0d0182526" -dependencies = [ - "phf_shared 0.8.0", - "rand 0.7.3", -] - -[[package]] -name = "phf_generator" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d" -dependencies = [ - "phf_shared 0.11.3", - "rand 0.8.6", -] - -[[package]] -name = "phf_macros" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f6fde18ff429ffc8fe78e2bf7f8b7a5a5a6e2a8b58bc5a9ac69198bbda9189c" -dependencies = [ - "phf_generator 0.8.0", - "phf_shared 0.8.0", - "proc-macro-hack", - "proc-macro2", - "quote", - "syn 1.0.109", -] - -[[package]] -name = "phf_shared" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c00cf8b9eafe68dde5e9eaa2cef8ee84a9336a47d566ec55ca16589633b65af7" -dependencies = [ - "siphasher 0.3.11", -] - -[[package]] -name = "phf_shared" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67eabc2ef2a60eb7faa00097bd1ffdb5bd28e62bf39990626a582201b7a754e5" -dependencies = [ - "siphasher 1.0.3", -] - [[package]] name = "pin-project-lite" version = "0.2.17" @@ -1445,6 +1260,12 @@ dependencies = [ "zerovec", ] +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + [[package]] name = "ppv-lite86" version = "0.2.21" @@ -1454,18 +1275,6 @@ dependencies = [ "zerocopy", ] -[[package]] -name = "precomputed-hash" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c" - -[[package]] -name = "proc-macro-hack" -version = "0.5.20+deprecated" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068" - [[package]] name = "proc-macro2" version = "1.0.106" @@ -1490,20 +1299,6 @@ version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" -[[package]] -name = "rand" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" -dependencies = [ - "getrandom 0.1.16", - "libc", - "rand_chacha 0.2.2", - "rand_core 0.5.1", - "rand_hc", - "rand_pcg", -] - [[package]] name = "rand" version = "0.8.6" @@ -1511,7 +1306,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5ca0ecfa931c29007047d1bc58e623ab12e5590e8c7cc53200d5202b69266d8a" dependencies = [ "libc", - "rand_chacha 0.3.1", + "rand_chacha", "rand_core 0.6.4", ] @@ -1526,16 +1321,6 @@ dependencies = [ "rand_core 0.10.1", ] -[[package]] -name = "rand_chacha" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402" -dependencies = [ - "ppv-lite86", - "rand_core 0.5.1", -] - [[package]] name = "rand_chacha" version = "0.3.1" @@ -1546,15 +1331,6 @@ dependencies = [ "rand_core 0.6.4", ] -[[package]] -name = "rand_core" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" -dependencies = [ - "getrandom 0.1.16", -] - [[package]] name = "rand_core" version = "0.6.4" @@ -1570,31 +1346,13 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" -[[package]] -name = "rand_hc" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c" -dependencies = [ - "rand_core 0.5.1", -] - -[[package]] -name = "rand_pcg" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16abd0c1b639e9eb4d7c50c0b8100b0d0f849be2349829c740fe8e6eb4816429" -dependencies = [ - "rand_core 0.5.1", -] - [[package]] name = "redox_syscall" version = "0.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" dependencies = [ - "bitflags 2.13.0", + "bitflags", ] [[package]] @@ -1670,22 +1428,13 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "rustc_version" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" -dependencies = [ - "semver", -] - [[package]] name = "rustix" version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" dependencies = [ - "bitflags 2.13.0", + "bitflags", "errno", "libc", "linux-raw-sys", @@ -1752,29 +1501,13 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" -[[package]] -name = "scraper" -version = "0.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48e02aa790c80c2e494130dec6a522033b6a23603ffc06360e9fe6c611ea2c12" -dependencies = [ - "cssparser", - "ego-tree", - "getopts", - "html5ever", - "matches", - "selectors", - "smallvec", - "tendril", -] - [[package]] name = "security-framework" version = "3.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" dependencies = [ - "bitflags 2.13.0", + "bitflags", "core-foundation 0.10.1", "core-foundation-sys", "libc", @@ -1791,32 +1524,6 @@ dependencies = [ "libc", ] -[[package]] -name = "selectors" -version = "0.22.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df320f1889ac4ba6bc0cdc9c9af7af4bd64bb927bccdf32d81140dc1f9be12fe" -dependencies = [ - "bitflags 1.3.2", - "cssparser", - "derive_more", - "fxhash", - "log", - "matches", - "phf", - "phf_codegen", - "precomputed-hash", - "servo_arc", - "smallvec", - "thin-slice", -] - -[[package]] -name = "semver" -version = "1.0.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" - [[package]] name = "serde" version = "1.0.228" @@ -1844,7 +1551,7 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn", ] [[package]] @@ -1853,7 +1560,7 @@ version = "1.0.150" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" dependencies = [ - "itoa 1.0.18", + "itoa", "memchr", "serde", "serde_core", @@ -1866,7 +1573,7 @@ version = "0.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457" dependencies = [ - "itoa 1.0.18", + "itoa", "serde", "serde_core", ] @@ -1878,21 +1585,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" dependencies = [ "form_urlencoded", - "itoa 1.0.18", + "itoa", "ryu", "serde", ] -[[package]] -name = "servo_arc" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d98238b800e0d1576d8b6e3de32827c2d74bee68bb97748dcf5071fb53965432" -dependencies = [ - "nodrop", - "stable_deref_trait", -] - [[package]] name = "sha1" version = "0.11.0" @@ -1952,16 +1649,16 @@ dependencies = [ ] [[package]] -name = "siphasher" -version = "0.3.11" +name = "simple_asn1" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38b58827f4464d87d377d175e90bf58eb00fd8716ff0a62f80356b5e61555d0d" - -[[package]] -name = "siphasher" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ee5873ec9cce0195efcb7a4e9507a04cd49aec9c83d0389df45b1ef7ba2e649" +checksum = "0d585997b0ac10be3c5ee635f1bab02d512760d14b7c468801ac8a01d9ae5f1d" +dependencies = [ + "num-bigint", + "num-traits", + "thiserror", + "time", +] [[package]] name = "slab" @@ -2056,7 +1753,7 @@ dependencies = [ "quote", "sqlx-core", "sqlx-macros-core", - "syn 2.0.118", + "syn", ] [[package]] @@ -2079,7 +1776,7 @@ dependencies = [ "sqlx-mysql", "sqlx-postgres", "sqlx-sqlite", - "syn 2.0.118", + "syn", "thiserror", "tokio", "url", @@ -2091,7 +1788,7 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "90b8020fe17c5f2c245bfa2505d7ef59c5604839527c740266ad2214acebea27" dependencies = [ - "bitflags 2.13.0", + "bitflags", "byteorder", "bytes", "chrono", @@ -2121,7 +1818,7 @@ checksum = "87a2bdd6e83f6b3ea525ca9fee568030508b58355a43d0b2c1674d5f79dcd65e" dependencies = [ "atoi", "base64", - "bitflags 2.13.0", + "bitflags", "byteorder", "chrono", "crc", @@ -2133,7 +1830,7 @@ dependencies = [ "hex", "hkdf", "hmac", - "itoa 1.0.18", + "itoa", "log", "md-5", "memchr", @@ -2182,31 +1879,6 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" -[[package]] -name = "string_cache" -version = "0.8.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf776ba3fa74f83bf4b63c3dcbbf82173db2632ed8452cb2d891d33f459de70f" -dependencies = [ - "new_debug_unreachable", - "parking_lot", - "phf_shared 0.11.3", - "precomputed-hash", - "serde", -] - -[[package]] -name = "string_cache_codegen" -version = "0.5.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c711928715f1fe0fe509c53b43e993a9a557babc2d0a3567d0a3006f1ac931a0" -dependencies = [ - "phf_generator 0.11.3", - "phf_shared 0.11.3", - "proc-macro2", - "quote", -] - [[package]] name = "stringprep" version = "0.1.5" @@ -2224,17 +1896,6 @@ version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" -[[package]] -name = "syn" -version = "1.0.109" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - [[package]] name = "syn" version = "2.0.118" @@ -2263,7 +1924,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn", ] [[package]] @@ -2272,7 +1933,7 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" dependencies = [ - "bitflags 2.13.0", + "bitflags", "core-foundation 0.9.4", "system-configuration-sys", ] @@ -2300,23 +1961,6 @@ dependencies = [ "windows-sys 0.61.2", ] -[[package]] -name = "tendril" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d24a120c5fc464a3458240ee02c299ebcb9d67b5249c8848b09d639dca8d7bb0" -dependencies = [ - "futf", - "mac", - "utf-8", -] - -[[package]] -name = "thin-slice" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8eaa81235c7058867fa8c0e7314f33dcce9c215f535d1913822a2b3f5e289f3c" - [[package]] name = "thiserror" version = "2.0.18" @@ -2334,7 +1978,7 @@ checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn", ] [[package]] @@ -2346,6 +1990,36 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "time" +version = "0.3.51" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85c17d80feb7334b40c484e45ed1a5273dfd8bfda537c3be2e74a06a6686f327" +dependencies = [ + "deranged", + "num-conv", + "powerfmt", + "serde_core", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" + +[[package]] +name = "time-macros" +version = "0.2.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcef1a61bdb119096e153208ec5cbec23944ce8bca13be5c7f60c634f7403935" +dependencies = [ + "num-conv", + "time-core", +] + [[package]] name = "tinystr" version = "0.8.3" @@ -2380,7 +2054,6 @@ dependencies = [ "bytes", "libc", "mio", - "parking_lot", "pin-project-lite", "signal-hook-registry", "socket2", @@ -2396,7 +2069,7 @@ checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn", ] [[package]] @@ -2465,7 +2138,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e9cd434a998747dd2c4276bc96ee2e0c7a2eadf3cae88e52be55a05fa9053f5" dependencies = [ - "bitflags 2.13.0", + "bitflags", "bytes", "http", "http-body", @@ -2481,7 +2154,7 @@ version = "0.6.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" dependencies = [ - "bitflags 2.13.0", + "bitflags", "bytes", "futures-util", "http", @@ -2525,7 +2198,7 @@ checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn", ] [[package]] @@ -2606,12 +2279,6 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7df058c713841ad818f1dc5d3fd88063241cc61f49f5fbea4b951e8cf5a8d71d" -[[package]] -name = "unicode-width" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" - [[package]] name = "untrusted" version = "0.9.0" @@ -2630,12 +2297,6 @@ dependencies = [ "serde", ] -[[package]] -name = "utf-8" -version = "0.7.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" - [[package]] name = "utf8_iter" version = "1.0.4" @@ -2681,12 +2342,6 @@ dependencies = [ "try-lock", ] -[[package]] -name = "wasi" -version = "0.9.0+wasi-snapshot-preview1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" - [[package]] name = "wasi" version = "0.11.1+wasi-snapshot-preview1" @@ -2735,7 +2390,7 @@ dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn 2.0.118", + "syn", "wasm-bindgen-shared", ] @@ -2785,7 +2440,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn", ] [[package]] @@ -2796,7 +2451,7 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn", ] [[package]] @@ -2941,7 +2596,7 @@ checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn", "synstructure", ] @@ -2962,7 +2617,7 @@ checksum = "1ae7f38b72ec2a254e2b87ef277cf2cd4fb97cbebf944faa6f33354da0867930" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn", ] [[package]] @@ -2982,7 +2637,7 @@ checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn", "synstructure", ] @@ -3022,7 +2677,7 @@ checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn", ] [[package]] diff --git a/frontend_old/rust-ai/Cargo.toml b/frontend_old/rust-ai/Cargo.toml index 3e8583a..e818717 100644 --- a/frontend_old/rust-ai/Cargo.toml +++ b/frontend_old/rust-ai/Cargo.toml @@ -6,7 +6,7 @@ description = "AI Sales Assistant backend for Coast IT CRM" [dependencies] axum = "0.7" -tokio = { version = "1", features = ["full"] } +tokio = { version = "1", features = ["rt-multi-thread", "macros", "sync", "time", "net", "process"] } reqwest = { version = "0.12", features = ["json", "blocking"] } serde = { version = "1", features = ["derive"] } serde_json = "1" @@ -17,5 +17,5 @@ tracing = "0.1" tracing-subscriber = { version = "0.3", features = ["env-filter"] } tower-http = { version = "0.5", features = ["cors"] } dotenvy = "0.15" -scraper = "0.12" -rand = "0.8" \ No newline at end of file +rand = "0.8" +jsonwebtoken = "9" \ No newline at end of file diff --git a/frontend_old/rust-ai/src/instructions.rs b/frontend_old/rust-ai/src/instructions.rs deleted file mode 100644 index 00fc80d..0000000 --- a/frontend_old/rust-ai/src/instructions.rs +++ /dev/null @@ -1,85 +0,0 @@ -use std::fs; -use std::path::PathBuf; -use std::sync::Arc; -use tokio::sync::RwLock; -use tracing::{error, info, warn}; - -/// Manages the ai.md self-improvement file. -/// Loaded on startup and periodically refreshed. -pub struct InstructionsManager { - path: PathBuf, - current: RwLock, -} - -impl InstructionsManager { - pub fn new(path: PathBuf) -> Self { - let initial = fs::read_to_string(&path).unwrap_or_default(); - if initial.is_empty() { - warn!("ai.md is empty or missing at {:?}", path); - } else { - info!("Loaded ai.md ({} bytes)", initial.len()); - } - Self { - path, - current: RwLock::new(initial), - } - } - - /// Read the current instructions - pub async fn get(&self) -> String { - self.current.read().await.clone() - } - - /// Append a new entry to the Improvement Log and optionally update the instructions. - /// Returns the updated full content. - pub async fn update(&self, entry: &str, new_content: Option<&str>) -> Result { - if let Some(content) = new_content { - // Full replacement - fs::write(&self.path, content).map_err(|e| format!("Write failed: {}", e))?; - let mut current = self.current.write().await; - *current = content.to_string(); - info!("ai.md fully replaced ({} bytes)", content.len()); - Ok(content.to_string()) - } else { - // Append to Improvement Log only - let mut current = self.current.write().await; - let log_entry = format!("\n- {} — {}", chrono::Utc::now().format("%Y-%m-%d %H:%M"), entry); - // Find the Improvement Log section and append - if let Some(pos) = current.rfind("\n## Improvement Log") { - // Find the next section after Improvement Log, or end of file - let after_log = ¤t[pos..]; - if let Some(section_start) = after_log[1..].find("\n## ") { - let insert_at = pos + 1 + section_start; - current.insert_str(insert_at, &format!("{}\n", log_entry)); - } else { - current.push_str(&format!("{}\n", log_entry)); - } - } else { - current.push_str(&format!("\n## Improvement Log\n{}", log_entry)); - } - fs::write(&self.path, &*current).map_err(|e| format!("Write failed: {}", e))?; - info!("ai.md improvement log appended: {}", entry); - Ok(current.clone()) - } - } - - /// Reload from disk - pub async fn reload(&self) { - match fs::read_to_string(&self.path) { - Ok(content) => { - let len = content.len(); - let mut current = self.current.write().await; - *current = content; - info!("ai.md reloaded from disk ({} bytes)", len); - } - Err(e) => error!("Failed to reload ai.md: {}", e), - } - } -} - -/// Wrapper for thread-safe sharing -pub type SharedInstructions = Arc; - -pub fn create_shared(path: PathBuf) -> SharedInstructions { - Arc::new(InstructionsManager::new(path)) -} diff --git a/frontend_old/rust-ai/src/main.rs b/frontend_old/rust-ai/src/main.rs index 4c9fdfe..174c1f5 100644 --- a/frontend_old/rust-ai/src/main.rs +++ b/frontend_old/rust-ai/src/main.rs @@ -1,93 +1,67 @@ use axum::{ extract::State, - http::Method, + http::{HeaderMap, Method, StatusCode}, routing::{get, post}, Json, Router, }; +use tower_http::cors::{CorsLayer, AllowOrigin, Any}; +use jsonwebtoken::{decode, DecodingKey, Validation, Algorithm}; use serde::{Deserialize, Serialize}; use sqlx::postgres::PgPoolOptions; +use std::collections::HashMap; use std::fs; -use std::sync::{Arc, Mutex}; -use tower_http::cors::{Any, CorsLayer}; -use tracing::{error, info}; +use std::sync::Arc; +use tokio::sync::Mutex; +use tracing::{error, info, warn}; use uuid::Uuid; -use reqwest::blocking::Client; -use scraper::{Html, Selector}; -use rand::rngs::StdRng; -use rand::SeedableRng; use rand::Rng; use std::time::Duration; -use std::thread; +use std::time::{SystemTime, UNIX_EPOCH}; -// ── Facebook Scraper ─────────────────────────────────────────── +// ── JWT Claims ──────────────────────────────────────────────── -// List of proxies -const PROXIES: &[&str] = &[ - "http://user:pass@192.168.1.1:8080", - "http://user:pass@192.168.1.2:8080", - "http://user:pass@192.168.1.3:8080", - "http://user:pass@192.168.1.4:8080", - "http://user:pass@192.168.1.5:8080", -]; - -// List of user agents -const USER_AGENTS: &[&str] = &[ - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3", - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.1.2 Safari/605.1.15", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:61.0) Gecko/20100101 Firefox/61.0", - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.1.2 Safari/605.1.15", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36", - // Add more user agents here -]; - -fn get_random_proxy() -> String { - let mut rng = rand::thread_rng(); - PROXIES[rng.gen_range(0..PROXIES.len())].to_string() +#[derive(Debug, Deserialize)] +struct Claims { + #[serde(rename = "userId")] + user_id: String, + role: String, } -fn get_random_user_agent() -> String { - let mut rng = rand::thread_rng(); - USER_AGENTS[rng.gen_range(0..USER_AGENTS.len())].to_string() +fn verify_jwt(token: &str, secret: &str) -> Option { + let key = DecodingKey::from_secret(secret.as_bytes()); + let validation = Validation::new(Algorithm::HS256); + decode::(token, &key, &validation).ok().map(|d| d.claims) } -fn scrape_conversations(url: &str) -> Result> { - let proxy = get_random_proxy(); - let user_agent = get_random_user_agent(); - let client = Client::builder() - .proxy(reqwest::Proxy::all(proxy)?) - .build()?; +// ── Rate limiter ────────────────────────────────────────────── - let response = client.get(url) - .header("User-Agent", user_agent) - .send()?; - - if response.status().is_success() { - let html = response.text()?; - Ok(Html::parse_document(&html)) - } else { - Err(format!("Failed to retrieve the page: {}", response.status()).into()) - } +struct RateLimiter { + buckets: Mutex>>, + max_requests: usize, + window_secs: u64, } -fn run_facebook_scraper() { - let url = "https://www.facebook.com/search/top/?q=need%20website%20create"; - match scrape_conversations(url) { - Ok(soup) => { - // Example: Extract conversation data - let selector = Selector::parse("div.conversation").unwrap(); - for element in soup.select(&selector) { - println!("Conversation: {}", element.inner_html()); - } - } - Err(e) => { - error!("Facebook scraper error: {}", e); +impl RateLimiter { + fn new(max_requests: usize, window_secs: u64) -> Self { + Self { + buckets: Mutex::new(HashMap::new()), + max_requests, + window_secs, } } - // Introduce random delay - let mut rng = rand::thread_rng(); - let delay = rng.gen_range(1..5); // Delay between 1 to 5 seconds - thread::sleep(Duration::from_secs(delay)); + async fn check(&self, key: &str) -> bool { + let now = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs(); + let mut buckets = self.buckets.lock().await; + let timestamps = buckets.entry(key.to_string()).or_default(); + timestamps.retain(|t| now - *t <= self.window_secs); + if timestamps.len() >= self.max_requests { + false + } else { + timestamps.push(now); + true + } + } } // ── Shared state ─────────────────────────────────────────────── @@ -97,6 +71,48 @@ struct AppState { ollama_url: String, model: String, jobs: Vec, + leads: Arc>, + http_client: reqwest::Client, + jwt_secret: String, + rate_limiter: RateLimiter, +} + +#[derive(Debug, Clone, Serialize)] +struct Lead { + title: String, + url: String, + source: String, + found_at: u64, + author: String, + date: String, + content: String, +} + +struct LeadStore { + leads: Vec, + max_size: usize, +} + +impl LeadStore { + fn new(max_size: usize) -> Self { + Self { leads: Vec::new(), max_size } + } + + fn push(&mut self, lead: Lead) { + if !self.leads.iter().any(|l| l.url == lead.url) { + self.leads.insert(0, lead); + self.leads.truncate(self.max_size); + } + } + + fn recent(&self, max_age_secs: u64, limit: usize) -> Vec { + let now = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs(); + self.leads.iter() + .filter(|l| now.saturating_sub(l.found_at) <= max_age_secs) + .take(limit) + .cloned() + .collect() + } } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -110,8 +126,6 @@ struct Job { #[derive(Debug, Deserialize)] struct ChatRequest { message: String, - user_id: String, - user_role: String, } #[derive(Debug, Serialize)] @@ -162,23 +176,73 @@ struct OllamaResponseMessage { content: String, } -// ── System prompt builder ───────────────────────────────────── +// ── Helpers ──────────────────────────────────────────────────── -fn build_system_prompt(jobs: &[Job]) -> String { +fn truncate(s: &str, max: usize) -> String { + s.chars().take(max).collect() +} + +fn extract_claims(headers: &HeaderMap, state: &AppState) -> Result { + let auth_header = headers.get("Authorization").and_then(|v| v.to_str().ok()).unwrap_or(""); + let token = auth_header.strip_prefix("Bearer ").unwrap_or(""); + let claims = verify_jwt(token, &state.jwt_secret).ok_or_else(|| { + (StatusCode::UNAUTHORIZED, "Unauthorized".to_string()) + })?; + match claims.role.as_str() { + "sales" | "admin" | "super_admin" => Ok(claims), + _ => Err((StatusCode::FORBIDDEN, "Forbidden".to_string())), + } +} + +fn format_leads_output(leads: &[Lead]) -> String { + if leads.is_empty() { + return "No new requests found yet.".to_string(); + } + leads + .iter() + .enumerate() + .map(|(i, l)| { + let author = if l.author.is_empty() { "Unknown" } else { &l.author }; + let date = truncate(&l.date, 10); + format!( + "{}. {}\n {}\n {}\n {}", + i + 1, + author, + date, + l.title, + l.url + ) + }) + .collect::>() + .join("\n") +} + +fn build_system_prompt(jobs: &[Job], leads: &[Lead]) -> String { let job_list: Vec = jobs .iter() .map(|j| format!("- {} ({}): {}", j.job_title, j.industry, j.description)) .collect(); let job_list_str = job_list.join("\n"); + let lead_summary: Vec = leads + .iter() + .map(|l| format!("{} | {} | {}", l.author, l.title, l.url)) + .collect(); + let lead_summary_str = if lead_summary.is_empty() { + "None yet.".to_string() + } else { + lead_summary.join("\n") + }; + format!( - "You are a Sales AI Assistant for Coast IT CRM. Your role is to help salespeople \ - with tips, strategies, and guidance.\n\n\ + "You are a Sales AI Assistant for Coast IT CRM.\n\n\ Available job categories to target:\n{}\n\n\ - Provide concise, actionable sales advice. When asked about a specific job category, \ - give targeted tips on finding and engaging prospects in that field. \ - Keep responses under 300 words unless asked for detail.", - job_list_str + Recent leads context:\n{}\n\n\ + Rules:\n\ + - When asked about leads, answer concisely under 150 words.\n\ + - If asked to suggest a sales strategy, give brief actionable advice.\n\ + - Be direct and professional. No fluff.", + job_list_str, lead_summary_str ) } @@ -186,90 +250,119 @@ fn build_system_prompt(jobs: &[Job]) -> String { async fn handle_chat( State(state): State>, + headers: HeaderMap, Json(req): Json, -) -> Result, (axum::http::StatusCode, String)> { - // Validate role - match req.user_role.as_str() { - "sales" | "admin" | "super_admin" => {} - _ => return Err((axum::http::StatusCode::FORBIDDEN, "Forbidden".to_string())), +) -> Result, (StatusCode, String)> { + let claims = extract_claims(&headers, &state)?; + + if !state.rate_limiter.check(&claims.user_id).await { + return Err((StatusCode::TOO_MANY_REQUESTS, "Rate limit exceeded".to_string())); } - let system_prompt = build_system_prompt(&state.jobs); + let msg_lower = req.message.to_lowercase(); + let msg_words: Vec<&str> = msg_lower.split_whitespace().collect(); + let has_listing = msg_words.iter().any(|w| ["listings", "listing", "leads", "links", "lists"].contains(w)); + let has_show = msg_lower.contains("show me") || msg_lower.contains("give me") || msg_lower.contains("pull"); + let has_job = msg_words.contains(&"jobs") || msg_words.contains(&"job"); + if has_listing || (has_show && has_job) || (has_show && msg_lower.contains("links")) || msg_lower.contains("recent leads") { + let now = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs(); + let service_url = "http://localhost:3008/scrape/all".to_string(); + if let Ok(resp) = state.http_client + .post(&service_url) + .timeout(Duration::from_secs(120)) + .send() + .await + { + if let Ok(leads_data) = resp.json::>().await { + info!("Scraped {} leads from {}", leads_data.len(), service_url); + let mut store = state.leads.lock().await; + for item in &leads_data { + store.push(Lead { + title: truncate(item["title"].as_str().unwrap_or(""), 120), + url: item["url"].as_str().unwrap_or("").to_string(), + source: item["source"].as_str().unwrap_or("unknown").to_string(), + found_at: now, + author: truncate(item["author"].as_str().unwrap_or(""), 60), + date: truncate(item["date"].as_str().unwrap_or(""), 30), + content: truncate(item["content"].as_str().unwrap_or(""), 300), + }); + } + } + } else { + error!("Scraper service unreachable: {}", service_url); + } + let recent_leads = state.leads.lock().await.recent(604800, 20); + let response = format_leads_output(&recent_leads); + let _ = sqlx::query( + "INSERT INTO ai_conversations (id, user_id, role, message, response) VALUES ($1, $2, $3, $4, $5)", + ) + .bind(Uuid::new_v4()) + .bind(Uuid::parse_str(&claims.user_id).unwrap_or(Uuid::nil())) + .bind(&claims.role) + .bind(&req.message) + .bind(&response) + .execute(&state.db) + .await; + return Ok(Json(ChatResponse { response })); + } + + let recent_leads = state.leads.lock().await.recent(604800, 20); + let system_prompt = build_system_prompt(&state.jobs, &recent_leads); let message_text = req.message.clone(); let ollama_req = OllamaRequest { model: state.model.clone(), messages: vec![ - OllamaChatMessage { - role: "system".to_string(), - content: system_prompt, - }, - OllamaChatMessage { - role: "user".to_string(), - content: req.message.clone(), - }, + OllamaChatMessage { role: "system".to_string(), content: system_prompt }, + OllamaChatMessage { role: "user".to_string(), content: req.message.clone() }, ], stream: false, - options: OllamaOptions { - temperature: 0.7, - num_predict: 1024, - }, + options: OllamaOptions { temperature: 0.7, num_predict: 1024 }, }; - let client = reqwest::Client::new(); - let resp = client + let resp = state.http_client .post(format!("{}/api/chat", state.ollama_url)) .json(&ollama_req) .send() .await .map_err(|e| { error!("Ollama request failed: {}", e); - ( - axum::http::StatusCode::SERVICE_UNAVAILABLE, - "AI service unavailable".to_string(), - ) + (StatusCode::SERVICE_UNAVAILABLE, "AI service unavailable".to_string()) })?; let ollama_resp: OllamaResponse = resp.json().await.map_err(|e| { error!("Failed to parse Ollama response: {}", e); - ( - axum::http::StatusCode::SERVICE_UNAVAILABLE, - "AI response parse error".to_string(), - ) + (StatusCode::SERVICE_UNAVAILABLE, "AI response parse error".to_string()) })?; - let response_text = ollama_resp - .message - .map(|m| m.content) - .unwrap_or_default(); + let response_text = ollama_resp.message.map(|m| m.content).unwrap_or_default(); - // Store in database - let user_id = Uuid::parse_str(&req.user_id).unwrap_or(Uuid::nil()); let _ = sqlx::query( "INSERT INTO ai_conversations (id, user_id, role, message, response) VALUES ($1, $2, $3, $4, $5)", ) .bind(Uuid::new_v4()) - .bind(user_id) - .bind(&req.user_role) + .bind(Uuid::parse_str(&claims.user_id).unwrap_or(Uuid::nil())) + .bind(&claims.role) .bind(&message_text) .bind(&response_text) .execute(&state.db) .await; - Ok(Json(ChatResponse { - response: response_text, - })) + Ok(Json(ChatResponse { response: response_text })) } // ── Jobs handler ─────────────────────────────────────────────── async fn handle_jobs( State(state): State>, -) -> Json { - Json(JobsResponse { - jobs: state.jobs.clone(), - }) + headers: HeaderMap, +) -> Result, (StatusCode, String)> { + let _claims = extract_claims(&headers, &state)?; + if !state.rate_limiter.check(&_claims.user_id).await { + return Err((StatusCode::TOO_MANY_REQUESTS, "Rate limit exceeded".to_string())); + } + Ok(Json(JobsResponse { jobs: state.jobs.clone() })) } // ── Health handler ───────────────────────────────────────────── @@ -296,52 +389,50 @@ async fn main() { dotenvy::dotenv().ok(); - let database_url = - std::env::var("DATABASE_URL").expect("DATABASE_URL must be set"); - let ollama_url = - std::env::var("OLLAMA_BASE_URL").unwrap_or_else(|_| "http://localhost:11434".to_string()); - let model = std::env::var("AI_MODEL").unwrap_or_else(|_| "sam860/dolphin3-llama3.2:3b".to_string()); - let host = std::env::var("AI_HOST").unwrap_or_else(|_| "0.0.0.0".to_string()); - let port: u16 = std::env::var("AI_PORT") - .unwrap_or_else(|_| "3001".to_string()) - .parse() - .expect("AI_PORT must be a number"); + let database_url = std::env::var("DATABASE_URL").expect("DATABASE_URL must be set"); + let jwt_secret = std::env::var("JWT_SECRET").expect("JWT_SECRET must be set"); + let ollama_url = std::env::var("OLLAMA_BASE_URL").unwrap_or_else(|_| "http://localhost:11434".to_string()); + let model = std::env::var("AI_MODEL").unwrap_or_else(|_| "dolphin-phi".to_string()); + let host = std::env::var("AI_HOST").unwrap_or_else(|_| "127.0.0.1".to_string()); + let port: u16 = std::env::var("AI_PORT").unwrap_or_else(|_| "3001".to_string()).parse().expect("AI_PORT must be a number"); - // Load jobs let jobs_path = std::env::var("JOBS_PATH").unwrap_or_else(|_| "data/ai/jobs.jsonl".to_string()); let jobs_content = fs::read_to_string(&jobs_path).unwrap_or_default(); - let jobs: Vec = jobs_content - .lines() - .filter(|l| !l.trim().is_empty()) - .filter_map(|l| serde_json::from_str(l).ok()) - .collect(); + let jobs: Vec = jobs_content.lines().filter(|l| !l.trim().is_empty()).filter_map(|l| serde_json::from_str(l).ok()).collect(); - info!( - "Loaded {} job categories, model: {}, Ollama: {}", - jobs.len(), - model, - ollama_url - ); + info!("Loaded {} job categories, model: {}, Ollama: {}", jobs.len(), model, ollama_url); - // Connect to database let db = PgPoolOptions::new() - .max_connections(5) + .max_connections(20) .connect(&database_url) .await .expect("Failed to connect to database"); info!("Connected to PostgreSQL"); + let http_client = reqwest::Client::builder() + .timeout(Duration::from_secs(120)) + .build() + .expect("Failed to build HTTP client"); + + let lead_store = Arc::new(Mutex::new(LeadStore::new(100))); + let state = Arc::new(AppState { db, ollama_url, model, jobs, + leads: lead_store.clone(), + http_client, + jwt_secret, + rate_limiter: RateLimiter::new(30, 60), }); - // CORS layer let cors = CorsLayer::new() - .allow_origin(Any) + .allow_origin(AllowOrigin::list([ + "http://localhost:3006".parse().unwrap(), + "http://127.0.0.1:3006".parse().unwrap(), + ])) .allow_methods([Method::GET, Method::POST]) .allow_headers(Any); @@ -359,24 +450,57 @@ async fn main() { .await .expect("Failed to bind address"); - // Start Facebook scraper in a separate thread - let rng = Arc::new(Mutex::new(StdRng::from_entropy())); - tokio::task::spawn_blocking(move || { - loop { - { - let _lock = rng.lock().unwrap(); - run_facebook_scraper(); + let bg_leads = lead_store.clone(); + let bg_url = "http://localhost:3008/scrape/all".to_string(); + tokio::spawn(async move { + let client = match reqwest::Client::builder() + .timeout(Duration::from_secs(120)) + .build() + { + Ok(c) => c, + Err(e) => { + error!("Failed to build background HTTP client: {} — scraper disabled", e); + return; } - // Introduce random delay - let delay = { - let mut rng = rng.lock().unwrap(); - rng.gen_range(1..5) - }; - thread::sleep(Duration::from_secs(delay)); + }; + loop { + let now = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs(); + match client.post(&bg_url).send().await { + Ok(resp) => { + if resp.status().is_success() { + match resp.json::>().await { + Ok(data) => { + let mut store = bg_leads.lock().await; + for item in &data { + store.push(Lead { + title: truncate(item["title"].as_str().unwrap_or(""), 120), + url: item["url"].as_str().unwrap_or("").to_string(), + source: item["source"].as_str().unwrap_or("unknown").to_string(), + found_at: now, + author: truncate(item["author"].as_str().unwrap_or(""), 60), + date: truncate(item["date"].as_str().unwrap_or(""), 30), + content: truncate(item["content"].as_str().unwrap_or(""), 300), + }); + } + } + Err(e) => { + warn!("Failed to parse scraper JSON: {}", e); + } + } + } else { + warn!("Scraper returned status: {}", resp.status()); + } + } + Err(e) => { + warn!("Scraper request failed: {}", e); + } + } + let delay = rand::thread_rng().gen_range(120..300); + tokio::time::sleep(Duration::from_secs(delay)).await; } }); axum::serve(listener, app) .await .expect("Server failed"); -} \ No newline at end of file +} diff --git a/frontend_old/scraper-service/index.js b/frontend_old/scraper-service/index.js new file mode 100644 index 0000000..46565c9 --- /dev/null +++ b/frontend_old/scraper-service/index.js @@ -0,0 +1,100 @@ +const express = require('express'); +const { chromium } = require('playwright'); + +const app = express(); +app.use(express.json()); +const PORT = process.env.PORT || 3007; + +app.get('/health', (req, res) => { + res.json({ status: 'ok' }); +}); + +app.post('/scrape/indeed', async (req, res) => { + const results = []; + let browser; + try { + browser = await chromium.launch({ headless: true }); + const context = await browser.newContext({ + userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', + }); + + const queries = [ + 'need a website built', + 'looking for someone to build my website', + 'need web designer', + 'looking for web developer', + 'need help with my website', + 'need ecommerce website', + 'need wordpress website', + 'looking for website designer', + ]; + + for (const q of queries) { + const url = `https://www.indeed.com/jobs?q=${encodeURIComponent(q)}&sort=date`; + try { + const page = await context.newPage(); + await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 20000 }); + await page.waitForTimeout(2000); + + const items = await page.evaluate(() => { + const cards = document.querySelectorAll('.job_seen_beacon'); + return Array.from(cards).slice(0, 8).map(card => { + const titleEl = card.querySelector('.jcs-JobTitle'); + const companyEl = card.querySelector('[data-testid="inlineHeader-companyName"]'); + const snippetEl = card.querySelector('.job-snippet'); + return { + title: titleEl?.textContent?.trim() || '', + url: titleEl?.href || '', + company: companyEl?.textContent?.trim() || '', + snippet: snippetEl?.textContent?.trim()?.slice(0, 200) || '', + }; + }); + }); + + for (const item of items) { + if (item.title && item.url && !results.some(r => r.url === item.url)) { + results.push({ + title: item.title, + url: item.url, + author: item.company, + date: '', + content: item.snippet, + source: 'indeed', + }); + } + } + await page.close(); + } catch (e) { + console.error(`Indeed failed ${q}:`, e.message); + } + } + + await browser.close(); + res.json(results.slice(0, 50)); + } catch (e) { + if (browser) await browser.close().catch(() => {}); + console.error('Indeed error:', e.message); + res.status(500).json({ error: e.message }); + } +}); + +app.post('/scrape/all', async (req, res) => { + try { + const [indeed] = await Promise.allSettled([ + fetch(`http://localhost:${PORT}/scrape/indeed`, { method: 'POST' }).then(r => r.json()).catch(() => []), + ]); + + const all = [ + ...(indeed.status === 'fulfilled' ? indeed.value : []), + ]; + console.log(`Scrape all: ${all.length} leads`); + res.json(all); + } catch (e) { + console.error('Scrape all error:', e.message); + res.status(500).json({ error: e.message }); + } +}); + +app.listen(PORT, () => { + console.log(`Scraper service running on port ${PORT}`); +}); diff --git a/frontend_old/scraper-service/package-lock.json b/frontend_old/scraper-service/package-lock.json new file mode 100644 index 0000000..fa37c78 --- /dev/null +++ b/frontend_old/scraper-service/package-lock.json @@ -0,0 +1,872 @@ +{ + "name": "scraper-service", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "scraper-service", + "version": "1.0.0", + "dependencies": { + "express": "^4.18.2", + "playwright": "^1.48.0" + } + }, + "node_modules/accepts": { + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz", + "integrity": "sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==", + "license": "MIT", + "dependencies": { + "mime-types": "~2.1.34", + "negotiator": "0.6.3" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/array-flatten": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", + "integrity": "sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg==", + "license": "MIT" + }, + "node_modules/body-parser": { + "version": "1.20.5", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.5.tgz", + "integrity": "sha512-3grm+/2tUOvu2cjJkvsIxrv/wVpfXQW4PsQHYm7yk4vfpu7Ekl6nEsYBoJUL6qDwZUx8wUhQ8tR2qz+ad9c9OA==", + "license": "MIT", + "dependencies": { + "bytes": "~3.1.2", + "content-type": "~1.0.5", + "debug": "2.6.9", + "depd": "2.0.0", + "destroy": "~1.2.0", + "http-errors": "~2.0.1", + "iconv-lite": "~0.4.24", + "on-finished": "~2.4.1", + "qs": "~6.15.1", + "raw-body": "~2.5.3", + "type-is": "~1.6.18", + "unpipe": "~1.0.0" + }, + "engines": { + "node": ">= 0.8", + "npm": "1.2.8000 || >= 1.4.16" + } + }, + "node_modules/bytes": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", + "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/call-bind-apply-helpers": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", + "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/call-bound": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/call-bound/-/call-bound-1.0.4.tgz", + "integrity": "sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "get-intrinsic": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/content-disposition": { + "version": "0.5.4", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz", + "integrity": "sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==", + "license": "MIT", + "dependencies": { + "safe-buffer": "5.2.1" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/content-type": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.5.tgz", + "integrity": "sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/cookie": { + "version": "0.7.2", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.7.2.tgz", + "integrity": "sha512-yki5XnKuf750l50uGTllt6kKILY4nQ1eNIQatoXEByZ5dWgnKqbnqmTrBE5B4N7lrMJKQ2ytWMiTO2o0v6Ew/w==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/cookie-signature": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.7.tgz", + "integrity": "sha512-NXdYc3dLr47pBkpUCHtKSwIOQXLVn8dZEuywboCOJY/osA0wFSLlSawr3KN8qXJEyX66FcONTH8EIlVuK0yyFA==", + "license": "MIT" + }, + "node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "license": "MIT", + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/depd": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", + "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/destroy": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.2.0.tgz", + "integrity": "sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg==", + "license": "MIT", + "engines": { + "node": ">= 0.8", + "npm": "1.2.8000 || >= 1.4.16" + } + }, + "node_modules/dunder-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", + "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.1", + "es-errors": "^1.3.0", + "gopd": "^1.2.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/ee-first": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", + "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==", + "license": "MIT" + }, + "node_modules/encodeurl": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-2.0.0.tgz", + "integrity": "sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/es-define-property": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", + "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-errors": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", + "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-object-atoms": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.2.tgz", + "integrity": "sha512-HWcBoN6NileqtSydK2FqHbS/LoDd2pqrnQHLyJzBj4kOp/ky2MWMN694xOfkK8/SnUsW2DH7EfyVlydKCsm1Zw==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/escape-html": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", + "integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==", + "license": "MIT" + }, + "node_modules/etag": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", + "integrity": "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/express": { + "version": "4.22.2", + "resolved": "https://registry.npmjs.org/express/-/express-4.22.2.tgz", + "integrity": "sha512-IuL+Elrou2ZvCFHs18/CIzy2Nzvo25nZ1/D2eIZlz7c+QUayAcYoiM2BthCjs+EBHVpjYjcuLDAiCWgeIX3X1Q==", + "license": "MIT", + "dependencies": { + "accepts": "~1.3.8", + "array-flatten": "1.1.1", + "body-parser": "~1.20.5", + "content-disposition": "~0.5.4", + "content-type": "~1.0.4", + "cookie": "~0.7.1", + "cookie-signature": "~1.0.6", + "debug": "2.6.9", + "depd": "2.0.0", + "encodeurl": "~2.0.0", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "finalhandler": "~1.3.1", + "fresh": "~0.5.2", + "http-errors": "~2.0.0", + "merge-descriptors": "1.0.3", + "methods": "~1.1.2", + "on-finished": "~2.4.1", + "parseurl": "~1.3.3", + "path-to-regexp": "~0.1.12", + "proxy-addr": "~2.0.7", + "qs": "~6.15.1", + "range-parser": "~1.2.1", + "safe-buffer": "5.2.1", + "send": "~0.19.0", + "serve-static": "~1.16.2", + "setprototypeof": "1.2.0", + "statuses": "~2.0.1", + "type-is": "~1.6.18", + "utils-merge": "1.0.1", + "vary": "~1.1.2" + }, + "engines": { + "node": ">= 0.10.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/finalhandler": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.3.2.tgz", + "integrity": "sha512-aA4RyPcd3badbdABGDuTXCMTtOneUCAYH/gxoYRTZlIJdF0YPWuGqiAsIrhNnnqdXGswYk6dGujem4w80UJFhg==", + "license": "MIT", + "dependencies": { + "debug": "2.6.9", + "encodeurl": "~2.0.0", + "escape-html": "~1.0.3", + "on-finished": "~2.4.1", + "parseurl": "~1.3.3", + "statuses": "~2.0.2", + "unpipe": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/forwarded": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", + "integrity": "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/fresh": { + "version": "0.5.2", + "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", + "integrity": "sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/fsevents": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", + "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/function-bind": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-intrinsic": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", + "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "es-define-property": "^1.0.1", + "es-errors": "^1.3.0", + "es-object-atoms": "^1.1.1", + "function-bind": "^1.1.2", + "get-proto": "^1.0.1", + "gopd": "^1.2.0", + "has-symbols": "^1.1.0", + "hasown": "^2.0.2", + "math-intrinsics": "^1.1.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", + "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", + "license": "MIT", + "dependencies": { + "dunder-proto": "^1.0.1", + "es-object-atoms": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/gopd": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", + "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-symbols": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", + "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/hasown": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.4.tgz", + "integrity": "sha512-T2UbfbBEF32wiepXIsMlTW9+dDYC6wMh/t/vYA4tuOMKqWz/n3vr1NFSxQiyP+zk2mXsoMA/i/7qV6LKut1t1A==", + "license": "MIT", + "dependencies": { + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/http-errors": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz", + "integrity": "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==", + "license": "MIT", + "dependencies": { + "depd": "~2.0.0", + "inherits": "~2.0.4", + "setprototypeof": "~1.2.0", + "statuses": "~2.0.2", + "toidentifier": "~1.0.1" + }, + "engines": { + "node": ">= 0.8" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/iconv-lite": { + "version": "0.4.24", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", + "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "license": "ISC" + }, + "node_modules/ipaddr.js": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", + "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==", + "license": "MIT", + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/math-intrinsics": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", + "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/media-typer": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", + "integrity": "sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/merge-descriptors": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.3.tgz", + "integrity": "sha512-gaNvAS7TZ897/rVaZ0nMtAyxNyi/pdbjbAwUpFQpN70GqnVfOiXpeUUMKRBmzXaSQ8DdTX4/0ms62r2K+hE6mQ==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/methods": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz", + "integrity": "sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz", + "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==", + "license": "MIT", + "bin": { + "mime": "cli.js" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/mime-db": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types": { + "version": "2.1.35", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "license": "MIT", + "dependencies": { + "mime-db": "1.52.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", + "license": "MIT" + }, + "node_modules/negotiator": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", + "integrity": "sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/object-inspect": { + "version": "1.13.4", + "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.4.tgz", + "integrity": "sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/on-finished": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", + "integrity": "sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==", + "license": "MIT", + "dependencies": { + "ee-first": "1.1.1" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/parseurl": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", + "integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/path-to-regexp": { + "version": "0.1.13", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.13.tgz", + "integrity": "sha512-A/AGNMFN3c8bOlvV9RreMdrv7jsmF9XIfDeCd87+I8RNg6s78BhJxMu69NEMHBSJFxKidViTEdruRwEk/WIKqA==", + "license": "MIT" + }, + "node_modules/playwright": { + "version": "1.61.0", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.61.0.tgz", + "integrity": "sha512-Z+7BeeqQPRRzklHsVFP4KTGIyMxKUmfeRA4WisM6G3/XW6nwGeX6fX9qYaDa+CiUqpOkb2f6X3nar05R3kSuJQ==", + "license": "Apache-2.0", + "dependencies": { + "playwright-core": "1.61.0" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "fsevents": "2.3.2" + } + }, + "node_modules/playwright-core": { + "version": "1.61.0", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.61.0.tgz", + "integrity": "sha512-caX7TrY3Ml6egyDX0WUcTHDxodl/b51y5wJOdCEA36QviK/s2g081hvmGs8eaE3DWb6NYZQ6BjO/QkNRPenoPA==", + "license": "Apache-2.0", + "bin": { + "playwright-core": "cli.js" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/proxy-addr": { + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", + "integrity": "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==", + "license": "MIT", + "dependencies": { + "forwarded": "0.2.0", + "ipaddr.js": "1.9.1" + }, + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/qs": { + "version": "6.15.2", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.15.2.tgz", + "integrity": "sha512-Rzq0KEyX/w/tEybncDgdkZrJgVUsUMk3xjh3t5bv3S1HTAtg+uOYt72+ZfwiQwKdysThkTBdL/rTi6HDmX9Ddw==", + "license": "BSD-3-Clause", + "dependencies": { + "side-channel": "^1.1.0" + }, + "engines": { + "node": ">=0.6" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/range-parser": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", + "integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/raw-body": { + "version": "2.5.3", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.3.tgz", + "integrity": "sha512-s4VSOf6yN0rvbRZGxs8Om5CWj6seneMwK3oDb4lWDH0UPhWcxwOWw5+qk24bxq87szX1ydrwylIOp2uG1ojUpA==", + "license": "MIT", + "dependencies": { + "bytes": "~3.1.2", + "http-errors": "~2.0.1", + "iconv-lite": "~0.4.24", + "unpipe": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", + "license": "MIT" + }, + "node_modules/send": { + "version": "0.19.2", + "resolved": "https://registry.npmjs.org/send/-/send-0.19.2.tgz", + "integrity": "sha512-VMbMxbDeehAxpOtWJXlcUS5E8iXh6QmN+BkRX1GARS3wRaXEEgzCcB10gTQazO42tpNIya8xIyNx8fll1OFPrg==", + "license": "MIT", + "dependencies": { + "debug": "2.6.9", + "depd": "2.0.0", + "destroy": "1.2.0", + "encodeurl": "~2.0.0", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "fresh": "~0.5.2", + "http-errors": "~2.0.1", + "mime": "1.6.0", + "ms": "2.1.3", + "on-finished": "~2.4.1", + "range-parser": "~1.2.1", + "statuses": "~2.0.2" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/send/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, + "node_modules/serve-static": { + "version": "1.16.3", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.16.3.tgz", + "integrity": "sha512-x0RTqQel6g5SY7Lg6ZreMmsOzncHFU7nhnRWkKgWuMTu5NN0DR5oruckMqRvacAN9d5w6ARnRBXl9xhDCgfMeA==", + "license": "MIT", + "dependencies": { + "encodeurl": "~2.0.0", + "escape-html": "~1.0.3", + "parseurl": "~1.3.3", + "send": "~0.19.1" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/setprototypeof": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", + "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==", + "license": "ISC" + }, + "node_modules/side-channel": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.1.1.tgz", + "integrity": "sha512-6x6dK6zJdpTzF4sQeNYxwtvBzf6Eg4GtlesS94HOvTudUeyK2WXAaIfmDgsyslYrRBeFIlsi54AYsFGUuhmvrQ==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "object-inspect": "^1.13.4", + "side-channel-list": "^1.0.1", + "side-channel-map": "^1.0.1", + "side-channel-weakmap": "^1.0.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-list": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/side-channel-list/-/side-channel-list-1.0.1.tgz", + "integrity": "sha512-mjn/0bi/oUURjc5Xl7IaWi/OJJJumuoJFQJfDDyO46+hBWsfaVM65TBHq2eoZBhzl9EchxOijpkbRC8SVBQU0w==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "object-inspect": "^1.13.4" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-map": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/side-channel-map/-/side-channel-map-1.0.1.tgz", + "integrity": "sha512-VCjCNfgMsby3tTdo02nbjtM/ewra6jPHmpThenkTYh8pG9ucZ/1P8So4u4FGBek/BjpOVsDCMoLA/iuBKIFXRA==", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.5", + "object-inspect": "^1.13.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-weakmap": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/side-channel-weakmap/-/side-channel-weakmap-1.0.2.tgz", + "integrity": "sha512-WPS/HvHQTYnHisLo9McqBHOJk2FkHO/tlpvldyrnem4aeQp4hai3gythswg6p01oSoTl58rcpiFAjF2br2Ak2A==", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.5", + "object-inspect": "^1.13.3", + "side-channel-map": "^1.0.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/statuses": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", + "integrity": "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/toidentifier": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", + "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==", + "license": "MIT", + "engines": { + "node": ">=0.6" + } + }, + "node_modules/type-is": { + "version": "1.6.18", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz", + "integrity": "sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==", + "license": "MIT", + "dependencies": { + "media-typer": "0.3.0", + "mime-types": "~2.1.24" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/unpipe": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", + "integrity": "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/utils-merge": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", + "integrity": "sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA==", + "license": "MIT", + "engines": { + "node": ">= 0.4.0" + } + }, + "node_modules/vary": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", + "integrity": "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + } + } +} diff --git a/frontend_old/scraper-service/package.json b/frontend_old/scraper-service/package.json new file mode 100644 index 0000000..c5afa35 --- /dev/null +++ b/frontend_old/scraper-service/package.json @@ -0,0 +1,13 @@ +{ + "name": "scraper-service", + "version": "1.0.0", + "private": true, + "scripts": { + "start": "node index.js", + "install-playwright": "npx playwright install chromium" + }, + "dependencies": { + "express": "^4.18.2", + "playwright": "^1.48.0" + } +} diff --git a/frontend_old/src/app/(dashboard)/chats/page.tsx b/frontend_old/src/app/(dashboard)/chats/page.tsx index 0af8154..9ed8906 100644 --- a/frontend_old/src/app/(dashboard)/chats/page.tsx +++ b/frontend_old/src/app/(dashboard)/chats/page.tsx @@ -1,6 +1,6 @@ "use client" -import { useState, useRef, useCallback, useEffect } from "react" +import { useState, useRef, useCallback, useEffect, useMemo } from "react" import { cn } from "@/lib/utils" import { Button } from "@/components/ui/button" import { Input } from "@/components/ui/input" @@ -90,6 +90,9 @@ function VoiceMessagePlayer({ src, initialDuration }: { src: string; initialDura ) } +const MAX_CACHED_CONVERSATIONS = 5 +const otherParticipant = (conv: any) => conv.otherUser + export default function ChatsPage() { const { theme } = useTheme() const { user } = useUser() @@ -129,6 +132,8 @@ export default function ChatsPage() { const mediaRecorderRef = useRef(null) const recordingChunksRef = useRef([]) const resizeStartRef = useRef({ x: 0, width: 0 }) + const blobUrlsRef = useRef([]) + const conversationOrderRef = useRef([]) const isOnlyEmoji = (text: string) => /^(\p{Extended_Pictographic}\s*)+$/u.test(text.trim()) @@ -141,12 +146,12 @@ export default function ChatsPage() { if (!user) return null - const messages = conversationMessages.get(activeChat || "") || [] - const conversation = conversations.find((c) => c.id === activeChat) - const otherParticipant = (conv: any) => conv.otherUser - const filteredConversations = searchQuery.trim() - ? conversations.filter((conv) => otherParticipant(conv).name.toLowerCase().includes(searchQuery.toLowerCase())) - : conversations + const messages = useMemo(() => conversationMessages.get(activeChat || "") || [], [conversationMessages, activeChat]) + const conversation = useMemo(() => conversations.find((c) => c.id === activeChat), [conversations, activeChat]) + const filteredConversations = useMemo(() => { + if (!searchQuery.trim()) return conversations + return conversations.filter((conv) => otherParticipant(conv).name.toLowerCase().includes(searchQuery.toLowerCase())) + }, [conversations, searchQuery]) // Fetch conversations from API useEffect(() => { @@ -161,7 +166,7 @@ export default function ChatsPage() { if (convs.length > 0) setActiveChat(convs[0].id) } } catch { - // ignore + console.warn("Failed to fetch conversations in chats page") } finally { setLoadingChats(false) } @@ -172,6 +177,10 @@ export default function ChatsPage() { // Fetch messages when active chat changes, and mark as read useEffect(() => { if (!activeChat) return + conversationOrderRef.current = [ + activeChat, + ...conversationOrderRef.current.filter((id) => id !== activeChat), + ] const fetchMsgs = async () => { try { const res = await fetch(`/api/conversations/${activeChat}/messages`) @@ -180,6 +189,30 @@ export default function ChatsPage() { setConversationMessages((prev) => { const next = new Map(prev) next.set(activeChat, data.messages || []) + const recent = conversationOrderRef.current.slice(0, MAX_CACHED_CONVERSATIONS) + const recentSet = new Set(recent) + for (const key of next.keys()) { + if (!recentSet.has(key)) next.delete(key) + } + const validMsgIds = new Set() + for (const msgs of next.values()) { + for (const m of msgs) validMsgIds.add(m.id) + } + setVoiceMessages((v) => { + const n = new Map(v) + for (const k of n.keys()) if (!validMsgIds.has(k)) n.delete(k) + return n + }) + setMessageAttachments((a) => { + const n = new Map(a) + for (const k of n.keys()) if (!validMsgIds.has(k)) n.delete(k) + return n + }) + setReplyMap((r) => { + const n = new Map(r) + for (const k of n.keys()) if (!validMsgIds.has(k)) n.delete(k) + return n + }) return next }) } @@ -206,7 +239,7 @@ export default function ChatsPage() { const existingIds = new Set(conversations.map((c) => otherParticipant(c).id)) setSearchResults(data.users.filter((u: any) => !existingIds.has(u.id))) } - } catch { /* ignore */ } + } catch { console.warn("Failed to search users in chats page") } setSearchingUsers(false) }, 300) return () => clearTimeout(timer) @@ -222,6 +255,14 @@ export default function ChatsPage() { return () => document.removeEventListener("mousedown", handleClickOutside) }, []) + // revoke blob URLs on unmount + useEffect(() => { + return () => { + blobUrlsRef.current.forEach((url) => URL.revokeObjectURL(url)) + blobUrlsRef.current = [] + } + }, []) + const handleResizeStart = useCallback((e: React.MouseEvent) => { e.preventDefault() setIsResizing(true) @@ -254,8 +295,16 @@ export default function ChatsPage() { setTimeout(() => autoResizeTextarea(), 0) } + const MAX_FILE_SIZE = 10 * 1024 * 1024 // 10MB + const handleFileSelect = (e: React.ChangeEvent) => { const files = Array.from(e.target.files ?? []) + const oversized = files.filter((f) => f.size > MAX_FILE_SIZE) + if (oversized.length > 0) { + toast.error(`${oversized[0].name} exceeds the 10MB file size limit`) + if (e.target) e.target.value = "" + return + } setAttachments((prev) => [...prev, ...files]) if (e.target) e.target.value = "" } @@ -319,6 +368,7 @@ export default function ChatsPage() { }) } } catch { + console.warn("Failed to send message in chats page") toast.error("Failed to send message") } } @@ -341,7 +391,11 @@ export default function ChatsPage() { return } const fileAttachments = attachments.length > 0 - ? attachments.map((f) => ({ name: f.name, type: f.type, url: URL.createObjectURL(f) })) + ? attachments.map((f) => { + const url = URL.createObjectURL(f) + blobUrlsRef.current.push(url) + return { name: f.name, type: f.type, url } + }) : undefined if (replyingTo) { const replySender = replyingTo.senderId === user.id ? user.name : otherParticipant(conversation!).name @@ -356,6 +410,10 @@ export default function ChatsPage() { } const handleDeleteMessage = (messageId: string) => { + const files = messageAttachments.get(messageId) + if (files) files.forEach((f) => { URL.revokeObjectURL(f.url); blobUrlsRef.current = blobUrlsRef.current.filter((u) => u !== f.url) }) + const voice = voiceMessages.get(messageId) + if (voice) { URL.revokeObjectURL(voice.url); blobUrlsRef.current = blobUrlsRef.current.filter((u) => u !== voice.url) } setConversationMessages((prev) => { const next = new Map(prev) const msgs = (next.get(activeChat || "") || []).filter((m) => m.id !== messageId) @@ -383,6 +441,7 @@ export default function ChatsPage() { recorder.onstop = () => { const blob = new Blob(recordingChunksRef.current, { type: "audio/webm" }) const url = URL.createObjectURL(blob) + blobUrlsRef.current.push(url) addMessageToChat("", { url, duration: recordingDurationRef.current }) stream.getTracks().forEach((t) => t.stop()) setRecordingDuration(0) @@ -392,6 +451,7 @@ export default function ChatsPage() { setIsRecording(true) setIsPaused(false) } catch { + console.warn("Failed to start recording in chats page") toast.error("Microphone access denied") } } @@ -425,6 +485,7 @@ export default function ChatsPage() { setIsPaused(false) setRecordingDuration(0) recordingDurationRef.current = 0 + recordingChunksRef.current = [] } useEffect(() => { @@ -532,7 +593,7 @@ export default function ChatsPage() { setSearchQuery("") } } - } catch { /* ignore */ } + } catch { console.warn("Failed to create conversation in chats page") } }} className="w-full flex items-center gap-3 p-4 text-left transition-colors hover:bg-muted/50" > @@ -911,6 +972,7 @@ export default function ChatsPage() { toast.success("Message forwarded") } } catch { + console.warn("Failed to forward message in chats page") toast.error("Failed to forward message") } }} diff --git a/frontend_old/src/app/(dashboard)/dashboard/page.tsx b/frontend_old/src/app/(dashboard)/dashboard/page.tsx index c3715fe..b6fe84f 100644 --- a/frontend_old/src/app/(dashboard)/dashboard/page.tsx +++ b/frontend_old/src/app/(dashboard)/dashboard/page.tsx @@ -38,7 +38,7 @@ export default function DashboardPage() { setStats(data) } } catch { - // ignore + console.warn("Failed to fetch dashboard stats") } } diff --git a/frontend_old/src/app/(dashboard)/layout.tsx b/frontend_old/src/app/(dashboard)/layout.tsx index 94b1d8d..53f5638 100644 --- a/frontend_old/src/app/(dashboard)/layout.tsx +++ b/frontend_old/src/app/(dashboard)/layout.tsx @@ -3,6 +3,7 @@ import { AppShell } from "@/components/layout/app-shell" import { UserProvider, useUser } from "@/providers/user-provider" import { NotificationProvider } from "@/providers/notification-provider" +import { ErrorBoundary } from "@/components/shared/error-boundary" import { Loader2 } from "lucide-react" function DashboardContent({ children }: { children: React.ReactNode }) { @@ -29,7 +30,9 @@ export default function DashboardLayout({ return ( - {children} + + {children} + ) diff --git a/frontend_old/src/app/(dashboard)/leads/[id]/page.tsx b/frontend_old/src/app/(dashboard)/leads/[id]/page.tsx index 6cbd4b7..babf5da 100644 --- a/frontend_old/src/app/(dashboard)/leads/[id]/page.tsx +++ b/frontend_old/src/app/(dashboard)/leads/[id]/page.tsx @@ -1,6 +1,6 @@ "use client" -import { useState, useEffect } from "react" +import { useState, useEffect, useCallback } from "react" import Link from "next/link" import { motion } from "framer-motion" import { use } from "react" @@ -26,22 +26,26 @@ export default function LeadDetailsPage({ params }: { params: Promise<{ id: stri const [leadNotes, setLeadNotes] = useState([]) const [loading, setLoading] = useState(true) + const fetchNotes = useCallback(async () => { + const notesRes = await fetch(`/api/leads/${id}/notes`) + if (notesRes.ok) setLeadNotes(await notesRes.json()) + }, [id]) + useEffect(() => { async function fetchData() { try { - const [leadRes, notesRes] = await Promise.all([ + const [leadRes] = await Promise.all([ fetch(`/api/leads/${id}`), - fetch(`/api/leads/${id}/notes`), + fetchNotes(), ]) if (leadRes.ok) setLead(await leadRes.json()) - if (notesRes.ok) setLeadNotes(await notesRes.json()) } catch { - // ignore + console.warn("Failed to fetch lead details") } setLoading(false) } fetchData() - }, [id]) + }, [id, fetchNotes]) if (loading) { return ( @@ -89,8 +93,19 @@ export default function LeadDetailsPage({ params }: { params: Promise<{ id: stri >