Files Changed
Security Fixes
File Change
.env.local Placeholder JWT secret (rotate to random value: openssl rand -base64 64)
src/middleware.ts Removed /api/auth/me bypass; API routes return JSON 401 (not 302 redirect)
src/lib/auth.ts sameSite: "lax" → "strict" (create + destroy cookie); SVG name chars are pre-computed (no injection)
src/lib/avatar.ts Added sanitizeName() — strips non-alphanumeric/safe chars, max 50 chars
src/app/api/users/route.ts POST restricted to super_admin; validates role against whitelist ["sales","admin","super_admin","dev"]; validates active defaults to true
src/app/api/users/[id]/route.ts DELETE restricted to super_admin; blocks self-deletion
src/app/api/leads/[id]/route.ts GET: non-admin users filtered by assigned_to; PATCH: ownership check + score validated 0-100 range
src/app/api/leads/route.ts DELETE: ownership + admin check; GET: ownership filter for non-admin users
src/app/api/conversations/[id]/messages/route.ts GET/POST: verify user is a conversation_participant before access
rust-ai/src/main.rs CORS: AllowOrigin::list(["localhost:3006", "127.0.0.1:3006"]) (was Any)
Performance Fixes
File Change
src/app/api/leads/route.ts Added limit (default 50), offset query params; ownership filter in SQL
src/app/api/users/route.ts Added limit (default 50), offset query params
src/app/api/conversations/route.ts Added LIMIT 50
src/app/api/leads/[id]/notes/route.ts Added limit (default 50), offset query params
src/app/api/conversations/[id]/messages/route.ts Added limit (default 100), offset, before query params
src/app/(dashboard)/chats/page.tsx Pruned to 5 cached conversations; useMemo wraps messages/conversation/filteredConversations; otherParticipant moved outside component; added 10MB file size limit; recordingChunksRef cleared on discard
src/app/(dashboard)/leads/[id]/page.tsx Optimistic status update rolls back on fetch failure
database/migrations/003_chat.sql Added composite index idx_messages_conversation_created on (conversation_id, created_at DESC)
Code Quality Fixes
File Change
src/lib/ai.ts Removed dead getInstructions()/updateInstructions() (called nonexistent /ai/instructions)
src/lib/auth.ts bcrypt.hash(password, 12) — kept (acceptable, OWASP-recommended)
src/app/login/page.tsx "Forgot password?" button now shows alert to contact admin (was no-op)
src/components/users/user-form-dialog.tsx Password min 6 → 8 chars
26 silent catch {} blocks across 16 files (see below) Added console.warn("...") context messages
This commit is contained in:
+15
@@ -34,9 +34,24 @@ yarn-error.log*
|
||||
# env files (can opt-in for committing if needed)
|
||||
.env*
|
||||
|
||||
# runtime data
|
||||
data/
|
||||
data/ai/
|
||||
data/ai/jobs.jsonl
|
||||
|
||||
# logs
|
||||
*.log
|
||||
*.out
|
||||
error-log
|
||||
|
||||
# vercel
|
||||
.vercel
|
||||
|
||||
# python
|
||||
browser-use-service/venv/
|
||||
browser-use-service/__pycache__/
|
||||
browser-use-service/.env
|
||||
|
||||
# typescript
|
||||
*.tsbuildinfo
|
||||
next-env.d.ts
|
||||
|
||||
+114
-57
@@ -1,4 +1,4 @@
|
||||
import os, json, asyncio, re, shutil, sqlite3, traceback, urllib.parse
|
||||
import os, json, asyncio, re, shutil, sqlite3, traceback, urllib.parse, random, time, logging
|
||||
from datetime import datetime, timedelta
|
||||
from bs4 import BeautifulSoup
|
||||
import httpx
|
||||
@@ -7,14 +7,17 @@ from pydantic import BaseModel
|
||||
import uvicorn
|
||||
from playwright.async_api import async_playwright
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format='%(asctime)s [%(levelname)s] %(message)s')
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
app = FastAPI()
|
||||
PORT = int(os.getenv("PORT", "3008"))
|
||||
|
||||
OLLAMA_URL = os.getenv("OLLAMA_URL", "http://localhost:11434")
|
||||
CLASSIFY_MODEL = os.getenv("CLASSIFY_MODEL", "dolphin-llama3:8b")
|
||||
|
||||
FX_PROFILE = os.getenv('FX_PROFILE', r'C:\Users\USER-PC\AppData\Roaming\Mozilla\Firefox\Profiles\h8p11vlj.default-release')
|
||||
FX_COOKIE_DB = os.path.join(FX_PROFILE, 'cookies.sqlite')
|
||||
FX_PROFILE = os.getenv('FX_PROFILE', '')
|
||||
FX_COOKIE_DB = os.path.join(FX_PROFILE, 'cookies.sqlite') if FX_PROFILE else ''
|
||||
|
||||
HEADERS = {
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
|
||||
@@ -22,7 +25,6 @@ HEADERS = {
|
||||
}
|
||||
|
||||
last_scrape_time: datetime | None = None
|
||||
_pw_browser = None
|
||||
|
||||
STRICT_KEYWORDS = [
|
||||
"website", "web design", "web develop", "web dev",
|
||||
@@ -65,40 +67,75 @@ FB_SEARCHES = [
|
||||
"need a site for my business",
|
||||
]
|
||||
|
||||
def get_fb_cookies():
|
||||
"""Copy Firefox cookie DB and extract Facebook cookies for Chromium injection."""
|
||||
tmp = os.path.join(os.path.dirname(FX_COOKIE_DB), 'cookies_tmp.sqlite')
|
||||
async def get_fb_cookies():
|
||||
if not FX_COOKIE_DB or not os.path.exists(FX_COOKIE_DB):
|
||||
logger.warning("FX_COOKIE_DB not found or FX_PROFILE not set")
|
||||
return []
|
||||
tmp = os.path.join(os.path.dirname(FX_COOKIE_DB), f'cookies_tmp_{os.getpid()}.sqlite')
|
||||
for attempt in range(3):
|
||||
try:
|
||||
shutil.copy2(FX_COOKIE_DB, tmp)
|
||||
break
|
||||
except PermissionError:
|
||||
if attempt < 2:
|
||||
await asyncio.sleep(0.5)
|
||||
else:
|
||||
logger.error("Failed to copy cookie DB after 3 attempts")
|
||||
return []
|
||||
try:
|
||||
if not os.path.exists(FX_COOKIE_DB):
|
||||
return []
|
||||
shutil.copy2(FX_COOKIE_DB, tmp)
|
||||
conn = sqlite3.connect(tmp)
|
||||
c = conn.cursor()
|
||||
c.execute("SELECT name, value, host, path FROM moz_cookies WHERE host LIKE '%facebook.com'")
|
||||
rows = c.fetchall()
|
||||
conn.close()
|
||||
os.remove(tmp)
|
||||
try:
|
||||
os.remove(tmp)
|
||||
except Exception:
|
||||
pass
|
||||
return [{
|
||||
"name": name, "value": value,
|
||||
"domain": host if host.startswith('.') else f'.{host}',
|
||||
"path": path,
|
||||
"httpOnly": False, "secure": False, "sameSite": "Lax",
|
||||
"httpOnly": True, "secure": True, "sameSite": "Lax",
|
||||
} for name, value, host, path in rows]
|
||||
except Exception as e:
|
||||
logger.error("Cookie DB read error: %s", e)
|
||||
try:
|
||||
os.remove(tmp)
|
||||
except Exception:
|
||||
pass
|
||||
return []
|
||||
|
||||
def _is_real_text(line: str) -> bool:
|
||||
if not line:
|
||||
return False
|
||||
# Count words (sequences of letters)
|
||||
words = re.findall(r'[A-Za-z]{2,}', line)
|
||||
return len(words) >= 2
|
||||
WEEKDAY_ORDER = ['monday', 'tuesday', 'wednesday', 'thursday', 'friday', 'saturday', 'sunday']
|
||||
|
||||
def _parse_fb_date(block: list[str]) -> str:
|
||||
for l in block:
|
||||
l = l.strip()
|
||||
m = re.match(r'(\d+)\s*(h|hr|hrs|hour|hours)\s*ago', l)
|
||||
if m:
|
||||
hours = int(m.group(1))
|
||||
d = datetime.now() - timedelta(hours=hours)
|
||||
return d.strftime('%Y-%m-%d')
|
||||
if l.lower() == 'yesterday':
|
||||
d = datetime.now() - timedelta(days=1)
|
||||
return d.strftime('%Y-%m-%d')
|
||||
low = l.lower()
|
||||
if low in WEEKDAY_ORDER:
|
||||
day_idx = WEEKDAY_ORDER.index(low)
|
||||
today_idx = datetime.now().weekday()
|
||||
days_ago = (today_idx - day_idx) % 7
|
||||
d = datetime.now() - timedelta(days=days_ago)
|
||||
return d.strftime('%Y-%m-%d')
|
||||
for fmt in ['%Y-%m-%d', '%d %B %Y', '%B %d %Y', '%d %b %Y', '%b %d %Y', '%d %b', '%b %d']:
|
||||
try:
|
||||
dt = datetime.strptime(l, fmt)
|
||||
return dt.strftime('%Y-%m-%d')
|
||||
except ValueError:
|
||||
pass
|
||||
return datetime.now().strftime('%Y-%m-%d')
|
||||
|
||||
def _extract_posts_from_text(raw: str, url: str) -> list[dict]:
|
||||
"""Parse Facebook search results page text into structured posts."""
|
||||
lines = [l.strip() for l in raw.split('\n')]
|
||||
|
||||
# Split into blocks separated by "Facebook" boilerplate lines (2+ consecutive)
|
||||
blocks = []
|
||||
cur = []
|
||||
fb_run = 0
|
||||
@@ -111,7 +148,6 @@ def _extract_posts_from_text(raw: str, url: str) -> list[dict]:
|
||||
else:
|
||||
fb_run = 0
|
||||
if l and len(l) >= 15:
|
||||
# Check for real words (not scattered reaction chars)
|
||||
words = re.findall(r'[A-Za-z]{2,}', l)
|
||||
if len(words) >= 2:
|
||||
cur.append(l)
|
||||
@@ -123,11 +159,9 @@ def _extract_posts_from_text(raw: str, url: str) -> list[dict]:
|
||||
for block in blocks:
|
||||
if not block:
|
||||
continue
|
||||
# Find keyword-matched content in this block
|
||||
kw_indices = [i for i, l in enumerate(block) if kw_match(l)]
|
||||
if not kw_indices:
|
||||
continue
|
||||
# Use first keyword match as anchor, grab context
|
||||
i = kw_indices[0]
|
||||
start = max(0, i - 2)
|
||||
end = min(len(block), i + 5)
|
||||
@@ -144,29 +178,31 @@ def _extract_posts_from_text(raw: str, url: str) -> list[dict]:
|
||||
"author": block[start] if start < i else '',
|
||||
"url": url,
|
||||
"source": "facebook",
|
||||
"date": datetime.now().strftime('%Y-%m-%d'),
|
||||
"date": _parse_fb_date(block),
|
||||
})
|
||||
|
||||
return posts
|
||||
|
||||
async def search_facebook(page, query: str) -> list[dict]:
|
||||
"""Search Facebook and extract post leads from raw page text."""
|
||||
url = f'https://www.facebook.com/search/posts/?q={urllib.parse.quote(query)}'
|
||||
try:
|
||||
await page.goto(url, wait_until='domcontentloaded', timeout=30000)
|
||||
# Wait for the page content to fully render (React/XHR loads)
|
||||
await page.wait_for_timeout(6000)
|
||||
except:
|
||||
except Exception as e:
|
||||
logger.warning("Facebook search navigation failed for '%s': %s", query, e)
|
||||
return []
|
||||
|
||||
raw = await page.evaluate('document.body.innerText')
|
||||
try:
|
||||
raw = await page.evaluate('document.body.innerText')
|
||||
except Exception as e:
|
||||
logger.warning("Failed to evaluate page text: %s", e)
|
||||
return []
|
||||
return _extract_posts_from_text(raw, url)
|
||||
|
||||
async def scrape_facebook() -> list[dict]:
|
||||
"""Launch headless Chromium, inject Firefox Facebook cookies, search for leads."""
|
||||
all_posts = []
|
||||
fb_cookies = get_fb_cookies()
|
||||
fb_cookies = await get_fb_cookies()
|
||||
if not fb_cookies:
|
||||
logger.warning("No Facebook cookies available")
|
||||
return []
|
||||
try:
|
||||
async with async_playwright() as pw:
|
||||
@@ -178,32 +214,34 @@ async def scrape_facebook() -> list[dict]:
|
||||
for c in fb_cookies:
|
||||
try:
|
||||
await context.add_cookies([c])
|
||||
except:
|
||||
pass
|
||||
except Exception as e:
|
||||
logger.warning("Failed to inject cookie %s: %s", c.get('name'), e)
|
||||
|
||||
page = await context.new_page()
|
||||
|
||||
await page.goto('https://www.facebook.com/', wait_until='domcontentloaded', timeout=30000)
|
||||
await page.wait_for_timeout(5000)
|
||||
|
||||
url = page.url
|
||||
if '/login' in url.lower():
|
||||
logger.warning("Facebook login page detected — cookies may be expired")
|
||||
return []
|
||||
|
||||
for query in FB_SEARCHES[:6]:
|
||||
try:
|
||||
posts = await search_facebook(page, query)
|
||||
all_posts.extend(posts)
|
||||
await page.wait_for_timeout(2000)
|
||||
except:
|
||||
continue
|
||||
except Exception:
|
||||
delay = random.uniform(3, 7)
|
||||
await page.wait_for_timeout(int(delay * 1000))
|
||||
except Exception as e:
|
||||
logger.warning("Facebook search '%s' failed: %s", query, e)
|
||||
except Exception as e:
|
||||
logger.error("Facebook scrape failed: %s", e)
|
||||
return []
|
||||
|
||||
seen = set()
|
||||
deduped = []
|
||||
for p in all_posts:
|
||||
key = p['content'][:100]
|
||||
key = p.get('content', '')[:100]
|
||||
if key not in seen:
|
||||
seen.add(key)
|
||||
deduped.append(p)
|
||||
@@ -221,7 +259,8 @@ async def ask_ollama(prompt: str) -> str:
|
||||
"options": {"temperature": 0.05, "num_predict": 1024},
|
||||
})
|
||||
r.raise_for_status()
|
||||
return r.json()["message"]["content"]
|
||||
data = r.json()
|
||||
return data["message"]["content"]
|
||||
|
||||
async def scrape_warriorforum() -> list[dict]:
|
||||
results = []
|
||||
@@ -245,8 +284,9 @@ async def scrape_warriorforum() -> list[dict]:
|
||||
txt = date_div.get_text(strip=True)
|
||||
m = re.search(r'(\d{10})', txt)
|
||||
if m:
|
||||
from datetime import datetime as dtdt
|
||||
date_str = dtdt.utcfromtimestamp(int(m.group(1))).strftime('%Y-%m-%d')
|
||||
date_str = datetime.utcfromtimestamp(int(m.group(1))).strftime('%Y-%m-%d')
|
||||
if not href.startswith('http'):
|
||||
href = 'https://www.warriorforum.com' + href
|
||||
results.append({
|
||||
"title": title, "url": href,
|
||||
"author": author,
|
||||
@@ -255,13 +295,12 @@ async def scrape_warriorforum() -> list[dict]:
|
||||
"date": date_str
|
||||
})
|
||||
except Exception:
|
||||
traceback.print_exc()
|
||||
logger.error("WarriorForum scrape failed", exc_info=True)
|
||||
return results
|
||||
|
||||
async def classify_leads(results: list[dict]) -> list[dict]:
|
||||
if not results:
|
||||
return []
|
||||
filtered = []
|
||||
briefs = [r["title"][:200] for r in results]
|
||||
prompt = f"""Classify each post as LEAD or NOT.
|
||||
LEAD = someone REQUESTING/POSTING/WANTING a website built, designed, or created for them.
|
||||
@@ -277,32 +316,50 @@ NOT LEAD:
|
||||
For each numbered post, answer ONLY "yes" (LEAD) or "no" (NOT LEAD):
|
||||
{chr(10).join(f'{i+1}. {t}' for i, t in enumerate(briefs))}
|
||||
Return a JSON array like ["yes","no","yes"] matching the order above."""
|
||||
ai_succeeded = False
|
||||
try:
|
||||
raw = await ask_ollama(prompt)
|
||||
raw = raw.strip()
|
||||
if raw.startswith("```json"): raw = raw[7:]
|
||||
if raw.startswith("```"): raw = raw.split("\n", 1)[-1].rsplit("```", 1)[0]
|
||||
# Strip markdown code fences
|
||||
if raw.startswith("```"):
|
||||
# Remove opening fence
|
||||
first_nl = raw.find('\n')
|
||||
if first_nl != -1:
|
||||
raw = raw[first_nl + 1:]
|
||||
# Remove closing fence
|
||||
if raw.endswith("```"):
|
||||
raw = raw[:-3]
|
||||
raw = raw.strip()
|
||||
answers = json.loads(raw)
|
||||
if isinstance(answers, list) and len(answers) == len(results):
|
||||
ai_succeeded = True
|
||||
filtered = []
|
||||
for i, ans in enumerate(answers):
|
||||
if isinstance(ans, str) and ans.lower() == 'yes':
|
||||
filtered.append(results[i])
|
||||
except:
|
||||
pass
|
||||
if not filtered:
|
||||
if filtered:
|
||||
return filtered[:10]
|
||||
# AI successfully classified but returned all "no" — respect that decision
|
||||
logger.info("AI classified all %d items as NOT LEAD — returning empty", len(results))
|
||||
return []
|
||||
except Exception as e:
|
||||
logger.warning("AI classification failed, falling back to keyword filter: %s", e)
|
||||
|
||||
# Fallback: only use keyword filter when AI failed
|
||||
if not ai_succeeded:
|
||||
filtered = []
|
||||
for r in results:
|
||||
t = r['title'].lower()
|
||||
# MUST match a web-dev keyword to be a lead
|
||||
if not kw_match_strict(t):
|
||||
continue
|
||||
# But NOT these (offerings, not requests)
|
||||
if any(kw in t for kw in ['i build', 'i offer', 'i create', 'i am a', 'web developer available',
|
||||
'affordable web', 'web hosting', 'free website',
|
||||
'limited time', 'special offer', 'sign up now',
|
||||
'offer']):
|
||||
continue
|
||||
filtered.append(r)
|
||||
return filtered[:10]
|
||||
return filtered[:10]
|
||||
return []
|
||||
|
||||
@app.get("/health")
|
||||
async def health():
|
||||
@@ -338,15 +395,15 @@ async def scrape_all():
|
||||
if wf:
|
||||
wf = await classify_leads(wf)
|
||||
results.extend(wf[:5])
|
||||
except:
|
||||
pass
|
||||
except Exception as e:
|
||||
logger.error("WarriorForum scrape in /scrape/all failed: %s", e)
|
||||
try:
|
||||
fb = await scrape_facebook()
|
||||
if fb:
|
||||
fb = await classify_leads(fb)
|
||||
results.extend(fb[:8])
|
||||
except:
|
||||
pass
|
||||
except Exception as e:
|
||||
logger.error("Facebook scrape in /scrape/all failed: %s", e)
|
||||
return results[:12]
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -22,7 +22,7 @@ $$ LANGUAGE plpgsql;
|
||||
-- 1. AUTHENTICATION & ROLE MANAGEMENT
|
||||
-- ============================================================================
|
||||
|
||||
CREATE TABLE roles (
|
||||
CREATE TABLE IF NOT EXISTS roles (
|
||||
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
||||
name VARCHAR(50) NOT NULL,
|
||||
display_name VARCHAR(100),
|
||||
@@ -35,7 +35,7 @@ CREATE TABLE roles (
|
||||
|
||||
CREATE UNIQUE INDEX uq_roles_name ON roles(name);
|
||||
|
||||
CREATE TABLE permissions (
|
||||
CREATE TABLE IF NOT EXISTS permissions (
|
||||
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
||||
resource VARCHAR(100) NOT NULL,
|
||||
action VARCHAR(50) NOT NULL,
|
||||
@@ -46,7 +46,7 @@ CREATE TABLE permissions (
|
||||
|
||||
CREATE UNIQUE INDEX uq_permissions ON permissions(resource, action);
|
||||
|
||||
CREATE TABLE role_permissions (
|
||||
CREATE TABLE IF NOT EXISTS role_permissions (
|
||||
role_id UUID NOT NULL REFERENCES roles(id) ON DELETE CASCADE,
|
||||
permission_id UUID NOT NULL REFERENCES permissions(id) ON DELETE CASCADE,
|
||||
granted_by UUID,
|
||||
@@ -54,7 +54,7 @@ CREATE TABLE role_permissions (
|
||||
PRIMARY KEY (role_id, permission_id)
|
||||
);
|
||||
|
||||
CREATE TABLE users (
|
||||
CREATE TABLE IF NOT EXISTS users (
|
||||
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
||||
username VARCHAR(100) NOT NULL,
|
||||
email VARCHAR(255) NOT NULL,
|
||||
@@ -83,7 +83,7 @@ CREATE INDEX idx_users_is_active ON users(is_active) WHERE deleted_at IS NULL;
|
||||
CREATE INDEX idx_users_is_locked ON users(is_locked) WHERE is_locked = TRUE AND deleted_at IS NULL;
|
||||
CREATE INDEX idx_users_created_by ON users(created_by);
|
||||
|
||||
CREATE TABLE user_roles (
|
||||
CREATE TABLE IF NOT EXISTS user_roles (
|
||||
user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||
role_id UUID NOT NULL REFERENCES roles(id) ON DELETE CASCADE,
|
||||
assigned_by UUID REFERENCES users(id),
|
||||
@@ -98,7 +98,7 @@ CREATE INDEX idx_user_roles_role ON user_roles(role_id);
|
||||
-- 2. BAN & SUSPENSION SYSTEM
|
||||
-- ============================================================================
|
||||
|
||||
CREATE TABLE banned_users (
|
||||
CREATE TABLE IF NOT EXISTS banned_users (
|
||||
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
||||
banned_user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||
banned_by UUID NOT NULL REFERENCES users(id),
|
||||
@@ -122,7 +122,7 @@ CREATE INDEX idx_banned_users_active ON banned_users(banned_user_id)
|
||||
WHERE is_reversed = FALSE;
|
||||
CREATE INDEX idx_banned_users_banned_by ON banned_users(banned_by);
|
||||
|
||||
CREATE TABLE suspended_users (
|
||||
CREATE TABLE IF NOT EXISTS suspended_users (
|
||||
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
||||
suspended_user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||
suspended_by UUID NOT NULL REFERENCES users(id),
|
||||
@@ -145,7 +145,7 @@ CREATE INDEX idx_suspended_users_suspended_by ON suspended_users(suspended_by);
|
||||
-- 3. SESSION & LOGIN MANAGEMENT
|
||||
-- ============================================================================
|
||||
|
||||
CREATE TABLE sessions (
|
||||
CREATE TABLE IF NOT EXISTS sessions (
|
||||
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
||||
user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||
token_hash VARCHAR(255) NOT NULL,
|
||||
@@ -161,7 +161,7 @@ CREATE INDEX idx_sessions_user ON sessions(user_id) WHERE is_active = TRUE;
|
||||
CREATE UNIQUE INDEX uq_sessions_token ON sessions(token_hash);
|
||||
CREATE INDEX idx_sessions_expires ON sessions(expires_at) WHERE is_active = TRUE;
|
||||
|
||||
CREATE TABLE login_attempts (
|
||||
CREATE TABLE IF NOT EXISTS login_attempts (
|
||||
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
||||
user_id UUID REFERENCES users(id),
|
||||
username_attempted VARCHAR(100) NOT NULL,
|
||||
@@ -181,7 +181,7 @@ CREATE INDEX idx_login_attempts_time ON login_attempts(attempted_at DESC);
|
||||
-- 4. CUSTOMER MANAGEMENT
|
||||
-- ============================================================================
|
||||
|
||||
CREATE TABLE customer_statuses (
|
||||
CREATE TABLE IF NOT EXISTS customer_statuses (
|
||||
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
||||
name VARCHAR(100) NOT NULL,
|
||||
description TEXT,
|
||||
@@ -196,7 +196,7 @@ CREATE TABLE customer_statuses (
|
||||
|
||||
CREATE UNIQUE INDEX uq_customer_statuses_name ON customer_statuses(name) WHERE deleted_at IS NULL;
|
||||
|
||||
CREATE TABLE customers (
|
||||
CREATE TABLE IF NOT EXISTS customers (
|
||||
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
||||
customer_type VARCHAR(20) NOT NULL,
|
||||
status_id UUID NOT NULL REFERENCES customer_statuses(id),
|
||||
@@ -218,7 +218,7 @@ CREATE INDEX idx_customers_score ON customers(score DESC) WHERE deleted_at IS NU
|
||||
CREATE INDEX idx_customers_tags ON customers USING GIN(tags) WHERE deleted_at IS NULL;
|
||||
CREATE INDEX idx_customers_created ON customers(created_at DESC) WHERE deleted_at IS NULL;
|
||||
|
||||
CREATE TABLE individual_customers (
|
||||
CREATE TABLE IF NOT EXISTS individual_customers (
|
||||
customer_id UUID PRIMARY KEY REFERENCES customers(id) ON DELETE CASCADE,
|
||||
first_name VARCHAR(100) NOT NULL,
|
||||
last_name VARCHAR(100) NOT NULL,
|
||||
@@ -233,7 +233,7 @@ CREATE TABLE individual_customers (
|
||||
|
||||
CREATE INDEX idx_individual_names ON individual_customers(last_name, first_name);
|
||||
|
||||
CREATE TABLE company_customers (
|
||||
CREATE TABLE IF NOT EXISTS company_customers (
|
||||
customer_id UUID PRIMARY KEY REFERENCES customers(id) ON DELETE CASCADE,
|
||||
company_name VARCHAR(255) NOT NULL,
|
||||
registration_number VARCHAR(100),
|
||||
@@ -250,7 +250,7 @@ CREATE TABLE company_customers (
|
||||
CREATE INDEX idx_company_name ON company_customers(company_name);
|
||||
CREATE INDEX idx_company_industry ON company_customers(industry);
|
||||
|
||||
CREATE TABLE contact_information (
|
||||
CREATE TABLE IF NOT EXISTS contact_information (
|
||||
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
||||
customer_id UUID NOT NULL REFERENCES customers(id) ON DELETE CASCADE,
|
||||
type VARCHAR(20) NOT NULL,
|
||||
@@ -270,7 +270,7 @@ CREATE INDEX idx_contacts_type ON contact_information(type) WHERE deleted_at IS
|
||||
CREATE INDEX idx_contacts_value ON contact_information(value) WHERE deleted_at IS NULL;
|
||||
CREATE UNIQUE INDEX uq_contacts_primary ON contact_information(customer_id, type) WHERE is_primary = TRUE AND deleted_at IS NULL;
|
||||
|
||||
CREATE TABLE addresses (
|
||||
CREATE TABLE IF NOT EXISTS addresses (
|
||||
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
||||
customer_id UUID NOT NULL REFERENCES customers(id) ON DELETE CASCADE,
|
||||
address_type VARCHAR(20) NOT NULL,
|
||||
@@ -294,7 +294,7 @@ CREATE INDEX idx_addresses_country ON addresses(country) WHERE deleted_at IS NUL
|
||||
CREATE INDEX idx_addresses_city ON addresses(city) WHERE deleted_at IS NULL;
|
||||
CREATE UNIQUE INDEX uq_addresses_primary ON addresses(customer_id, address_type) WHERE is_primary = TRUE AND deleted_at IS NULL;
|
||||
|
||||
CREATE TABLE customer_notes (
|
||||
CREATE TABLE IF NOT EXISTS customer_notes (
|
||||
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
||||
customer_id UUID NOT NULL REFERENCES customers(id) ON DELETE CASCADE,
|
||||
author_id UUID NOT NULL REFERENCES users(id),
|
||||
@@ -315,7 +315,7 @@ CREATE INDEX idx_notes_pinned ON customer_notes(is_pinned) WHERE is_pinned = TRU
|
||||
-- 5. CUSTOMER & INCIDENT REPORTS
|
||||
-- ============================================================================
|
||||
|
||||
CREATE TABLE customer_reports (
|
||||
CREATE TABLE IF NOT EXISTS customer_reports (
|
||||
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
||||
customer_id UUID NOT NULL REFERENCES customers(id),
|
||||
reported_by UUID NOT NULL REFERENCES users(id),
|
||||
@@ -339,7 +339,7 @@ CREATE INDEX idx_customer_reports_reported_by ON customer_reports(reported_by);
|
||||
CREATE INDEX idx_customer_reports_status ON customer_reports(status);
|
||||
CREATE INDEX idx_customer_reports_severity ON customer_reports(severity);
|
||||
|
||||
CREATE TABLE incident_reports (
|
||||
CREATE TABLE IF NOT EXISTS incident_reports (
|
||||
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
||||
reported_by UUID NOT NULL REFERENCES users(id),
|
||||
target_user_id UUID REFERENCES users(id),
|
||||
@@ -368,7 +368,7 @@ CREATE INDEX idx_incident_reports_severity ON incident_reports(severity);
|
||||
-- 6. LEAD MANAGEMENT
|
||||
-- ============================================================================
|
||||
|
||||
CREATE TABLE lead_sources (
|
||||
CREATE TABLE IF NOT EXISTS lead_sources (
|
||||
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
||||
name VARCHAR(100) NOT NULL,
|
||||
description TEXT,
|
||||
@@ -380,7 +380,7 @@ CREATE TABLE lead_sources (
|
||||
|
||||
CREATE UNIQUE INDEX uq_lead_sources_name ON lead_sources(name) WHERE deleted_at IS NULL;
|
||||
|
||||
CREATE TABLE lead_stages (
|
||||
CREATE TABLE IF NOT EXISTS lead_stages (
|
||||
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
||||
name VARCHAR(100) NOT NULL,
|
||||
description TEXT,
|
||||
@@ -395,7 +395,7 @@ CREATE TABLE lead_stages (
|
||||
CREATE UNIQUE INDEX uq_lead_stages_name ON lead_stages(name) WHERE deleted_at IS NULL;
|
||||
CREATE INDEX idx_lead_stages_sort ON lead_stages(sort_order);
|
||||
|
||||
CREATE TABLE leads (
|
||||
CREATE TABLE IF NOT EXISTS leads (
|
||||
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
||||
source_id UUID REFERENCES lead_sources(id),
|
||||
stage_id UUID NOT NULL REFERENCES lead_stages(id),
|
||||
@@ -431,7 +431,7 @@ CREATE INDEX idx_leads_converted ON leads(converted_customer_id) WHERE converted
|
||||
CREATE INDEX idx_leads_email ON leads(email) WHERE deleted_at IS NULL;
|
||||
CREATE INDEX idx_leads_active ON leads(stage_id, assigned_to) WHERE deleted_at IS NULL AND converted_customer_id IS NULL;
|
||||
|
||||
CREATE TABLE lead_conversions (
|
||||
CREATE TABLE IF NOT EXISTS lead_conversions (
|
||||
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
||||
lead_id UUID NOT NULL REFERENCES leads(id) ON DELETE CASCADE,
|
||||
customer_id UUID NOT NULL REFERENCES customers(id) ON DELETE CASCADE,
|
||||
@@ -447,7 +447,7 @@ CREATE INDEX idx_lead_conversions_customer ON lead_conversions(customer_id);
|
||||
-- 7. PRODUCT CATALOG
|
||||
-- ============================================================================
|
||||
|
||||
CREATE TABLE product_categories (
|
||||
CREATE TABLE IF NOT EXISTS product_categories (
|
||||
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
||||
name VARCHAR(255) NOT NULL,
|
||||
description TEXT,
|
||||
@@ -462,7 +462,7 @@ CREATE TABLE product_categories (
|
||||
CREATE UNIQUE INDEX uq_product_categories_name ON product_categories(name) WHERE deleted_at IS NULL;
|
||||
CREATE INDEX idx_product_categories_parent ON product_categories(parent_id);
|
||||
|
||||
CREATE TABLE products (
|
||||
CREATE TABLE IF NOT EXISTS products (
|
||||
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
||||
category_id UUID REFERENCES product_categories(id),
|
||||
name VARCHAR(255) NOT NULL,
|
||||
@@ -487,7 +487,7 @@ CREATE INDEX idx_products_active ON products(is_active) WHERE deleted_at IS NULL
|
||||
-- 8. SALES PIPELINE
|
||||
-- ============================================================================
|
||||
|
||||
CREATE TABLE deal_stages (
|
||||
CREATE TABLE IF NOT EXISTS deal_stages (
|
||||
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
||||
name VARCHAR(100) NOT NULL,
|
||||
description TEXT,
|
||||
@@ -502,7 +502,7 @@ CREATE TABLE deal_stages (
|
||||
CREATE UNIQUE INDEX uq_deal_stages_name ON deal_stages(name) WHERE deleted_at IS NULL;
|
||||
CREATE INDEX idx_deal_stages_sort ON deal_stages(sort_order);
|
||||
|
||||
CREATE TABLE opportunities (
|
||||
CREATE TABLE IF NOT EXISTS opportunities (
|
||||
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
||||
customer_id UUID NOT NULL REFERENCES customers(id),
|
||||
lead_id UUID REFERENCES leads(id),
|
||||
@@ -530,7 +530,7 @@ CREATE INDEX idx_opportunities_won ON opportunities(is_won) WHERE deleted_at IS
|
||||
CREATE INDEX idx_opportunities_created ON opportunities(created_at DESC) WHERE deleted_at IS NULL;
|
||||
CREATE INDEX idx_opportunities_owner_stage ON opportunities(owner_id, stage_id) WHERE deleted_at IS NULL AND is_won IS NULL;
|
||||
|
||||
CREATE TABLE opportunity_products (
|
||||
CREATE TABLE IF NOT EXISTS opportunity_products (
|
||||
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
||||
opportunity_id UUID NOT NULL REFERENCES opportunities(id) ON DELETE CASCADE,
|
||||
product_id UUID NOT NULL REFERENCES products(id),
|
||||
@@ -549,7 +549,7 @@ CREATE INDEX idx_opp_products_product ON opportunity_products(product_id);
|
||||
-- 9. COMMUNICATION TRACKING
|
||||
-- ============================================================================
|
||||
|
||||
CREATE TABLE communication_types (
|
||||
CREATE TABLE IF NOT EXISTS communication_types (
|
||||
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
||||
name VARCHAR(50) NOT NULL,
|
||||
description TEXT,
|
||||
@@ -561,7 +561,7 @@ CREATE TABLE communication_types (
|
||||
|
||||
CREATE UNIQUE INDEX uq_comm_types_name ON communication_types(name);
|
||||
|
||||
CREATE TABLE communications (
|
||||
CREATE TABLE IF NOT EXISTS communications (
|
||||
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
||||
customer_id UUID NOT NULL REFERENCES customers(id),
|
||||
opportunity_id UUID REFERENCES opportunities(id),
|
||||
@@ -585,7 +585,7 @@ CREATE INDEX idx_communications_type ON communications(type_id) WHERE deleted_at
|
||||
CREATE INDEX idx_communications_created_by ON communications(created_by) WHERE deleted_at IS NULL;
|
||||
CREATE INDEX idx_communications_started ON communications(started_at) WHERE deleted_at IS NULL;
|
||||
|
||||
CREATE TABLE communication_participants (
|
||||
CREATE TABLE IF NOT EXISTS communication_participants (
|
||||
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
||||
communication_id UUID NOT NULL REFERENCES communications(id) ON DELETE CASCADE,
|
||||
user_id UUID REFERENCES users(id),
|
||||
@@ -602,7 +602,7 @@ CREATE INDEX idx_comm_participants_user ON communication_participants(user_id);
|
||||
-- 10. TASKS AND ACTIVITIES
|
||||
-- ============================================================================
|
||||
|
||||
CREATE TABLE task_priorities (
|
||||
CREATE TABLE IF NOT EXISTS task_priorities (
|
||||
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
||||
name VARCHAR(50) NOT NULL,
|
||||
color VARCHAR(7),
|
||||
@@ -613,7 +613,7 @@ CREATE TABLE task_priorities (
|
||||
|
||||
CREATE UNIQUE INDEX uq_task_priorities_name ON task_priorities(name);
|
||||
|
||||
CREATE TABLE tasks (
|
||||
CREATE TABLE IF NOT EXISTS tasks (
|
||||
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
||||
customer_id UUID REFERENCES customers(id),
|
||||
opportunity_id UUID REFERENCES opportunities(id),
|
||||
@@ -645,7 +645,7 @@ CREATE INDEX idx_tasks_reminder ON tasks(reminder_at) WHERE reminder_sent = FALS
|
||||
-- 11. INVOICE SUPPORT
|
||||
-- ============================================================================
|
||||
|
||||
CREATE TABLE invoice_statuses (
|
||||
CREATE TABLE IF NOT EXISTS invoice_statuses (
|
||||
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
||||
name VARCHAR(50) NOT NULL,
|
||||
description TEXT,
|
||||
@@ -657,7 +657,7 @@ CREATE TABLE invoice_statuses (
|
||||
|
||||
CREATE UNIQUE INDEX uq_invoice_statuses_name ON invoice_statuses(name);
|
||||
|
||||
CREATE TABLE invoices (
|
||||
CREATE TABLE IF NOT EXISTS invoices (
|
||||
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
||||
customer_id UUID NOT NULL REFERENCES customers(id),
|
||||
opportunity_id UUID REFERENCES opportunities(id),
|
||||
@@ -689,7 +689,7 @@ CREATE INDEX idx_invoices_issued ON invoices(issued_date DESC) WHERE deleted_at
|
||||
CREATE INDEX idx_invoices_due ON invoices(due_date) WHERE deleted_at IS NULL;
|
||||
-- Skipped: cannot use NOW() in partial index predicate (not IMMUTABLE)
|
||||
|
||||
CREATE TABLE invoice_items (
|
||||
CREATE TABLE IF NOT EXISTS invoice_items (
|
||||
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
||||
invoice_id UUID NOT NULL REFERENCES invoices(id) ON DELETE CASCADE,
|
||||
product_id UUID REFERENCES products(id),
|
||||
@@ -705,7 +705,7 @@ CREATE TABLE invoice_items (
|
||||
CREATE INDEX idx_invoice_items_invoice ON invoice_items(invoice_id);
|
||||
CREATE INDEX idx_invoice_items_product ON invoice_items(product_id);
|
||||
|
||||
CREATE TABLE payments (
|
||||
CREATE TABLE IF NOT EXISTS payments (
|
||||
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
||||
invoice_id UUID NOT NULL REFERENCES invoices(id),
|
||||
amount DECIMAL(15,2) NOT NULL CHECK (amount > 0),
|
||||
@@ -729,7 +729,7 @@ CREATE INDEX idx_payments_transaction ON payments(transaction_id) WHERE transact
|
||||
-- 12. AUDIT LOGGING
|
||||
-- ============================================================================
|
||||
|
||||
CREATE TABLE audit_logs (
|
||||
CREATE TABLE IF NOT EXISTS audit_logs (
|
||||
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
||||
table_name VARCHAR(100) NOT NULL,
|
||||
record_id UUID NOT NULL,
|
||||
@@ -755,7 +755,7 @@ CREATE INDEX idx_audit_session ON audit_logs(session_id);
|
||||
-- 13. DUPLICATE DETECTION
|
||||
-- ============================================================================
|
||||
|
||||
CREATE TABLE customer_duplicates (
|
||||
CREATE TABLE IF NOT EXISTS customer_duplicates (
|
||||
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
|
||||
customer_id UUID NOT NULL REFERENCES customers(id) ON DELETE CASCADE,
|
||||
duplicate_customer_id UUID NOT NULL REFERENCES customers(id) ON DELETE CASCADE,
|
||||
@@ -810,11 +810,11 @@ BEGIN
|
||||
)
|
||||
LOOP
|
||||
EXECUTE format(
|
||||
'CREATE TRIGGER trg_%I_updated_at
|
||||
'DROP TRIGGER IF EXISTS trg_%I_updated_at ON %I; CREATE TRIGGER trg_%I_updated_at
|
||||
BEFORE UPDATE ON %I
|
||||
FOR EACH ROW
|
||||
EXECUTE FUNCTION update_updated_at_column()',
|
||||
t, t
|
||||
t, t, t, t
|
||||
);
|
||||
END LOOP;
|
||||
END;
|
||||
@@ -885,6 +885,7 @@ BEGIN
|
||||
END;
|
||||
$$ LANGUAGE plpgsql SECURITY DEFINER;
|
||||
|
||||
DROP TRIGGER IF EXISTS trg_enforce_create_user ON users;
|
||||
CREATE TRIGGER trg_enforce_create_user
|
||||
BEFORE INSERT ON users
|
||||
FOR EACH ROW
|
||||
@@ -935,6 +936,7 @@ BEGIN
|
||||
END;
|
||||
$$ LANGUAGE plpgsql SECURITY DEFINER;
|
||||
|
||||
DROP TRIGGER IF EXISTS trg_enforce_role_assignment ON user_roles;
|
||||
CREATE TRIGGER trg_enforce_role_assignment
|
||||
BEFORE INSERT ON user_roles
|
||||
FOR EACH ROW
|
||||
@@ -973,7 +975,7 @@ BEGIN
|
||||
audit_action,
|
||||
CASE WHEN audit_action IN ('UPDATE', 'DELETE') THEN old_row ELSE NULL END,
|
||||
CASE WHEN audit_action IN ('CREATE', 'UPDATE') THEN new_row ELSE NULL END,
|
||||
NULL
|
||||
NULLIF(current_setting('app.current_user_id', true), '')
|
||||
);
|
||||
|
||||
RETURN COALESCE(NEW, OLD);
|
||||
@@ -998,6 +1000,7 @@ BEGIN
|
||||
END;
|
||||
$$ LANGUAGE plpgsql SECURITY DEFINER;
|
||||
|
||||
DROP TRIGGER IF EXISTS trg_audit_ban ON banned_users;
|
||||
CREATE TRIGGER trg_audit_ban
|
||||
AFTER INSERT OR UPDATE ON banned_users
|
||||
FOR EACH ROW
|
||||
@@ -1021,6 +1024,7 @@ BEGIN
|
||||
END;
|
||||
$$ LANGUAGE plpgsql SECURITY DEFINER;
|
||||
|
||||
DROP TRIGGER IF EXISTS trg_audit_suspension ON suspended_users;
|
||||
CREATE TRIGGER trg_audit_suspension
|
||||
AFTER INSERT OR UPDATE ON suspended_users
|
||||
FOR EACH ROW
|
||||
@@ -1051,6 +1055,7 @@ BEGIN
|
||||
END;
|
||||
$$ LANGUAGE plpgsql SECURITY DEFINER;
|
||||
|
||||
DROP TRIGGER IF EXISTS trg_audit_login ON login_attempts;
|
||||
CREATE TRIGGER trg_audit_login
|
||||
AFTER INSERT ON login_attempts
|
||||
FOR EACH ROW
|
||||
|
||||
@@ -29,6 +29,8 @@ CREATE INDEX IF NOT EXISTS idx_messages_conversation_id ON messages(conversation
|
||||
CREATE INDEX IF NOT EXISTS idx_messages_created_at ON messages(created_at);
|
||||
CREATE INDEX IF NOT EXISTS idx_conversation_participants_user_id ON conversation_participants(user_id);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_messages_conversation_created ON messages(conversation_id, created_at DESC);
|
||||
|
||||
-- Seed conversations between superadmin and other users
|
||||
INSERT INTO conversations (id, created_at) VALUES
|
||||
('c0000000-0000-0000-0000-000000000001', NOW() - INTERVAL '2 hours'),
|
||||
|
||||
@@ -4,6 +4,9 @@
|
||||
-- Usage: psql -U postgres -d crm -f run_all.sql
|
||||
-- ============================================================================
|
||||
|
||||
BEGIN;
|
||||
\set ON_ERROR_STOP on
|
||||
|
||||
\echo '=== Running 001_schema.sql (Tables + Constraints + Functions + Views) ==='
|
||||
\i 001_schema.sql
|
||||
|
||||
@@ -32,9 +35,4 @@
|
||||
\i 009_settings.sql
|
||||
|
||||
\echo '=== Migration Complete ==='
|
||||
\echo ''
|
||||
\echo 'Test accounts created:'
|
||||
\echo ' superadmin_demo / SuperAdmin@2026'
|
||||
\echo ' admin_demo / AdminAccess@2026'
|
||||
\echo ' sales_demo / SalesAccess@2026'
|
||||
\echo ' dev_demo / DevTesting@2026'
|
||||
COMMIT;
|
||||
|
||||
+1
-1
@@ -2,7 +2,7 @@ import type { NextConfig } from "next"
|
||||
|
||||
const nextConfig: NextConfig = {
|
||||
eslint: {
|
||||
ignoreDuringBuilds: true,
|
||||
ignoreDuringBuilds: false,
|
||||
},
|
||||
images: {
|
||||
remotePatterns: [
|
||||
|
||||
+2
-3
@@ -6,12 +6,12 @@
|
||||
"dev": "npm run dev:precheck & npm run dev:ollama & npm run dev:start",
|
||||
"dev:start": "concurrently -n AI,BROWSE,NEXT -c cyan,magenta,green \"npm run dev:rust\" \"npm run dev:browser-use\" \"npm run dev:next\"",
|
||||
"dev:next": "next dev -p 3006",
|
||||
"dev:precheck": "powershell -NoProfile -Command \"$targetPorts=3001,3006,3008; netstat -ano | Select-String LISTENING | ForEach-Object { $line=$_.ToString(); foreach($p in $targetPorts){ if($line -match ('[:]'+$p+'\\s')){ $foundPid=($line -split '\\s+')[-1]; try{ Stop-Process -Id $foundPid -Force -ErrorAction SilentlyContinue; Write-Host ('Freed port '+$p) }catch{} } } }; exit 0\"",
|
||||
"dev:precheck": "powershell -NoProfile -Command \"Get-NetTCPConnection -State Listen | Where-Object { $_.LocalPort -in 3001,3006,3008 } | ForEach-Object { Stop-Process -Id $_.OwningProcess -Force -ErrorAction SilentlyContinue; Write-Host ('Freed port '+$_.LocalPort) }\"",
|
||||
"dev:ollama": "powershell -NoProfile -Command \"if (-not (Get-Process ollama -ErrorAction SilentlyContinue)) { Start-Process ollama -ArgumentList 'serve' -WindowStyle Hidden; Start-Sleep 3 }; exit 0\"",
|
||||
"dev:rust": "cd rust-ai && cargo run",
|
||||
"dev:browser-use": "cd browser-use-service && python main.py",
|
||||
"build": "next build",
|
||||
"start": "npm run dev:next",
|
||||
"start": "next start -p 3006",
|
||||
"lint": "eslint"
|
||||
},
|
||||
"dependencies": {
|
||||
@@ -37,7 +37,6 @@
|
||||
"bcryptjs": "^3.0.3",
|
||||
"class-variance-authority": "^0.7.1",
|
||||
"clsx": "^2.1.1",
|
||||
"emoji-mart": "^5.6.0",
|
||||
"framer-motion": "^11.15.0",
|
||||
"jose": "^6.2.3",
|
||||
"lucide-react": "^0.468.0",
|
||||
|
||||
Generated
+137
-482
File diff suppressed because it is too large
Load Diff
+3
-3
@@ -6,7 +6,7 @@ description = "AI Sales Assistant backend for Coast IT CRM"
|
||||
|
||||
[dependencies]
|
||||
axum = "0.7"
|
||||
tokio = { version = "1", features = ["full"] }
|
||||
tokio = { version = "1", features = ["rt-multi-thread", "macros", "sync", "time", "net", "process"] }
|
||||
reqwest = { version = "0.12", features = ["json", "blocking"] }
|
||||
serde = { version = "1", features = ["derive"] }
|
||||
serde_json = "1"
|
||||
@@ -17,5 +17,5 @@ tracing = "0.1"
|
||||
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
|
||||
tower-http = { version = "0.5", features = ["cors"] }
|
||||
dotenvy = "0.15"
|
||||
scraper = "0.12"
|
||||
rand = "0.8"
|
||||
rand = "0.8"
|
||||
jsonwebtoken = "9"
|
||||
@@ -1,85 +0,0 @@
|
||||
use std::fs;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
use tokio::sync::RwLock;
|
||||
use tracing::{error, info, warn};
|
||||
|
||||
/// Manages the ai.md self-improvement file.
|
||||
/// Loaded on startup and periodically refreshed.
|
||||
pub struct InstructionsManager {
|
||||
path: PathBuf,
|
||||
current: RwLock<String>,
|
||||
}
|
||||
|
||||
impl InstructionsManager {
|
||||
pub fn new(path: PathBuf) -> Self {
|
||||
let initial = fs::read_to_string(&path).unwrap_or_default();
|
||||
if initial.is_empty() {
|
||||
warn!("ai.md is empty or missing at {:?}", path);
|
||||
} else {
|
||||
info!("Loaded ai.md ({} bytes)", initial.len());
|
||||
}
|
||||
Self {
|
||||
path,
|
||||
current: RwLock::new(initial),
|
||||
}
|
||||
}
|
||||
|
||||
/// Read the current instructions
|
||||
pub async fn get(&self) -> String {
|
||||
self.current.read().await.clone()
|
||||
}
|
||||
|
||||
/// Append a new entry to the Improvement Log and optionally update the instructions.
|
||||
/// Returns the updated full content.
|
||||
pub async fn update(&self, entry: &str, new_content: Option<&str>) -> Result<String, String> {
|
||||
if let Some(content) = new_content {
|
||||
// Full replacement
|
||||
fs::write(&self.path, content).map_err(|e| format!("Write failed: {}", e))?;
|
||||
let mut current = self.current.write().await;
|
||||
*current = content.to_string();
|
||||
info!("ai.md fully replaced ({} bytes)", content.len());
|
||||
Ok(content.to_string())
|
||||
} else {
|
||||
// Append to Improvement Log only
|
||||
let mut current = self.current.write().await;
|
||||
let log_entry = format!("\n- {} — {}", chrono::Utc::now().format("%Y-%m-%d %H:%M"), entry);
|
||||
// Find the Improvement Log section and append
|
||||
if let Some(pos) = current.rfind("\n## Improvement Log") {
|
||||
// Find the next section after Improvement Log, or end of file
|
||||
let after_log = ¤t[pos..];
|
||||
if let Some(section_start) = after_log[1..].find("\n## ") {
|
||||
let insert_at = pos + 1 + section_start;
|
||||
current.insert_str(insert_at, &format!("{}\n", log_entry));
|
||||
} else {
|
||||
current.push_str(&format!("{}\n", log_entry));
|
||||
}
|
||||
} else {
|
||||
current.push_str(&format!("\n## Improvement Log\n{}", log_entry));
|
||||
}
|
||||
fs::write(&self.path, &*current).map_err(|e| format!("Write failed: {}", e))?;
|
||||
info!("ai.md improvement log appended: {}", entry);
|
||||
Ok(current.clone())
|
||||
}
|
||||
}
|
||||
|
||||
/// Reload from disk
|
||||
pub async fn reload(&self) {
|
||||
match fs::read_to_string(&self.path) {
|
||||
Ok(content) => {
|
||||
let len = content.len();
|
||||
let mut current = self.current.write().await;
|
||||
*current = content;
|
||||
info!("ai.md reloaded from disk ({} bytes)", len);
|
||||
}
|
||||
Err(e) => error!("Failed to reload ai.md: {}", e),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Wrapper for thread-safe sharing
|
||||
pub type SharedInstructions = Arc<InstructionsManager>;
|
||||
|
||||
pub fn create_shared(path: PathBuf) -> SharedInstructions {
|
||||
Arc::new(InstructionsManager::new(path))
|
||||
}
|
||||
+186
-142
@@ -1,20 +1,69 @@
|
||||
use axum::{
|
||||
extract::State,
|
||||
http::Method,
|
||||
http::{HeaderMap, Method, StatusCode},
|
||||
routing::{get, post},
|
||||
Json, Router,
|
||||
};
|
||||
use tower_http::cors::{CorsLayer, AllowOrigin, Any};
|
||||
use jsonwebtoken::{decode, DecodingKey, Validation, Algorithm};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use sqlx::postgres::PgPoolOptions;
|
||||
use std::collections::HashMap;
|
||||
use std::fs;
|
||||
use std::sync::{Arc, Mutex};
|
||||
use tower_http::cors::{Any, CorsLayer};
|
||||
use tracing::{error, info};
|
||||
use std::sync::Arc;
|
||||
use tokio::sync::Mutex;
|
||||
use tracing::{error, info, warn};
|
||||
use uuid::Uuid;
|
||||
use rand::Rng;
|
||||
use std::time::Duration;
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
// ── JWT Claims ────────────────────────────────────────────────
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct Claims {
|
||||
#[serde(rename = "userId")]
|
||||
user_id: String,
|
||||
role: String,
|
||||
}
|
||||
|
||||
fn verify_jwt(token: &str, secret: &str) -> Option<Claims> {
|
||||
let key = DecodingKey::from_secret(secret.as_bytes());
|
||||
let validation = Validation::new(Algorithm::HS256);
|
||||
decode::<Claims>(token, &key, &validation).ok().map(|d| d.claims)
|
||||
}
|
||||
|
||||
// ── Rate limiter ──────────────────────────────────────────────
|
||||
|
||||
struct RateLimiter {
|
||||
buckets: Mutex<HashMap<String, Vec<u64>>>,
|
||||
max_requests: usize,
|
||||
window_secs: u64,
|
||||
}
|
||||
|
||||
impl RateLimiter {
|
||||
fn new(max_requests: usize, window_secs: u64) -> Self {
|
||||
Self {
|
||||
buckets: Mutex::new(HashMap::new()),
|
||||
max_requests,
|
||||
window_secs,
|
||||
}
|
||||
}
|
||||
|
||||
async fn check(&self, key: &str) -> bool {
|
||||
let now = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs();
|
||||
let mut buckets = self.buckets.lock().await;
|
||||
let timestamps = buckets.entry(key.to_string()).or_default();
|
||||
timestamps.retain(|t| now - *t <= self.window_secs);
|
||||
if timestamps.len() >= self.max_requests {
|
||||
false
|
||||
} else {
|
||||
timestamps.push(now);
|
||||
true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ── Shared state ───────────────────────────────────────────────
|
||||
|
||||
struct AppState {
|
||||
@@ -23,6 +72,9 @@ struct AppState {
|
||||
model: String,
|
||||
jobs: Vec<Job>,
|
||||
leads: Arc<Mutex<LeadStore>>,
|
||||
http_client: reqwest::Client,
|
||||
jwt_secret: String,
|
||||
rate_limiter: RateLimiter,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
@@ -47,7 +99,6 @@ impl LeadStore {
|
||||
}
|
||||
|
||||
fn push(&mut self, lead: Lead) {
|
||||
// Deduplicate by URL
|
||||
if !self.leads.iter().any(|l| l.url == lead.url) {
|
||||
self.leads.insert(0, lead);
|
||||
self.leads.truncate(self.max_size);
|
||||
@@ -55,9 +106,9 @@ impl LeadStore {
|
||||
}
|
||||
|
||||
fn recent(&self, max_age_secs: u64, limit: usize) -> Vec<Lead> {
|
||||
let now = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs();
|
||||
let now = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs();
|
||||
self.leads.iter()
|
||||
.filter(|l| now - l.found_at <= max_age_secs)
|
||||
.filter(|l| now.saturating_sub(l.found_at) <= max_age_secs)
|
||||
.take(limit)
|
||||
.cloned()
|
||||
.collect()
|
||||
@@ -75,8 +126,6 @@ struct Job {
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct ChatRequest {
|
||||
message: String,
|
||||
user_id: String,
|
||||
user_role: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
@@ -127,7 +176,23 @@ struct OllamaResponseMessage {
|
||||
content: String,
|
||||
}
|
||||
|
||||
// ── System prompt builder ─────────────────────────────────────
|
||||
// ── Helpers ────────────────────────────────────────────────────
|
||||
|
||||
fn truncate(s: &str, max: usize) -> String {
|
||||
s.chars().take(max).collect()
|
||||
}
|
||||
|
||||
fn extract_claims(headers: &HeaderMap, state: &AppState) -> Result<Claims, (StatusCode, String)> {
|
||||
let auth_header = headers.get("Authorization").and_then(|v| v.to_str().ok()).unwrap_or("");
|
||||
let token = auth_header.strip_prefix("Bearer ").unwrap_or("");
|
||||
let claims = verify_jwt(token, &state.jwt_secret).ok_or_else(|| {
|
||||
(StatusCode::UNAUTHORIZED, "Unauthorized".to_string())
|
||||
})?;
|
||||
match claims.role.as_str() {
|
||||
"sales" | "admin" | "super_admin" => Ok(claims),
|
||||
_ => Err((StatusCode::FORBIDDEN, "Forbidden".to_string())),
|
||||
}
|
||||
}
|
||||
|
||||
fn format_leads_output(leads: &[Lead]) -> String {
|
||||
if leads.is_empty() {
|
||||
@@ -138,7 +203,7 @@ fn format_leads_output(leads: &[Lead]) -> String {
|
||||
.enumerate()
|
||||
.map(|(i, l)| {
|
||||
let author = if l.author.is_empty() { "Unknown" } else { &l.author };
|
||||
let date = if l.date.is_empty() { "Unknown" } else { &l.date[..l.date.len().min(10)] };
|
||||
let date = truncate(&l.date, 10);
|
||||
format!(
|
||||
"{}. {}\n {}\n {}\n {}",
|
||||
i + 1,
|
||||
@@ -161,9 +226,7 @@ fn build_system_prompt(jobs: &[Job], leads: &[Lead]) -> String {
|
||||
|
||||
let lead_summary: Vec<String> = leads
|
||||
.iter()
|
||||
.map(|l| {
|
||||
format!("{} | {} | {}", l.author, l.title, l.url)
|
||||
})
|
||||
.map(|l| format!("{} | {} | {}", l.author, l.title, l.url))
|
||||
.collect();
|
||||
let lead_summary_str = if lead_summary.is_empty() {
|
||||
"None yet.".to_string()
|
||||
@@ -187,62 +250,56 @@ fn build_system_prompt(jobs: &[Job], leads: &[Lead]) -> String {
|
||||
|
||||
async fn handle_chat(
|
||||
State(state): State<Arc<AppState>>,
|
||||
headers: HeaderMap,
|
||||
Json(req): Json<ChatRequest>,
|
||||
) -> Result<Json<ChatResponse>, (axum::http::StatusCode, String)> {
|
||||
// Validate role
|
||||
match req.user_role.as_str() {
|
||||
"sales" | "admin" | "super_admin" => {}
|
||||
_ => return Err((axum::http::StatusCode::FORBIDDEN, "Forbidden".to_string())),
|
||||
) -> Result<Json<ChatResponse>, (StatusCode, String)> {
|
||||
let claims = extract_claims(&headers, &state)?;
|
||||
|
||||
if !state.rate_limiter.check(&claims.user_id).await {
|
||||
return Err((StatusCode::TOO_MANY_REQUESTS, "Rate limit exceeded".to_string()));
|
||||
}
|
||||
|
||||
// Intercept lead listing requests — scrape on demand then return
|
||||
let msg_lower = req.message.to_lowercase();
|
||||
let msg_words: Vec<&str> = msg_lower.split_whitespace().collect();
|
||||
let has_listing = msg_words.iter().any(|w| ["listings", "listing", "leads", "links", "lists"].contains(w));
|
||||
let has_show = msg_lower.contains("show me") || msg_lower.contains("give me") || msg_lower.contains("pull");
|
||||
let has_job = msg_words.contains(&"jobs") || msg_words.contains(&"job");
|
||||
if has_listing || (has_show && has_job) || (has_show && msg_lower.contains("links")) || msg_lower.contains("recent leads") {
|
||||
// Scrape fresh leads now — call both services
|
||||
let now = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs();
|
||||
let client = reqwest::Client::new();
|
||||
let services = [
|
||||
"http://localhost:3008/scrape/all".to_string(),
|
||||
];
|
||||
for service_url in &services {
|
||||
if let Ok(resp) = client
|
||||
.post(service_url)
|
||||
.timeout(Duration::from_secs(120))
|
||||
.send()
|
||||
.await
|
||||
{
|
||||
if let Ok(leads_data) = resp.json::<Vec<serde_json::Value>>().await {
|
||||
info!("Scraped {} leads from {}", leads_data.len(), service_url);
|
||||
let mut store = state.leads.lock().unwrap();
|
||||
for item in &leads_data {
|
||||
store.push(Lead {
|
||||
title: item["title"].as_str().unwrap_or("").chars().take(120).collect(),
|
||||
url: item["url"].as_str().unwrap_or("").to_string(),
|
||||
source: item["source"].as_str().unwrap_or("unknown").to_string(),
|
||||
found_at: now,
|
||||
author: item["author"].as_str().unwrap_or("").chars().take(60).collect(),
|
||||
date: item["date"].as_str().unwrap_or("").chars().take(30).collect(),
|
||||
content: item["content"].as_str().unwrap_or("").chars().take(300).collect(),
|
||||
});
|
||||
}
|
||||
let now = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs();
|
||||
let service_url = "http://localhost:3008/scrape/all".to_string();
|
||||
if let Ok(resp) = state.http_client
|
||||
.post(&service_url)
|
||||
.timeout(Duration::from_secs(120))
|
||||
.send()
|
||||
.await
|
||||
{
|
||||
if let Ok(leads_data) = resp.json::<Vec<serde_json::Value>>().await {
|
||||
info!("Scraped {} leads from {}", leads_data.len(), service_url);
|
||||
let mut store = state.leads.lock().await;
|
||||
for item in &leads_data {
|
||||
store.push(Lead {
|
||||
title: truncate(item["title"].as_str().unwrap_or(""), 120),
|
||||
url: item["url"].as_str().unwrap_or("").to_string(),
|
||||
source: item["source"].as_str().unwrap_or("unknown").to_string(),
|
||||
found_at: now,
|
||||
author: truncate(item["author"].as_str().unwrap_or(""), 60),
|
||||
date: truncate(item["date"].as_str().unwrap_or(""), 30),
|
||||
content: truncate(item["content"].as_str().unwrap_or(""), 300),
|
||||
});
|
||||
}
|
||||
} else {
|
||||
error!("Scraper service unreachable: {}", service_url);
|
||||
}
|
||||
} else {
|
||||
error!("Scraper service unreachable: {}", service_url);
|
||||
}
|
||||
|
||||
let recent_leads = state.leads.lock().unwrap().recent(604800, 20);
|
||||
let recent_leads = state.leads.lock().await.recent(604800, 20);
|
||||
let response = format_leads_output(&recent_leads);
|
||||
let _ = sqlx::query(
|
||||
"INSERT INTO ai_conversations (id, user_id, role, message, response) VALUES ($1, $2, $3, $4, $5)",
|
||||
)
|
||||
.bind(Uuid::new_v4())
|
||||
.bind(Uuid::parse_str(&req.user_id).unwrap_or(Uuid::nil()))
|
||||
.bind(&req.user_role)
|
||||
.bind(Uuid::parse_str(&claims.user_id).unwrap_or(Uuid::nil()))
|
||||
.bind(&claims.role)
|
||||
.bind(&req.message)
|
||||
.bind(&response)
|
||||
.execute(&state.db)
|
||||
@@ -250,84 +307,62 @@ async fn handle_chat(
|
||||
return Ok(Json(ChatResponse { response }));
|
||||
}
|
||||
|
||||
let recent_leads = state.leads.lock().unwrap().recent(604800, 20);
|
||||
|
||||
let recent_leads = state.leads.lock().await.recent(604800, 20);
|
||||
let system_prompt = build_system_prompt(&state.jobs, &recent_leads);
|
||||
|
||||
let message_text = req.message.clone();
|
||||
|
||||
let ollama_req = OllamaRequest {
|
||||
model: state.model.clone(),
|
||||
messages: vec![
|
||||
OllamaChatMessage {
|
||||
role: "system".to_string(),
|
||||
content: system_prompt,
|
||||
},
|
||||
OllamaChatMessage {
|
||||
role: "user".to_string(),
|
||||
content: req.message.clone(),
|
||||
},
|
||||
OllamaChatMessage { role: "system".to_string(), content: system_prompt },
|
||||
OllamaChatMessage { role: "user".to_string(), content: req.message.clone() },
|
||||
],
|
||||
stream: false,
|
||||
options: OllamaOptions {
|
||||
temperature: 0.7,
|
||||
num_predict: 1024,
|
||||
},
|
||||
options: OllamaOptions { temperature: 0.7, num_predict: 1024 },
|
||||
};
|
||||
|
||||
let client = reqwest::Client::new();
|
||||
let resp = client
|
||||
let resp = state.http_client
|
||||
.post(format!("{}/api/chat", state.ollama_url))
|
||||
.json(&ollama_req)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| {
|
||||
error!("Ollama request failed: {}", e);
|
||||
(
|
||||
axum::http::StatusCode::SERVICE_UNAVAILABLE,
|
||||
"AI service unavailable".to_string(),
|
||||
)
|
||||
(StatusCode::SERVICE_UNAVAILABLE, "AI service unavailable".to_string())
|
||||
})?;
|
||||
|
||||
let ollama_resp: OllamaResponse = resp.json().await.map_err(|e| {
|
||||
error!("Failed to parse Ollama response: {}", e);
|
||||
(
|
||||
axum::http::StatusCode::SERVICE_UNAVAILABLE,
|
||||
"AI response parse error".to_string(),
|
||||
)
|
||||
(StatusCode::SERVICE_UNAVAILABLE, "AI response parse error".to_string())
|
||||
})?;
|
||||
|
||||
let response_text = ollama_resp
|
||||
.message
|
||||
.map(|m| m.content)
|
||||
.unwrap_or_default();
|
||||
let response_text = ollama_resp.message.map(|m| m.content).unwrap_or_default();
|
||||
|
||||
// Store in database
|
||||
let user_id = Uuid::parse_str(&req.user_id).unwrap_or(Uuid::nil());
|
||||
let _ = sqlx::query(
|
||||
"INSERT INTO ai_conversations (id, user_id, role, message, response) VALUES ($1, $2, $3, $4, $5)",
|
||||
)
|
||||
.bind(Uuid::new_v4())
|
||||
.bind(user_id)
|
||||
.bind(&req.user_role)
|
||||
.bind(Uuid::parse_str(&claims.user_id).unwrap_or(Uuid::nil()))
|
||||
.bind(&claims.role)
|
||||
.bind(&message_text)
|
||||
.bind(&response_text)
|
||||
.execute(&state.db)
|
||||
.await;
|
||||
|
||||
Ok(Json(ChatResponse {
|
||||
response: response_text,
|
||||
}))
|
||||
Ok(Json(ChatResponse { response: response_text }))
|
||||
}
|
||||
|
||||
// ── Jobs handler ───────────────────────────────────────────────
|
||||
|
||||
async fn handle_jobs(
|
||||
State(state): State<Arc<AppState>>,
|
||||
) -> Json<JobsResponse> {
|
||||
Json(JobsResponse {
|
||||
jobs: state.jobs.clone(),
|
||||
})
|
||||
headers: HeaderMap,
|
||||
) -> Result<Json<JobsResponse>, (StatusCode, String)> {
|
||||
let _claims = extract_claims(&headers, &state)?;
|
||||
if !state.rate_limiter.check(&_claims.user_id).await {
|
||||
return Err((StatusCode::TOO_MANY_REQUESTS, "Rate limit exceeded".to_string()));
|
||||
}
|
||||
Ok(Json(JobsResponse { jobs: state.jobs.clone() }))
|
||||
}
|
||||
|
||||
// ── Health handler ─────────────────────────────────────────────
|
||||
@@ -354,42 +389,32 @@ async fn main() {
|
||||
|
||||
dotenvy::dotenv().ok();
|
||||
|
||||
let database_url =
|
||||
std::env::var("DATABASE_URL").expect("DATABASE_URL must be set");
|
||||
let ollama_url =
|
||||
std::env::var("OLLAMA_BASE_URL").unwrap_or_else(|_| "http://localhost:11434".to_string());
|
||||
let database_url = std::env::var("DATABASE_URL").expect("DATABASE_URL must be set");
|
||||
let jwt_secret = std::env::var("JWT_SECRET").expect("JWT_SECRET must be set");
|
||||
let ollama_url = std::env::var("OLLAMA_BASE_URL").unwrap_or_else(|_| "http://localhost:11434".to_string());
|
||||
let model = std::env::var("AI_MODEL").unwrap_or_else(|_| "dolphin-phi".to_string());
|
||||
let host = std::env::var("AI_HOST").unwrap_or_else(|_| "0.0.0.0".to_string());
|
||||
let port: u16 = std::env::var("AI_PORT")
|
||||
.unwrap_or_else(|_| "3001".to_string())
|
||||
.parse()
|
||||
.expect("AI_PORT must be a number");
|
||||
let host = std::env::var("AI_HOST").unwrap_or_else(|_| "127.0.0.1".to_string());
|
||||
let port: u16 = std::env::var("AI_PORT").unwrap_or_else(|_| "3001".to_string()).parse().expect("AI_PORT must be a number");
|
||||
|
||||
// Load jobs
|
||||
let jobs_path = std::env::var("JOBS_PATH").unwrap_or_else(|_| "data/ai/jobs.jsonl".to_string());
|
||||
let jobs_content = fs::read_to_string(&jobs_path).unwrap_or_default();
|
||||
let jobs: Vec<Job> = jobs_content
|
||||
.lines()
|
||||
.filter(|l| !l.trim().is_empty())
|
||||
.filter_map(|l| serde_json::from_str(l).ok())
|
||||
.collect();
|
||||
let jobs: Vec<Job> = jobs_content.lines().filter(|l| !l.trim().is_empty()).filter_map(|l| serde_json::from_str(l).ok()).collect();
|
||||
|
||||
info!(
|
||||
"Loaded {} job categories, model: {}, Ollama: {}",
|
||||
jobs.len(),
|
||||
model,
|
||||
ollama_url
|
||||
);
|
||||
info!("Loaded {} job categories, model: {}, Ollama: {}", jobs.len(), model, ollama_url);
|
||||
|
||||
// Connect to database
|
||||
let db = PgPoolOptions::new()
|
||||
.max_connections(5)
|
||||
.max_connections(20)
|
||||
.connect(&database_url)
|
||||
.await
|
||||
.expect("Failed to connect to database");
|
||||
|
||||
info!("Connected to PostgreSQL");
|
||||
|
||||
let http_client = reqwest::Client::builder()
|
||||
.timeout(Duration::from_secs(120))
|
||||
.build()
|
||||
.expect("Failed to build HTTP client");
|
||||
|
||||
let lead_store = Arc::new(Mutex::new(LeadStore::new(100)));
|
||||
|
||||
let state = Arc::new(AppState {
|
||||
@@ -398,11 +423,16 @@ async fn main() {
|
||||
model,
|
||||
jobs,
|
||||
leads: lead_store.clone(),
|
||||
http_client,
|
||||
jwt_secret,
|
||||
rate_limiter: RateLimiter::new(30, 60),
|
||||
});
|
||||
|
||||
// CORS layer
|
||||
let cors = CorsLayer::new()
|
||||
.allow_origin(Any)
|
||||
.allow_origin(AllowOrigin::list([
|
||||
"http://localhost:3006".parse().unwrap(),
|
||||
"http://127.0.0.1:3006".parse().unwrap(),
|
||||
]))
|
||||
.allow_methods([Method::GET, Method::POST])
|
||||
.allow_headers(Any);
|
||||
|
||||
@@ -420,43 +450,57 @@ async fn main() {
|
||||
.await
|
||||
.expect("Failed to bind address");
|
||||
|
||||
// Start background scrapers
|
||||
let bg_leads = lead_store.clone();
|
||||
tokio::task::spawn_blocking(move || {
|
||||
let bg_client = reqwest::blocking::Client::builder()
|
||||
let bg_url = "http://localhost:3008/scrape/all".to_string();
|
||||
tokio::spawn(async move {
|
||||
let client = match reqwest::Client::builder()
|
||||
.timeout(Duration::from_secs(120))
|
||||
.build().ok();
|
||||
.build()
|
||||
{
|
||||
Ok(c) => c,
|
||||
Err(e) => {
|
||||
error!("Failed to build background HTTP client: {} — scraper disabled", e);
|
||||
return;
|
||||
}
|
||||
};
|
||||
loop {
|
||||
if let Some(ref c) = bg_client {
|
||||
let urls = vec![
|
||||
"http://localhost:3008/scrape/all".to_string(),
|
||||
];
|
||||
let now = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs();
|
||||
for url in &urls {
|
||||
if let Ok(resp) = c.post(url).send() {
|
||||
if let Ok(data) = resp.json::<Vec<serde_json::Value>>() {
|
||||
let mut store = bg_leads.lock().unwrap();
|
||||
for item in &data {
|
||||
store.push(Lead {
|
||||
title: item["title"].as_str().unwrap_or("").chars().take(120).collect(),
|
||||
url: item["url"].as_str().unwrap_or("").to_string(),
|
||||
source: item["source"].as_str().unwrap_or("unknown").to_string(),
|
||||
found_at: now,
|
||||
author: item["author"].as_str().unwrap_or("").chars().take(60).collect(),
|
||||
date: item["date"].as_str().unwrap_or("").chars().take(30).collect(),
|
||||
content: item["content"].as_str().unwrap_or("").chars().take(300).collect(),
|
||||
});
|
||||
let now = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs();
|
||||
match client.post(&bg_url).send().await {
|
||||
Ok(resp) => {
|
||||
if resp.status().is_success() {
|
||||
match resp.json::<Vec<serde_json::Value>>().await {
|
||||
Ok(data) => {
|
||||
let mut store = bg_leads.lock().await;
|
||||
for item in &data {
|
||||
store.push(Lead {
|
||||
title: truncate(item["title"].as_str().unwrap_or(""), 120),
|
||||
url: item["url"].as_str().unwrap_or("").to_string(),
|
||||
source: item["source"].as_str().unwrap_or("unknown").to_string(),
|
||||
found_at: now,
|
||||
author: truncate(item["author"].as_str().unwrap_or(""), 60),
|
||||
date: truncate(item["date"].as_str().unwrap_or(""), 30),
|
||||
content: truncate(item["content"].as_str().unwrap_or(""), 300),
|
||||
});
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
warn!("Failed to parse scraper JSON: {}", e);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
warn!("Scraper returned status: {}", resp.status());
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
warn!("Scraper request failed: {}", e);
|
||||
}
|
||||
}
|
||||
let delay = rand::thread_rng().gen_range(120..300);
|
||||
std::thread::sleep(Duration::from_secs(delay));
|
||||
tokio::time::sleep(Duration::from_secs(delay)).await;
|
||||
}
|
||||
});
|
||||
|
||||
axum::serve(listener, app)
|
||||
.await
|
||||
.expect("Server failed");
|
||||
}
|
||||
}
|
||||
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
Binary file not shown.
BIN
Binary file not shown.
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user