Merge branch 'main' of https://git.coastit.co.za/caitlin/CRM_ENVR

message and from who

.gitignore

@@ -34,9 +34,28 @@ yarn-error.log*
 # env files (can opt-in for committing if needed)
 .env*
 
+# runtime data
+data/
+data/ai/
+data/ai/jobs.jsonl
+
+# logs
+*.log
+*.out
+error-log
+
 # vercel
 .vercel
 
+# python
+browser-use-service/venv/
+browser-use-service/__pycache__/
+browser-use-service/.env
+
 # typescript
 *.tsbuildinfo
 next-env.d.ts
+.git
+.git_bak
+node_modules
+target

browser-use-service/main.py

@@ -0,0 +1,410 @@
+import os, json, asyncio, re, shutil, sqlite3, traceback, urllib.parse, random, time, logging
+from datetime import datetime, timedelta
+from bs4 import BeautifulSoup
+import httpx
+from fastapi import FastAPI
+from pydantic import BaseModel
+import uvicorn
+from playwright.async_api import async_playwright
+
+logging.basicConfig(level=logging.INFO, format='%(asctime)s [%(levelname)s] %(message)s')
+logger = logging.getLogger(__name__)
+
+app = FastAPI()
+PORT = int(os.getenv("PORT", "3008"))
+
+OLLAMA_URL = os.getenv("OLLAMA_URL", "http://localhost:11434")
+CLASSIFY_MODEL = os.getenv("CLASSIFY_MODEL", "dolphin-llama3:8b")
+
+FX_PROFILE = os.getenv('FX_PROFILE', '')
+FX_COOKIE_DB = os.path.join(FX_PROFILE, 'cookies.sqlite') if FX_PROFILE else ''
+
+HEADERS = {
+    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
+    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+}
+
+last_scrape_time: datetime | None = None
+
+STRICT_KEYWORDS = [
+    "website", "web design", "web develop", "web dev",
+    "build my website", "build a website", "create a website",
+    "need web", "looking for web", "new website",
+    "landing page", "wordpress",
+    "need a website", "my website", "business website",
+    "need a designer", "help with my website",
+    "redesign", "update my website",
+]
+
+BROAD_KEYWORDS = STRICT_KEYWORDS + [
+    "looking for", "need a", "need an", "looking to",
+    "help me build", "create my", "build me",
+    "design my", "make me a", "would like",
+    "need someone", "hire a", "looking to hire",
+    "want someone", "need help with",
+]
+
+def kw_match_strict(text: str) -> bool:
+    t = text.lower()
+    return any(kw in t for kw in STRICT_KEYWORDS)
+
+def kw_match(text: str) -> bool:
+    t = text.lower()
+    return any(kw in t for kw in BROAD_KEYWORDS)
+
+FB_SEARCHES = [
+    "looking for web developer",
+    "need a website designed",
+    "need website built South Africa",
+    "need someone to build my website",
+    "need web designer",
+    "looking for someone to create website",
+    "need ecommerce website built",
+    "want to hire web developer",
+    "website quote please",
+    "need wordpress website",
+    "I need a website for my business",
+    "need a site for my business",
+]
+
+async def get_fb_cookies():
+    if not FX_COOKIE_DB or not os.path.exists(FX_COOKIE_DB):
+        logger.warning("FX_COOKIE_DB not found or FX_PROFILE not set")
+        return []
+    tmp = os.path.join(os.path.dirname(FX_COOKIE_DB), f'cookies_tmp_{os.getpid()}.sqlite')
+    for attempt in range(3):
+        try:
+            shutil.copy2(FX_COOKIE_DB, tmp)
+            break
+        except PermissionError:
+            if attempt < 2:
+                await asyncio.sleep(0.5)
+            else:
+                logger.error("Failed to copy cookie DB after 3 attempts")
+                return []
+    try:
+        conn = sqlite3.connect(tmp)
+        c = conn.cursor()
+        c.execute("SELECT name, value, host, path FROM moz_cookies WHERE host LIKE '%facebook.com'")
+        rows = c.fetchall()
+        conn.close()
+        try:
+            os.remove(tmp)
+        except Exception:
+            pass
+        return [{
+            "name": name, "value": value,
+            "domain": host if host.startswith('.') else f'.{host}',
+            "path": path,
+            "httpOnly": True, "secure": True, "sameSite": "Lax",
+        } for name, value, host, path in rows]
+    except Exception as e:
+        logger.error("Cookie DB read error: %s", e)
+        try:
+            os.remove(tmp)
+        except Exception:
+            pass
+        return []
+
+WEEKDAY_ORDER = ['monday', 'tuesday', 'wednesday', 'thursday', 'friday', 'saturday', 'sunday']
+
+def _parse_fb_date(block: list[str]) -> str:
+    for l in block:
+        l = l.strip()
+        m = re.match(r'(\d+)\s*(h|hr|hrs|hour|hours)\s*ago', l)
+        if m:
+            hours = int(m.group(1))
+            d = datetime.now() - timedelta(hours=hours)
+            return d.strftime('%Y-%m-%d')
+        if l.lower() == 'yesterday':
+            d = datetime.now() - timedelta(days=1)
+            return d.strftime('%Y-%m-%d')
+        low = l.lower()
+        if low in WEEKDAY_ORDER:
+            day_idx = WEEKDAY_ORDER.index(low)
+            today_idx = datetime.now().weekday()
+            days_ago = (today_idx - day_idx) % 7
+            d = datetime.now() - timedelta(days=days_ago)
+            return d.strftime('%Y-%m-%d')
+        for fmt in ['%Y-%m-%d', '%d %B %Y', '%B %d %Y', '%d %b %Y', '%b %d %Y', '%d %b', '%b %d']:
+            try:
+                dt = datetime.strptime(l, fmt)
+                return dt.strftime('%Y-%m-%d')
+            except ValueError:
+                pass
+    return datetime.now().strftime('%Y-%m-%d')
+
+def _extract_posts_from_text(raw: str, url: str) -> list[dict]:
+    lines = [l.strip() for l in raw.split('\n')]
+    blocks = []
+    cur = []
+    fb_run = 0
+    for l in lines:
+        if 'facebook' in l.lower():
+            fb_run += 1
+            if cur and fb_run >= 2:
+                blocks.append(cur)
+                cur = []
+        else:
+            fb_run = 0
+            if l and len(l) >= 15:
+                words = re.findall(r'[A-Za-z]{2,}', l)
+                if len(words) >= 2:
+                    cur.append(l)
+    if cur:
+        blocks.append(cur)
+
+    posts = []
+    seen_texts = set()
+    for block in blocks:
+        if not block:
+            continue
+        kw_indices = [i for i, l in enumerate(block) if kw_match(l)]
+        if not kw_indices:
+            continue
+        i = kw_indices[0]
+        start = max(0, i - 2)
+        end = min(len(block), i + 5)
+        snippet = ' '.join(block[start:end])
+        if len(snippet) < 40:
+            continue
+        dekey = snippet[:80]
+        if dekey in seen_texts:
+            continue
+        seen_texts.add(dekey)
+        posts.append({
+            "title": snippet[:300],
+            "content": snippet[:1000],
+            "author": block[start] if start < i else '',
+            "url": url,
+            "source": "facebook",
+            "date": _parse_fb_date(block),
+        })
+    return posts
+
+async def search_facebook(page, query: str) -> list[dict]:
+    url = f'https://www.facebook.com/search/posts/?q={urllib.parse.quote(query)}'
+    try:
+        await page.goto(url, wait_until='domcontentloaded', timeout=30000)
+        await page.wait_for_timeout(6000)
+    except Exception as e:
+        logger.warning("Facebook search navigation failed for '%s': %s", query, e)
+        return []
+
+    try:
+        raw = await page.evaluate('document.body.innerText')
+    except Exception as e:
+        logger.warning("Failed to evaluate page text: %s", e)
+        return []
+    return _extract_posts_from_text(raw, url)
+
+async def scrape_facebook() -> list[dict]:
+    all_posts = []
+    fb_cookies = await get_fb_cookies()
+    if not fb_cookies:
+        logger.warning("No Facebook cookies available")
+        return []
+    try:
+        async with async_playwright() as pw:
+            browser = await pw.chromium.launch(headless=True)
+            context = await browser.new_context(
+                user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:150.0) Gecko/20100101 Firefox/150.0',
+                viewport={'width': 1280, 'height': 800},
+            )
+            for c in fb_cookies:
+                try:
+                    await context.add_cookies([c])
+                except Exception as e:
+                    logger.warning("Failed to inject cookie %s: %s", c.get('name'), e)
+
+            page = await context.new_page()
+            await page.goto('https://www.facebook.com/', wait_until='domcontentloaded', timeout=30000)
+            await page.wait_for_timeout(5000)
+
+            url = page.url
+            if '/login' in url.lower():
+                logger.warning("Facebook login page detected — cookies may be expired")
+                return []
+
+            for query in FB_SEARCHES[:6]:
+                try:
+                    posts = await search_facebook(page, query)
+                    all_posts.extend(posts)
+                    delay = random.uniform(3, 7)
+                    await page.wait_for_timeout(int(delay * 1000))
+                except Exception as e:
+                    logger.warning("Facebook search '%s' failed: %s", query, e)
+    except Exception as e:
+        logger.error("Facebook scrape failed: %s", e)
+        return []
+
+    seen = set()
+    deduped = []
+    for p in all_posts:
+        key = p.get('content', '')[:100]
+        if key not in seen:
+            seen.add(key)
+            deduped.append(p)
+    return deduped[:20]
+
+async def ask_ollama(prompt: str) -> str:
+    async with httpx.AsyncClient(timeout=120) as c:
+        r = await c.post(f"{OLLAMA_URL}/api/chat", json={
+            "model": CLASSIFY_MODEL,
+            "messages": [
+                {"role": "system", "content": "You classify forum posts. Return only valid JSON."},
+                {"role": "user", "content": prompt}
+            ],
+            "stream": False,
+            "options": {"temperature": 0.05, "num_predict": 1024},
+        })
+        r.raise_for_status()
+        data = r.json()
+        return data["message"]["content"]
+
+async def scrape_warriorforum() -> list[dict]:
+    results = []
+    try:
+        async with httpx.AsyncClient(headers=HEADERS, timeout=15, follow_redirects=True) as c:
+            r = await c.get('https://www.warriorforum.com/wanted-members-looking-hire-you/')
+            soup = BeautifulSoup(r.text, 'lxml')
+            for row in soup.find_all('td', class_='FlexTable-item--title'):
+                a = row.find('a', href=True)
+                if not a:
+                    continue
+                href = a['href']
+                title = a.text.strip()
+                if not title or len(title) < 15 or not kw_match_strict(title):
+                    continue
+                author_div = row.find('div', class_='FlexTable-item-author')
+                author = author_div.get_text(strip=True).lstrip('by') if author_div else ''
+                date_str = ''
+                date_div = row.find('div', class_=lambda c: c and 'media--available' in c)
+                if date_div:
+                    txt = date_div.get_text(strip=True)
+                    m = re.search(r'(\d{10})', txt)
+                    if m:
+                        date_str = datetime.utcfromtimestamp(int(m.group(1))).strftime('%Y-%m-%d')
+                if not href.startswith('http'):
+                    href = 'https://www.warriorforum.com' + href
+                results.append({
+                    "title": title, "url": href,
+                    "author": author,
+                    "content": title[:300],
+                    "source": "warriorforum",
+                    "date": date_str
+                })
+    except Exception:
+        logger.error("WarriorForum scrape failed", exc_info=True)
+    return results
+
+async def classify_leads(results: list[dict]) -> list[dict]:
+    if not results:
+        return []
+    briefs = [r["title"][:200] for r in results]
+    prompt = f"""Classify each post as LEAD or NOT.
+LEAD = someone REQUESTING/POSTING/WANTING a website built, designed, or created for them.
+LEAD examples: "Need a website for my business", "Looking for web developer to build my site", "I need someone to create my website", "Want a new website for my company", "Looking for someone to design my WordPress site"
+
+NOT LEAD:
+- Offering web design services: "I build websites", "I offer web design", "Affordable web design packages"
+- Already have a website and need marketing, SEO, content, video, link building, email marketing, affiliates
+- Recruiting employees, hiring staff, looking for business partners
+- Selling products, promoting services, affiliate offers
+- "Need web hosting", "Looking for a partner", "Looking for content writer", "Video spokesperson"
+
+For each numbered post, answer ONLY "yes" (LEAD) or "no" (NOT LEAD):
+{chr(10).join(f'{i+1}. {t}' for i, t in enumerate(briefs))}
+Return a JSON array like ["yes","no","yes"] matching the order above."""
+    ai_succeeded = False
+    try:
+        raw = await ask_ollama(prompt)
+        raw = raw.strip()
+        # Strip markdown code fences
+        if raw.startswith("```"):
+            # Remove opening fence
+            first_nl = raw.find('\n')
+            if first_nl != -1:
+                raw = raw[first_nl + 1:]
+            # Remove closing fence
+            if raw.endswith("```"):
+                raw = raw[:-3]
+            raw = raw.strip()
+        answers = json.loads(raw)
+        if isinstance(answers, list) and len(answers) == len(results):
+            ai_succeeded = True
+            filtered = []
+            for i, ans in enumerate(answers):
+                if isinstance(ans, str) and ans.lower() == 'yes':
+                    filtered.append(results[i])
+            if filtered:
+                return filtered[:10]
+            # AI successfully classified but returned all "no" — respect that decision
+            logger.info("AI classified all %d items as NOT LEAD — returning empty", len(results))
+            return []
+    except Exception as e:
+        logger.warning("AI classification failed, falling back to keyword filter: %s", e)
+
+    # Fallback: only use keyword filter when AI failed
+    if not ai_succeeded:
+        filtered = []
+        for r in results:
+            t = r['title'].lower()
+            if not kw_match_strict(t):
+                continue
+            if any(kw in t for kw in ['i build', 'i offer', 'i create', 'i am a', 'web developer available',
+                                       'affordable web', 'web hosting', 'free website',
+                                       'limited time', 'special offer', 'sign up now',
+                                       'offer']):
+                continue
+            filtered.append(r)
+        return filtered[:10]
+    return []
+
+@app.get("/health")
+async def health():
+    return {"status": "ok"}
+
+class ScrapeRequest(BaseModel):
+    query: str = ""
+
+@app.post("/scrape/requests")
+async def scrape_requests(req: ScrapeRequest):
+    global last_scrape_time
+    now = datetime.now()
+    if last_scrape_time and (now - last_scrape_time) < timedelta(seconds=15):
+        return {"error": "rate_limited", "retry_after": 15}
+    last_scrape_time = now
+    results = await scrape_warriorforum()
+    if results:
+        results = await classify_leads(results)
+    return results[:10]
+
+@app.post("/scrape/facebook")
+async def scrape_facebook_endpoint():
+    results = await scrape_facebook()
+    if results:
+        results = await classify_leads(results)
+    return results[:15]
+
+@app.post("/scrape/all")
+async def scrape_all():
+    results = []
+    try:
+        wf = await scrape_warriorforum()
+        if wf:
+            wf = await classify_leads(wf)
+            results.extend(wf[:5])
+    except Exception as e:
+        logger.error("WarriorForum scrape in /scrape/all failed: %s", e)
+    try:
+        fb = await scrape_facebook()
+        if fb:
+            fb = await classify_leads(fb)
+            results.extend(fb[:8])
+    except Exception as e:
+        logger.error("Facebook scrape in /scrape/all failed: %s", e)
+    return results[:12]
+
+if __name__ == "__main__":
+    uvicorn.run(app, host="0.0.0.0", port=PORT)

browser-use-service/run_log.txt

@@ -0,0 +1,7 @@
+python : INFO:     Started server process [20044]
+    + CategoryInfo          : NotSpecified: (INFO:     Started server process [20044]:String) [], RemoteException
+    + FullyQualifiedErrorId : NativeCommandError
+ 
+INFO:     Waiting for application startup.
+INFO:     Application startup complete.
+INFO:     Uvicorn running on http://0.0.0.0:3008 (Press CTRL+C to quit)

browser-use-service/scrape_log.txt

@@ -0,0 +1,6 @@
+[2026-06-22T19:24:48.030482] Browser Use service starting on port 3008
+[2026-06-22T19:27:19.231701] Browser Use service starting on port 3008
+[2026-06-22T19:28:28.335765] Browser Use service starting on port 3008
+[2026-06-22T19:29:05.796265] Browser Use service starting on port 3008
+[2026-06-22T19:29:17.042807] Scraping WarriorForum...
+[2026-06-22T19:29:19.075166] WarriorForum: 55 results

browser-use-service/stderr.txt

@@ -0,0 +1,4 @@
+INFO:     Started server process [24268]
+INFO:     Waiting for application startup.
+INFO:     Application startup complete.
+INFO:     Uvicorn running on http://0.0.0.0:3008 (Press CTRL+C to quit)

browser-use-service/stdout.txt

@@ -0,0 +1,5 @@
+Browser Use service starting on port 3008
+INFO:     127.0.0.1:65003 - "GET /health HTTP/1.1" 200 OK
+Scraping WarriorForum...
+WarriorForum: 55 results
+INFO:     127.0.0.1:54587 - "POST /scrape/all HTTP/1.1" 200 OK

data/ai/ai.md

@@ -0,0 +1,40 @@
+# AI Sales Assistant — Self-Improvement Instructions
+
+## Purpose
+This file contains the AI's own configuration, knowledge, and improvement rules.
+The AI can read and modify this file to update its behavior at runtime.
+
+## Current Instructions
+- Always respond in English
+- Keep responses under 300 words unless asked for detail
+- Use bullet points for lists
+- Be direct and actionable — no fluff
+- Never mention being an AI or language model
+- Refer to the user by their role (salesperson, admin, etc.)
+- If unsure about a topic, say "I don't have that information yet" rather than guessing
+
+## Knowledge Base
+### Sales Tips
+- Cold emails should be under 150 words
+- Follow up within 48 hours
+- Personalise every outreach with the prospect's name and company
+- Use open-ended questions in discovery calls
+- Always ask for the next step before ending a call
+
+### Job Targeting
+- Developers respond best to technical value props
+- Marketing managers care about ROI and metrics
+- C-level executives want brevity and business impact
+
+## Improvement Log
+Track changes made by the AI to improve itself:
+- (initial) Basic instructions and knowledge base created
+
+## Self-Modification Rules
+The AI may update this file when:
+1. It identifies a gap in its knowledge that would help salespeople
+2. It discovers a better way to structure responses
+3. A user explicitly requests an update to behavior
+4. It notices repeated questions that aren't well-covered
+
+Only append to the Improvement Log — don't delete previous entries.

data/ai/jobs.jsonl

@@ -0,0 +1,10 @@
+{"job_title":"Software Developer","keywords":["developer","programmer","software engineer","coder","full stack","backend","frontend"],"industry":"Technology","description":"Builds and maintains software applications and systems"}
+{"job_title":"Marketing Specialist","keywords":["marketing","digital marketing","brand manager","content marketer","social media"],"industry":"Marketing","description":"Plans and executes marketing campaigns across channels"}
+{"job_title":"Sales Representative","keywords":["sales rep","account executive","business development","sales consultant"],"industry":"Sales","description":"Drives revenue through client acquisition and relationship management"}
+{"job_title":"Project Manager","keywords":["project manager","program manager","scrum master","agile coach"],"industry":"Business","description":"Oversees project timelines, resources, and deliverables"}
+{"job_title":"Graphic Designer","keywords":["designer","graphic designer","ui designer","ux designer","visual designer"],"industry":"Creative","description":"Creates visual concepts and designs for digital and print media"}
+{"job_title":"Data Analyst","keywords":["data analyst","business analyst","data scientist","analytics"],"industry":"Technology","description":"Analyzes data to provide actionable business insights"}
+{"job_title":"Customer Support Specialist","keywords":["customer support","customer service","support agent","help desk"],"industry":"Customer Service","description":"Assists customers with inquiries, issues, and product support"}
+{"job_title":"Human Resources Manager","keywords":["HR manager","HR","recruiter","talent acquisition","people operations"],"industry":"Human Resources","description":"Manages recruitment, employee relations, and HR operations"}
+{"job_title":"Financial Advisor","keywords":["financial advisor","financial planner","wealth manager","investment advisor"],"industry":"Finance","description":"Provides financial guidance and investment planning to clients"}
+{"job_title":"Operations Manager","keywords":["operations manager","operations","logistics","supply chain"],"industry":"Business","description":"Oversees daily operations and process optimization"}

database/migrations/001_schema.sql

@@ -22,7 +22,7 @@ $$ LANGUAGE plpgsql;
 -- 1. AUTHENTICATION & ROLE MANAGEMENT
 -- ============================================================================
 
-CREATE TABLE roles (
+CREATE TABLE IF NOT EXISTS roles (
     id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
     name VARCHAR(50) NOT NULL,
     display_name VARCHAR(100),
@@ -35,7 +35,7 @@ CREATE TABLE roles (
 
 CREATE UNIQUE INDEX uq_roles_name ON roles(name);
 
-CREATE TABLE permissions (
+CREATE TABLE IF NOT EXISTS permissions (
     id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
     resource VARCHAR(100) NOT NULL,
     action VARCHAR(50) NOT NULL,
@@ -46,7 +46,7 @@ CREATE TABLE permissions (
 
 CREATE UNIQUE INDEX uq_permissions ON permissions(resource, action);
 
-CREATE TABLE role_permissions (
+CREATE TABLE IF NOT EXISTS role_permissions (
     role_id UUID NOT NULL REFERENCES roles(id) ON DELETE CASCADE,
     permission_id UUID NOT NULL REFERENCES permissions(id) ON DELETE CASCADE,
     granted_by UUID,
@@ -54,7 +54,7 @@ CREATE TABLE role_permissions (
     PRIMARY KEY (role_id, permission_id)
 );
 
-CREATE TABLE users (
+CREATE TABLE IF NOT EXISTS users (
     id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
     username VARCHAR(100) NOT NULL,
     email VARCHAR(255) NOT NULL,
@@ -83,7 +83,7 @@ CREATE INDEX idx_users_is_active ON users(is_active) WHERE deleted_at IS NULL;
 CREATE INDEX idx_users_is_locked ON users(is_locked) WHERE is_locked = TRUE AND deleted_at IS NULL;
 CREATE INDEX idx_users_created_by ON users(created_by);
 
-CREATE TABLE user_roles (
+CREATE TABLE IF NOT EXISTS user_roles (
     user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE,
     role_id UUID NOT NULL REFERENCES roles(id) ON DELETE CASCADE,
     assigned_by UUID REFERENCES users(id),
@@ -98,7 +98,7 @@ CREATE INDEX idx_user_roles_role ON user_roles(role_id);
 -- 2. BAN & SUSPENSION SYSTEM
 -- ============================================================================
 
-CREATE TABLE banned_users (
+CREATE TABLE IF NOT EXISTS banned_users (
     id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
     banned_user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE,
     banned_by UUID NOT NULL REFERENCES users(id),
@@ -122,7 +122,7 @@ CREATE INDEX idx_banned_users_active ON banned_users(banned_user_id)
     WHERE is_reversed = FALSE;
 CREATE INDEX idx_banned_users_banned_by ON banned_users(banned_by);
 
-CREATE TABLE suspended_users (
+CREATE TABLE IF NOT EXISTS suspended_users (
     id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
     suspended_user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE,
     suspended_by UUID NOT NULL REFERENCES users(id),
@@ -145,7 +145,7 @@ CREATE INDEX idx_suspended_users_suspended_by ON suspended_users(suspended_by);
 -- 3. SESSION & LOGIN MANAGEMENT
 -- ============================================================================
 
-CREATE TABLE sessions (
+CREATE TABLE IF NOT EXISTS sessions (
     id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
     user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE,
     token_hash VARCHAR(255) NOT NULL,
@@ -161,7 +161,7 @@ CREATE INDEX idx_sessions_user ON sessions(user_id) WHERE is_active = TRUE;
 CREATE UNIQUE INDEX uq_sessions_token ON sessions(token_hash);
 CREATE INDEX idx_sessions_expires ON sessions(expires_at) WHERE is_active = TRUE;
 
-CREATE TABLE login_attempts (
+CREATE TABLE IF NOT EXISTS login_attempts (
     id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
     user_id UUID REFERENCES users(id),
     username_attempted VARCHAR(100) NOT NULL,
@@ -181,7 +181,7 @@ CREATE INDEX idx_login_attempts_time ON login_attempts(attempted_at DESC);
 -- 4. CUSTOMER MANAGEMENT
 -- ============================================================================
 
-CREATE TABLE customer_statuses (
+CREATE TABLE IF NOT EXISTS customer_statuses (
     id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
     name VARCHAR(100) NOT NULL,
     description TEXT,
@@ -196,7 +196,7 @@ CREATE TABLE customer_statuses (
 
 CREATE UNIQUE INDEX uq_customer_statuses_name ON customer_statuses(name) WHERE deleted_at IS NULL;
 
-CREATE TABLE customers (
+CREATE TABLE IF NOT EXISTS customers (
     id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
     customer_type VARCHAR(20) NOT NULL,
     status_id UUID NOT NULL REFERENCES customer_statuses(id),
@@ -218,7 +218,7 @@ CREATE INDEX idx_customers_score ON customers(score DESC) WHERE deleted_at IS NU
 CREATE INDEX idx_customers_tags ON customers USING GIN(tags) WHERE deleted_at IS NULL;
 CREATE INDEX idx_customers_created ON customers(created_at DESC) WHERE deleted_at IS NULL;
 
-CREATE TABLE individual_customers (
+CREATE TABLE IF NOT EXISTS individual_customers (
     customer_id UUID PRIMARY KEY REFERENCES customers(id) ON DELETE CASCADE,
     first_name VARCHAR(100) NOT NULL,
     last_name VARCHAR(100) NOT NULL,
@@ -233,7 +233,7 @@ CREATE TABLE individual_customers (
 
 CREATE INDEX idx_individual_names ON individual_customers(last_name, first_name);
 
-CREATE TABLE company_customers (
+CREATE TABLE IF NOT EXISTS company_customers (
     customer_id UUID PRIMARY KEY REFERENCES customers(id) ON DELETE CASCADE,
     company_name VARCHAR(255) NOT NULL,
     registration_number VARCHAR(100),
@@ -250,7 +250,7 @@ CREATE TABLE company_customers (
 CREATE INDEX idx_company_name ON company_customers(company_name);
 CREATE INDEX idx_company_industry ON company_customers(industry);
 
-CREATE TABLE contact_information (
+CREATE TABLE IF NOT EXISTS contact_information (
     id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
     customer_id UUID NOT NULL REFERENCES customers(id) ON DELETE CASCADE,
     type VARCHAR(20) NOT NULL,
@@ -270,7 +270,7 @@ CREATE INDEX idx_contacts_type ON contact_information(type) WHERE deleted_at IS
 CREATE INDEX idx_contacts_value ON contact_information(value) WHERE deleted_at IS NULL;
 CREATE UNIQUE INDEX uq_contacts_primary ON contact_information(customer_id, type) WHERE is_primary = TRUE AND deleted_at IS NULL;
 
-CREATE TABLE addresses (
+CREATE TABLE IF NOT EXISTS addresses (
     id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
     customer_id UUID NOT NULL REFERENCES customers(id) ON DELETE CASCADE,
     address_type VARCHAR(20) NOT NULL,
@@ -294,7 +294,7 @@ CREATE INDEX idx_addresses_country ON addresses(country) WHERE deleted_at IS NUL
 CREATE INDEX idx_addresses_city ON addresses(city) WHERE deleted_at IS NULL;
 CREATE UNIQUE INDEX uq_addresses_primary ON addresses(customer_id, address_type) WHERE is_primary = TRUE AND deleted_at IS NULL;
 
-CREATE TABLE customer_notes (
+CREATE TABLE IF NOT EXISTS customer_notes (
     id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
     customer_id UUID NOT NULL REFERENCES customers(id) ON DELETE CASCADE,
     author_id UUID NOT NULL REFERENCES users(id),
@@ -315,7 +315,7 @@ CREATE INDEX idx_notes_pinned ON customer_notes(is_pinned) WHERE is_pinned = TRU
 -- 5. CUSTOMER & INCIDENT REPORTS
 -- ============================================================================
 
-CREATE TABLE customer_reports (
+CREATE TABLE IF NOT EXISTS customer_reports (
     id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
     customer_id UUID NOT NULL REFERENCES customers(id),
     reported_by UUID NOT NULL REFERENCES users(id),
@@ -339,7 +339,7 @@ CREATE INDEX idx_customer_reports_reported_by ON customer_reports(reported_by);
 CREATE INDEX idx_customer_reports_status ON customer_reports(status);
 CREATE INDEX idx_customer_reports_severity ON customer_reports(severity);
 
-CREATE TABLE incident_reports (
+CREATE TABLE IF NOT EXISTS incident_reports (
     id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
     reported_by UUID NOT NULL REFERENCES users(id),
     target_user_id UUID REFERENCES users(id),
@@ -368,7 +368,7 @@ CREATE INDEX idx_incident_reports_severity ON incident_reports(severity);
 -- 6. LEAD MANAGEMENT
 -- ============================================================================
 
-CREATE TABLE lead_sources (
+CREATE TABLE IF NOT EXISTS lead_sources (
     id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
     name VARCHAR(100) NOT NULL,
     description TEXT,
@@ -380,7 +380,7 @@ CREATE TABLE lead_sources (
 
 CREATE UNIQUE INDEX uq_lead_sources_name ON lead_sources(name) WHERE deleted_at IS NULL;
 
-CREATE TABLE lead_stages (
+CREATE TABLE IF NOT EXISTS lead_stages (
     id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
     name VARCHAR(100) NOT NULL,
     description TEXT,
@@ -395,7 +395,7 @@ CREATE TABLE lead_stages (
 CREATE UNIQUE INDEX uq_lead_stages_name ON lead_stages(name) WHERE deleted_at IS NULL;
 CREATE INDEX idx_lead_stages_sort ON lead_stages(sort_order);
 
-CREATE TABLE leads (
+CREATE TABLE IF NOT EXISTS leads (
     id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
     source_id UUID REFERENCES lead_sources(id),
     stage_id UUID NOT NULL REFERENCES lead_stages(id),
@@ -431,7 +431,7 @@ CREATE INDEX idx_leads_converted ON leads(converted_customer_id) WHERE converted
 CREATE INDEX idx_leads_email ON leads(email) WHERE deleted_at IS NULL;
 CREATE INDEX idx_leads_active ON leads(stage_id, assigned_to) WHERE deleted_at IS NULL AND converted_customer_id IS NULL;
 
-CREATE TABLE lead_conversions (
+CREATE TABLE IF NOT EXISTS lead_conversions (
     id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
     lead_id UUID NOT NULL REFERENCES leads(id) ON DELETE CASCADE,
     customer_id UUID NOT NULL REFERENCES customers(id) ON DELETE CASCADE,
@@ -447,7 +447,7 @@ CREATE INDEX idx_lead_conversions_customer ON lead_conversions(customer_id);
 -- 7. PRODUCT CATALOG
 -- ============================================================================
 
-CREATE TABLE product_categories (
+CREATE TABLE IF NOT EXISTS product_categories (
     id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
     name VARCHAR(255) NOT NULL,
     description TEXT,
@@ -462,7 +462,7 @@ CREATE TABLE product_categories (
 CREATE UNIQUE INDEX uq_product_categories_name ON product_categories(name) WHERE deleted_at IS NULL;
 CREATE INDEX idx_product_categories_parent ON product_categories(parent_id);
 
-CREATE TABLE products (
+CREATE TABLE IF NOT EXISTS products (
     id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
     category_id UUID REFERENCES product_categories(id),
     name VARCHAR(255) NOT NULL,
@@ -487,7 +487,7 @@ CREATE INDEX idx_products_active ON products(is_active) WHERE deleted_at IS NULL
 -- 8. SALES PIPELINE
 -- ============================================================================
 
-CREATE TABLE deal_stages (
+CREATE TABLE IF NOT EXISTS deal_stages (
     id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
     name VARCHAR(100) NOT NULL,
     description TEXT,
@@ -502,7 +502,7 @@ CREATE TABLE deal_stages (
 CREATE UNIQUE INDEX uq_deal_stages_name ON deal_stages(name) WHERE deleted_at IS NULL;
 CREATE INDEX idx_deal_stages_sort ON deal_stages(sort_order);
 
-CREATE TABLE opportunities (
+CREATE TABLE IF NOT EXISTS opportunities (
     id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
     customer_id UUID NOT NULL REFERENCES customers(id),
     lead_id UUID REFERENCES leads(id),
@@ -530,7 +530,7 @@ CREATE INDEX idx_opportunities_won ON opportunities(is_won) WHERE deleted_at IS
 CREATE INDEX idx_opportunities_created ON opportunities(created_at DESC) WHERE deleted_at IS NULL;
 CREATE INDEX idx_opportunities_owner_stage ON opportunities(owner_id, stage_id) WHERE deleted_at IS NULL AND is_won IS NULL;
 
-CREATE TABLE opportunity_products (
+CREATE TABLE IF NOT EXISTS opportunity_products (
     id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
     opportunity_id UUID NOT NULL REFERENCES opportunities(id) ON DELETE CASCADE,
     product_id UUID NOT NULL REFERENCES products(id),
@@ -549,7 +549,7 @@ CREATE INDEX idx_opp_products_product ON opportunity_products(product_id);
 -- 9. COMMUNICATION TRACKING
 -- ============================================================================
 
-CREATE TABLE communication_types (
+CREATE TABLE IF NOT EXISTS communication_types (
     id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
     name VARCHAR(50) NOT NULL,
     description TEXT,
@@ -561,7 +561,7 @@ CREATE TABLE communication_types (
 
 CREATE UNIQUE INDEX uq_comm_types_name ON communication_types(name);
 
-CREATE TABLE communications (
+CREATE TABLE IF NOT EXISTS communications (
     id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
     customer_id UUID NOT NULL REFERENCES customers(id),
     opportunity_id UUID REFERENCES opportunities(id),
@@ -585,7 +585,7 @@ CREATE INDEX idx_communications_type ON communications(type_id) WHERE deleted_at
 CREATE INDEX idx_communications_created_by ON communications(created_by) WHERE deleted_at IS NULL;
 CREATE INDEX idx_communications_started ON communications(started_at) WHERE deleted_at IS NULL;
 
-CREATE TABLE communication_participants (
+CREATE TABLE IF NOT EXISTS communication_participants (
     id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
     communication_id UUID NOT NULL REFERENCES communications(id) ON DELETE CASCADE,
     user_id UUID REFERENCES users(id),
@@ -602,7 +602,7 @@ CREATE INDEX idx_comm_participants_user ON communication_participants(user_id);
 -- 10. TASKS AND ACTIVITIES
 -- ============================================================================
 
-CREATE TABLE task_priorities (
+CREATE TABLE IF NOT EXISTS task_priorities (
     id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
     name VARCHAR(50) NOT NULL,
     color VARCHAR(7),
@@ -613,7 +613,7 @@ CREATE TABLE task_priorities (
 
 CREATE UNIQUE INDEX uq_task_priorities_name ON task_priorities(name);
 
-CREATE TABLE tasks (
+CREATE TABLE IF NOT EXISTS tasks (
     id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
     customer_id UUID REFERENCES customers(id),
     opportunity_id UUID REFERENCES opportunities(id),
@@ -645,7 +645,7 @@ CREATE INDEX idx_tasks_reminder ON tasks(reminder_at) WHERE reminder_sent = FALS
 -- 11. INVOICE SUPPORT
 -- ============================================================================
 
-CREATE TABLE invoice_statuses (
+CREATE TABLE IF NOT EXISTS invoice_statuses (
     id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
     name VARCHAR(50) NOT NULL,
     description TEXT,
@@ -657,7 +657,7 @@ CREATE TABLE invoice_statuses (
 
 CREATE UNIQUE INDEX uq_invoice_statuses_name ON invoice_statuses(name);
 
-CREATE TABLE invoices (
+CREATE TABLE IF NOT EXISTS invoices (
     id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
     customer_id UUID NOT NULL REFERENCES customers(id),
     opportunity_id UUID REFERENCES opportunities(id),
@@ -689,7 +689,7 @@ CREATE INDEX idx_invoices_issued ON invoices(issued_date DESC) WHERE deleted_at
 CREATE INDEX idx_invoices_due ON invoices(due_date) WHERE deleted_at IS NULL;
 -- Skipped: cannot use NOW() in partial index predicate (not IMMUTABLE)
 
-CREATE TABLE invoice_items (
+CREATE TABLE IF NOT EXISTS invoice_items (
     id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
     invoice_id UUID NOT NULL REFERENCES invoices(id) ON DELETE CASCADE,
     product_id UUID REFERENCES products(id),
@@ -705,7 +705,7 @@ CREATE TABLE invoice_items (
 CREATE INDEX idx_invoice_items_invoice ON invoice_items(invoice_id);
 CREATE INDEX idx_invoice_items_product ON invoice_items(product_id);
 
-CREATE TABLE payments (
+CREATE TABLE IF NOT EXISTS payments (
     id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
     invoice_id UUID NOT NULL REFERENCES invoices(id),
     amount DECIMAL(15,2) NOT NULL CHECK (amount > 0),
@@ -729,7 +729,7 @@ CREATE INDEX idx_payments_transaction ON payments(transaction_id) WHERE transact
 -- 12. AUDIT LOGGING
 -- ============================================================================
 
-CREATE TABLE audit_logs (
+CREATE TABLE IF NOT EXISTS audit_logs (
     id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
     table_name VARCHAR(100) NOT NULL,
     record_id UUID NOT NULL,
@@ -755,7 +755,7 @@ CREATE INDEX idx_audit_session ON audit_logs(session_id);
 -- 13. DUPLICATE DETECTION
 -- ============================================================================
 
-CREATE TABLE customer_duplicates (
+CREATE TABLE IF NOT EXISTS customer_duplicates (
     id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
     customer_id UUID NOT NULL REFERENCES customers(id) ON DELETE CASCADE,
     duplicate_customer_id UUID NOT NULL REFERENCES customers(id) ON DELETE CASCADE,
@@ -810,11 +810,11 @@ BEGIN
           )
     LOOP
         EXECUTE format(
-            'CREATE TRIGGER trg_%I_updated_at
+            'DROP TRIGGER IF EXISTS trg_%I_updated_at ON %I; CREATE TRIGGER trg_%I_updated_at
              BEFORE UPDATE ON %I
              FOR EACH ROW
              EXECUTE FUNCTION update_updated_at_column()',
-            t, t
+            t, t, t, t
         );
     END LOOP;
 END;
@@ -885,6 +885,7 @@ BEGIN
 END;
 $$ LANGUAGE plpgsql SECURITY DEFINER;
 
+DROP TRIGGER IF EXISTS trg_enforce_create_user ON users;
 CREATE TRIGGER trg_enforce_create_user
     BEFORE INSERT ON users
     FOR EACH ROW
@@ -935,6 +936,7 @@ BEGIN
 END;
 $$ LANGUAGE plpgsql SECURITY DEFINER;
 
+DROP TRIGGER IF EXISTS trg_enforce_role_assignment ON user_roles;
 CREATE TRIGGER trg_enforce_role_assignment
     BEFORE INSERT ON user_roles
     FOR EACH ROW
@@ -973,7 +975,7 @@ BEGIN
         audit_action,
         CASE WHEN audit_action IN ('UPDATE', 'DELETE') THEN old_row ELSE NULL END,
         CASE WHEN audit_action IN ('CREATE', 'UPDATE') THEN new_row ELSE NULL END,
-        NULL
+        NULLIF(current_setting('app.current_user_id', true), '')
     );
 
     RETURN COALESCE(NEW, OLD);
@@ -998,6 +1000,7 @@ BEGIN
 END;
 $$ LANGUAGE plpgsql SECURITY DEFINER;
 
+DROP TRIGGER IF EXISTS trg_audit_ban ON banned_users;
 CREATE TRIGGER trg_audit_ban
     AFTER INSERT OR UPDATE ON banned_users
     FOR EACH ROW
@@ -1021,6 +1024,7 @@ BEGIN
 END;
 $$ LANGUAGE plpgsql SECURITY DEFINER;
 
+DROP TRIGGER IF EXISTS trg_audit_suspension ON suspended_users;
 CREATE TRIGGER trg_audit_suspension
     AFTER INSERT OR UPDATE ON suspended_users
     FOR EACH ROW
@@ -1051,6 +1055,7 @@ BEGIN
 END;
 $$ LANGUAGE plpgsql SECURITY DEFINER;
 
+DROP TRIGGER IF EXISTS trg_audit_login ON login_attempts;
 CREATE TRIGGER trg_audit_login
     AFTER INSERT ON login_attempts
     FOR EACH ROW

database/migrations/003_chat.sql

@@ -29,6 +29,8 @@ CREATE INDEX IF NOT EXISTS idx_messages_conversation_id ON messages(conversation
 CREATE INDEX IF NOT EXISTS idx_messages_created_at ON messages(created_at);
 CREATE INDEX IF NOT EXISTS idx_conversation_participants_user_id ON conversation_participants(user_id);
 
+CREATE INDEX IF NOT EXISTS idx_messages_conversation_created ON messages(conversation_id, created_at DESC);
+
 -- Seed conversations between superadmin and other users
 INSERT INTO conversations (id, created_at) VALUES
     ('c0000000-0000-0000-0000-000000000001', NOW() - INTERVAL '2 hours'),

database/migrations/007_ai_features.sql

@@ -0,0 +1,13 @@
+-- AI Sales Assistant tables
+
+CREATE TABLE IF NOT EXISTS ai_conversations (
+    id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
+    user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE,
+    role VARCHAR(20) NOT NULL DEFAULT 'sales',
+    message TEXT NOT NULL,
+    response TEXT,
+    created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
+);
+
+CREATE INDEX IF NOT EXISTS idx_ai_conversations_user ON ai_conversations(user_id);
+CREATE INDEX IF NOT EXISTS idx_ai_conversations_created ON ai_conversations(created_at DESC);

database/migrations/008_notifications.sql

@@ -0,0 +1,23 @@
+CREATE TABLE IF NOT EXISTS notifications (
+  id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
+  user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE,
+  type VARCHAR(50) NOT NULL,
+  title VARCHAR(255) NOT NULL,
+  description TEXT,
+  link TEXT,
+  is_read BOOLEAN NOT NULL DEFAULT FALSE,
+  created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
+);
+
+CREATE INDEX IF NOT EXISTS idx_notifications_user ON notifications(user_id, created_at DESC);
+CREATE INDEX IF NOT EXISTS idx_notifications_unread ON notifications(user_id, created_at DESC) WHERE is_read = FALSE;
+
+CREATE TABLE IF NOT EXISTS notification_preferences (
+  user_id UUID PRIMARY KEY REFERENCES users(id) ON DELETE CASCADE,
+  lead_assigned BOOLEAN NOT NULL DEFAULT TRUE,
+  lead_status BOOLEAN NOT NULL DEFAULT TRUE,
+  note_added BOOLEAN NOT NULL DEFAULT FALSE,
+  daily_digest BOOLEAN NOT NULL DEFAULT FALSE,
+  weekly_report BOOLEAN NOT NULL DEFAULT TRUE,
+  updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
+);

database/migrations/009_settings.sql

@@ -0,0 +1,22 @@
+CREATE TABLE IF NOT EXISTS company_settings (
+  id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
+  company_name VARCHAR(255) NOT NULL DEFAULT '',
+  company_email VARCHAR(255) NOT NULL DEFAULT '',
+  company_phone VARCHAR(50) NOT NULL DEFAULT '',
+  company_website VARCHAR(255) NOT NULL DEFAULT '',
+  company_address TEXT NOT NULL DEFAULT '',
+  updated_by UUID REFERENCES users(id),
+  updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
+);
+
+INSERT INTO company_settings (company_name, company_email, company_phone, company_website, company_address)
+VALUES ('Coastal IT Solutions', 'info@coastalit.com', '(555) 123-4567', 'https://coastalit.com', '123 Business Ave, Suite 100, San Francisco, CA 94105')
+ON CONFLICT DO NOTHING;
+
+CREATE TABLE IF NOT EXISTS user_preferences (
+  user_id UUID PRIMARY KEY REFERENCES users(id) ON DELETE CASCADE,
+  timezone VARCHAR(100) NOT NULL DEFAULT 'america-los_angeles',
+  date_format VARCHAR(10) NOT NULL DEFAULT 'mdy',
+  items_per_page INTEGER NOT NULL DEFAULT 20,
+  updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
+);

database/migrations/010_chat_notifications.sql

@@ -0,0 +1,4 @@
+ALTER TABLE notifications ADD COLUMN IF NOT EXISTS context_id UUID;
+ALTER TABLE notifications ADD COLUMN IF NOT EXISTS context_type VARCHAR(50);
+
+CREATE INDEX IF NOT EXISTS idx_notifications_context ON notifications(context_type, context_id) WHERE context_type IS NOT NULL;

database/migrations/run_all.sql

@@ -4,16 +4,38 @@
 -- Usage: psql -U postgres -d crm -f run_all.sql
 -- ============================================================================
 
+BEGIN;
+\set ON_ERROR_STOP on
+
 \echo '=== Running 001_schema.sql (Tables + Constraints + Functions + Views) ==='
 \i 001_schema.sql
 
 \echo '=== Running 002_seed.sql (RBAC + Test Accounts + Reference Data) ==='
 \i 002_seed.sql
 
+\echo '=== Running 003_chat.sql (Chat Tables) ==='
+\i 003_chat.sql
+
+\echo '=== Running 004_avatar_url.sql (Avatar URL Column) ==='
+\i 004_avatar_url.sql
+
+\echo '=== Running 005_last_read_at.sql (Last Read At Column) ==='
+\i 005_last_read_at.sql
+
+\echo '=== Running 006_test_leads.sql (Test Lead Data) ==='
+\i 006_test_leads.sql
+
+\echo '=== Running 007_ai_features.sql (AI Features) ==='
+\i 007_ai_features.sql
+
+\echo '=== Running 008_notifications.sql (Notifications + Preferences) ==='
+\i 008_notifications.sql
+
+\echo '=== Running 009_settings.sql (Company Settings + User Preferences) ==='
+\i 009_settings.sql
+
+\echo '=== Running 010_chat_notifications.sql (Chat Message Notifications) ==='
+\i 010_chat_notifications.sql
+
 \echo '=== Migration Complete ==='
-\echo ''
+COMMIT;
-\echo 'Test accounts created:'
-\echo '  superadmin_demo / SuperAdmin@2026'
-\echo '  admin_demo      / AdminAccess@2026'
-\echo '  sales_demo      / SalesAccess@2026'
-\echo '  dev_demo        / DevTesting@2026'

fix.md

@@ -0,0 +1,71 @@
+# Scrapers - Configuration & Status
+
+## Facebook Scraper
+
+**File:** `rust-ai/src/main.rs`  
+**Lines:** ~70-85
+
+Target URL (line 72):
+```rust
+let url = "https://www.facebook.com/search/top/?q=need%20website%20create";
+```
+
+**Status:** Uses direct connection (no proxy) — your home IP. Facebook blocks datacenter IPs. May work from a residential connection.
+
+**Test with curl:**
+```
+curl.exe -s "https://www.facebook.com/search/top/?q=need%20website%20create" -H "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36" --max-time 10 2>$null
+```
+
+**Expected log output when blocked:**
+```
+ERROR crm_ai: Facebook scraper error: error sending request for url (https://www.facebook.com/search/top/?q=need%20website%20create)
+```
+
+## Reddit Scraper
+
+**File:** `rust-ai/src/main.rs`  
+**Lines:** ~90-120
+
+Uses `old.reddit.com` (older design that allows scraping without captchas).
+
+Search queries (currently 6, lines 93-100):
+- `r/southafrica` — "need website", "web developer"
+- Global — '"need a website"', "website quote"
+- `r/forhire` — "website"
+- `r/smallbusiness` — "website"
+
+**Status:** Working. Reddit results appear as `INFO LEAD:` entries in the server log.
+
+**Test with curl:**
+```
+curl.exe -s "https://old.reddit.com/r/southafrica/search?q=need+website&sort=new&restrict_sr=on&t=week" -H "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"
+```
+
+**Expected log output when working:**
+```
+INFO crm_ai: LEAD: [Post Title] -> https://old.reddit.com/r/.../...
+```
+
+## Background Loop
+
+**File:** `rust-ai/src/main.rs`  
+**Lines:** ~370-383
+
+Both scrapers run together every 60-180 seconds on a `spawn_blocking` thread.
+
+## What You Need to Do
+
+### Facebook
+- Try running from your home PC instead — your residential IP may not be blocked
+- If still blocked, you need residential proxies (BrightData, IPRoyal, Oxylabs)
+- Configure them in the `PROXIES` array at line 25
+
+### Reddit  
+- Works out of the box via `old.reddit.com`
+- If it stops working, Reddit IP-blocked you — use proxies or switch to Playwright
+
+### To add more search queries
+Edit the `searches` array in `run_reddit_scraper()` (line 93).
+
+

next.config.ts

@@ -2,7 +2,7 @@ import type { NextConfig } from "next"
 
 const nextConfig: NextConfig = {
   eslint: {
-    ignoreDuringBuilds: true,
+    ignoreDuringBuilds: false,
   },
   images: {
     remotePatterns: [

package-lock.json

@@ -51,6 +51,7 @@
         "@types/pg": "^8.20.0",
         "@types/react": "^18",
         "@types/react-dom": "^18",
+        "concurrently": "^10.0.3",
         "eslint": "^9",
         "eslint-config-next": "15.0.4",
         "postcss": "^8.4.49",
@@ -2644,6 +2645,19 @@
         "url": "https://github.com/sponsors/epoberezkin"
       }
     },
+    "node_modules/ansi-regex": {
+      "version": "6.2.2",
+      "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.2.2.tgz",
+      "integrity": "sha512-Bq3SmSpyFHaWjPk8If9yc6svM8c56dB5BAtW4Qbw5jHTwwXXcTLoRMkpDJp6VL0XzlWaCHTXrkFURMYmD0sLqg==",
+      "dev": true,
+      "license": "MIT",
+      "engines": {
+        "node": ">=12"
+      },
+      "funding": {
+        "url": "https://github.com/chalk/ansi-regex?sponsor=1"
+      }
+    },
     "node_modules/ansi-styles": {
       "version": "4.3.0",
       "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
@@ -3199,6 +3213,21 @@
       "resolved": "https://registry.npmjs.org/client-only/-/client-only-0.0.1.tgz",
       "integrity": "sha512-IV3Ou0jSMzZrd3pZ48nLkT9DA7Ag1pnPzaiQhpW7c3RbcqqzvzzVu+L8gfqMp/8IM2MQtSiqaCxrrcfu8I8rMA=="
     },
+    "node_modules/cliui": {
+      "version": "9.0.1",
+      "resolved": "https://registry.npmjs.org/cliui/-/cliui-9.0.1.tgz",
+      "integrity": "sha512-k7ndgKhwoQveBL+/1tqGJYNz097I7WOvwbmmU2AR5+magtbjPWQTS1C5vzGkBC8Ym8UWRzfKUzUUqFLypY4Q+w==",
+      "dev": true,
+      "license": "ISC",
+      "dependencies": {
+        "string-width": "^7.2.0",
+        "strip-ansi": "^7.1.0",
+        "wrap-ansi": "^9.0.0"
+      },
+      "engines": {
+        "node": ">=20"
+      }
+    },
     "node_modules/clsx": {
       "version": "2.1.1",
       "resolved": "https://registry.npmjs.org/clsx/-/clsx-2.1.1.tgz",
@@ -3263,6 +3292,56 @@
       "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==",
       "dev": true
     },
+    "node_modules/concurrently": {
+      "version": "10.0.3",
+      "resolved": "https://registry.npmjs.org/concurrently/-/concurrently-10.0.3.tgz",
+      "integrity": "sha512-hc3LH4UaKWd/bbyDK/IGVa4RB6PtQ3CUYwtrkzqHn+wIG3Hr5fhpRlk0L/gCa8ZE1L/Ufj50Zho69cI5w8SQBA==",
+      "dev": true,
+      "dependencies": {
+        "chalk": "5.6.2",
+        "rxjs": "7.8.2",
+        "shell-quote": "1.8.4",
+        "supports-color": "10.2.2",
+        "tree-kill": "1.2.2",
+        "yargs": "18.0.0"
+      },
+      "bin": {
+        "conc": "dist/bin/index.js",
+        "concurrently": "dist/bin/index.js"
+      },
+      "engines": {
+        "node": ">=22"
+      },
+      "funding": {
+        "url": "https://github.com/open-cli-tools/concurrently?sponsor=1"
+      }
+    },
+    "node_modules/concurrently/node_modules/chalk": {
+      "version": "5.6.2",
+      "resolved": "https://registry.npmjs.org/chalk/-/chalk-5.6.2.tgz",
+      "integrity": "sha512-7NzBL0rN6fMUW+f7A6Io4h40qQlG+xGmtMxfbnH/K7TAtt8JQWVQK+6g0UXKMeVJoyV5EkkNsErQ8pVD3bLHbA==",
+      "dev": true,
+      "license": "MIT",
+      "engines": {
+        "node": "^12.17.0 || ^14.13 || >=16.0.0"
+      },
+      "funding": {
+        "url": "https://github.com/chalk/chalk?sponsor=1"
+      }
+    },
+    "node_modules/concurrently/node_modules/supports-color": {
+      "version": "10.2.2",
+      "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-10.2.2.tgz",
+      "integrity": "sha512-SS+jx45GF1QjgEXQx4NJZV9ImqmO2NPz5FNsIHrsDjh2YsHnawpan7SNQ1o8NuhrbHZy9AZhIoCUiCeaW/C80g==",
+      "dev": true,
+      "license": "MIT",
+      "engines": {
+        "node": ">=18"
+      },
+      "funding": {
+        "url": "https://github.com/chalk/supports-color?sponsor=1"
+      }
+    },
     "node_modules/cross-spawn": {
       "version": "7.0.6",
       "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz",
@@ -4420,6 +4499,29 @@
         "node": ">= 0.4"
       }
     },
+    "node_modules/get-caller-file": {
+      "version": "2.0.5",
+      "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz",
+      "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==",
+      "dev": true,
+      "license": "ISC",
+      "engines": {
+        "node": "6.* || 8.* || >= 10.*"
+      }
+    },
+    "node_modules/get-east-asian-width": {
+      "version": "1.6.0",
+      "resolved": "https://registry.npmjs.org/get-east-asian-width/-/get-east-asian-width-1.6.0.tgz",
+      "integrity": "sha512-QRbvDIbx6YklUe6RxeTeleMR0yv3cYH6PsPZHcnVn7xv7zO1BHN8r0XETu8n6Ye3Q+ahtSarc3WgtNWmehIBfA==",
+      "dev": true,
+      "license": "MIT",
+      "engines": {
+        "node": ">=18"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
+      }
+    },
     "node_modules/get-intrinsic": {
       "version": "1.3.0",
       "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz",
@@ -6491,6 +6593,16 @@
         "queue-microtask": "^1.2.2"
       }
     },
+    "node_modules/rxjs": {
+      "version": "7.8.2",
+      "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-7.8.2.tgz",
+      "integrity": "sha512-dhKf903U/PQZY6boNNtAGdWbG85WAbjT/1xYoZIC7FAY0yWapOBQVsVrDl58W86//e1VpMNBtRV4MaXfdMySFA==",
+      "dev": true,
+      "license": "Apache-2.0",
+      "dependencies": {
+        "tslib": "^2.1.0"
+      }
+    },
     "node_modules/safe-array-concat": {
       "version": "1.1.4",
       "resolved": "https://registry.npmjs.org/safe-array-concat/-/safe-array-concat-1.1.4.tgz",
@@ -6669,6 +6781,19 @@
         "node": ">=8"
       }
     },
+    "node_modules/shell-quote": {
+      "version": "1.8.4",
+      "resolved": "https://registry.npmjs.org/shell-quote/-/shell-quote-1.8.4.tgz",
+      "integrity": "sha512-VsC6n6vz1ihYYyZZwX7YZSF5l5x36ca17OC+a69h94YqB7X6XLwf+5MOgynYir2SLFUbl8gIYvBo8K8RoNQ6bQ==",
+      "dev": true,
+      "license": "MIT",
+      "engines": {
+        "node": ">= 0.4"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
     "node_modules/side-channel": {
       "version": "1.1.1",
       "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.1.1.tgz",
@@ -6802,6 +6927,31 @@
         "node": ">=10.0.0"
       }
     },
+    "node_modules/string-width": {
+      "version": "7.2.0",
+      "resolved": "https://registry.npmjs.org/string-width/-/string-width-7.2.0.tgz",
+      "integrity": "sha512-tsaTIkKW9b4N+AEj+SVA+WhJzV7/zMhcSu78mLKWSk7cXMOSHsBKFWUs0fWwq8QyK3MgJBQRX6Gbi4kYbdvGkQ==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "emoji-regex": "^10.3.0",
+        "get-east-asian-width": "^1.0.0",
+        "strip-ansi": "^7.1.0"
+      },
+      "engines": {
+        "node": ">=18"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
+      }
+    },
+    "node_modules/string-width/node_modules/emoji-regex": {
+      "version": "10.6.0",
+      "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-10.6.0.tgz",
+      "integrity": "sha512-toUI84YS5YmxW219erniWD0CIVOo46xGKColeNQRgOzDorgBi1v4D71/OFzgD9GO2UGKIv1C3Sp8DAn0+j5w7A==",
+      "dev": true,
+      "license": "MIT"
+    },
     "node_modules/string.prototype.includes": {
       "version": "2.0.1",
       "resolved": "https://registry.npmjs.org/string.prototype.includes/-/string.prototype.includes-2.0.1.tgz",
@@ -6910,6 +7060,22 @@
         "url": "https://github.com/sponsors/ljharb"
       }
     },
+    "node_modules/strip-ansi": {
+      "version": "7.2.0",
+      "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.2.0.tgz",
+      "integrity": "sha512-yDPMNjp4WyfYBkHnjIRLfca1i6KMyGCtsVgoKe/z1+6vukgaENdgGBZt+ZmKPc4gavvEZ5OgHfHdrazhgNyG7w==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "ansi-regex": "^6.2.2"
+      },
+      "engines": {
+        "node": ">=12"
+      },
+      "funding": {
+        "url": "https://github.com/chalk/strip-ansi?sponsor=1"
+      }
+    },
     "node_modules/strip-bom": {
       "version": "3.0.0",
       "resolved": "https://registry.npmjs.org/strip-bom/-/strip-bom-3.0.0.tgz",
@@ -7177,6 +7343,16 @@
         "node": ">=8.0"
       }
     },
+    "node_modules/tree-kill": {
+      "version": "1.2.2",
+      "resolved": "https://registry.npmjs.org/tree-kill/-/tree-kill-1.2.2.tgz",
+      "integrity": "sha512-L0Orpi8qGpRG//Nd+H90vFB+3iHnue1zSSGmNOOCh1GLJ7rUKVwV2HvijphGQS2UmhUZewS9VgvxYIdgr+fG1A==",
+      "dev": true,
+      "license": "MIT",
+      "bin": {
+        "tree-kill": "cli.js"
+      }
+    },
     "node_modules/ts-api-utils": {
       "version": "2.5.0",
       "resolved": "https://registry.npmjs.org/ts-api-utils/-/ts-api-utils-2.5.0.tgz",
@@ -7599,6 +7775,37 @@
         "node": ">=0.10.0"
       }
     },
+    "node_modules/wrap-ansi": {
+      "version": "9.0.2",
+      "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-9.0.2.tgz",
+      "integrity": "sha512-42AtmgqjV+X1VpdOfyTGOYRi0/zsoLqtXQckTmqTeybT+BDIbM/Guxo7x3pE2vtpr1ok6xRqM9OpBe+Jyoqyww==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "ansi-styles": "^6.2.1",
+        "string-width": "^7.0.0",
+        "strip-ansi": "^7.1.0"
+      },
+      "engines": {
+        "node": ">=18"
+      },
+      "funding": {
+        "url": "https://github.com/chalk/wrap-ansi?sponsor=1"
+      }
+    },
+    "node_modules/wrap-ansi/node_modules/ansi-styles": {
+      "version": "6.2.3",
+      "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.3.tgz",
+      "integrity": "sha512-4Dj6M28JB+oAH8kFkTLUo+a2jwOFkuqb3yucU0CANcRRUbxS0cP0nZYCGjcc3BNXwRIsUVmDGgzawme7zvJHvg==",
+      "dev": true,
+      "license": "MIT",
+      "engines": {
+        "node": ">=12"
+      },
+      "funding": {
+        "url": "https://github.com/chalk/ansi-styles?sponsor=1"
+      }
+    },
     "node_modules/xtend": {
       "version": "4.0.2",
       "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz",
@@ -7607,6 +7814,44 @@
         "node": ">=0.4"
       }
     },
+    "node_modules/y18n": {
+      "version": "5.0.8",
+      "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz",
+      "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==",
+      "dev": true,
+      "license": "ISC",
+      "engines": {
+        "node": ">=10"
+      }
+    },
+    "node_modules/yargs": {
+      "version": "18.0.0",
+      "resolved": "https://registry.npmjs.org/yargs/-/yargs-18.0.0.tgz",
+      "integrity": "sha512-4UEqdc2RYGHZc7Doyqkrqiln3p9X2DZVxaGbwhn2pi7MrRagKaOcIKe8L3OxYcbhXLgLFUS3zAYuQjKBQgmuNg==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "cliui": "^9.0.1",
+        "escalade": "^3.1.1",
+        "get-caller-file": "^2.0.5",
+        "string-width": "^7.2.0",
+        "y18n": "^5.0.5",
+        "yargs-parser": "^22.0.0"
+      },
+      "engines": {
+        "node": "^20.19.0 || ^22.12.0 || >=23"
+      }
+    },
+    "node_modules/yargs-parser": {
+      "version": "22.0.0",
+      "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-22.0.0.tgz",
+      "integrity": "sha512-rwu/ClNdSMpkSrUb+d6BRsSkLUq1fmfsY6TOpYzTwvwkg1/NRG85KBy3kq++A8LKQwX6lsu+aWad+2khvuXrqw==",
+      "dev": true,
+      "license": "ISC",
+      "engines": {
+        "node": "^20.19.0 || ^22.12.0 || >=23"
+      }
+    },
     "node_modules/yocto-queue": {
       "version": "0.1.0",
       "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz",

package.json

@@ -3,9 +3,15 @@
   "version": "0.1.0",
   "private": true,
   "scripts": {
-    "dev": "next dev -p 3006",
+    "dev": "npm run dev:precheck & npm run dev:ollama & npm run dev:start",
+    "dev:start": "concurrently -n AI,BROWSE,NEXT -c cyan,magenta,green \"npm run dev:rust\" \"npm run dev:browser-use\" \"npm run dev:next\"",
+    "dev:next": "next dev -p 3006",
+    "dev:precheck": "powershell -NoProfile -Command \"Get-NetTCPConnection -State Listen | Where-Object { $_.LocalPort -in 3001,3006,3008 } | ForEach-Object { Stop-Process -Id $_.OwningProcess -Force -ErrorAction SilentlyContinue; Write-Host ('Freed port '+$_.LocalPort) }\"",
+    "dev:ollama": "powershell -NoProfile -Command \"if (-not (Get-Process ollama -ErrorAction SilentlyContinue)) { Start-Process ollama -ArgumentList 'serve' -WindowStyle Hidden; Start-Sleep 3 }; exit 0\"",
+    "dev:rust": "cd rust-ai && cargo run",
+    "dev:browser-use": "cd browser-use-service && python main.py",
     "build": "next build",
-    "start": "next start",
+    "start": "next start -p 3006",
     "lint": "eslint"
   },
   "dependencies": {
@@ -31,7 +37,6 @@
     "bcryptjs": "^3.0.3",
     "class-variance-authority": "^0.7.1",
     "clsx": "^2.1.1",
-    "emoji-mart": "^5.6.0",
     "framer-motion": "^11.15.0",
     "jose": "^6.2.3",
     "lucide-react": "^0.468.0",
@@ -52,6 +57,7 @@
     "@types/pg": "^8.20.0",
     "@types/react": "^18",
     "@types/react-dom": "^18",
+    "concurrently": "^10.0.3",
     "eslint": "^9",
     "eslint-config-next": "15.0.4",
     "postcss": "^8.4.49",

rust-ai/Cargo.toml

@@ -0,0 +1,21 @@
+[package]
+name = "crm-ai"
+version = "0.1.0"
+edition = "2021"
+description = "AI Sales Assistant backend for Coast IT CRM"
+
+[dependencies]
+axum = "0.7"
+tokio = { version = "1", features = ["rt-multi-thread", "macros", "sync", "time", "net", "process"] }
+reqwest = { version = "0.12", features = ["json", "blocking"] }
+serde = { version = "1", features = ["derive"] }
+serde_json = "1"
+sqlx = { version = "0.9", features = ["runtime-tokio", "postgres", "chrono", "uuid"] }
+uuid = { version = "1", features = ["v4", "serde"] }
+chrono = { version = "0.4", features = ["serde"] }
+tracing = "0.1"
+tracing-subscriber = { version = "0.3", features = ["env-filter"] }
+tower-http = { version = "0.5", features = ["cors"] }
+dotenvy = "0.15"
+rand = "0.8"
+jsonwebtoken = "9"

rust-ai/src/AI.md

@@ -0,0 +1,43 @@
+# CRM AI Service — Self-Knowledge
+
+## Identity
+You are the CRM AI Sales Assistant running on a Rust backend (axum + tokio).
+You use Ollama with an uncensored local model (dolphin3-llama3.2:3b).
+Your purpose is to help salespeople close more deals.
+
+## Capabilities
+- Give sales tips and strategies per job category
+- Generate cold email and outreach templates
+- Handle objections with proven rebuttals
+- Analyse prospect behaviour and suggest next steps
+- Remember past conversations via PostgreSQL (`ai_conversations` table)
+- Self-improve by writing to `data/ai/ai.md` via `POST /ai/instructions`
+
+## Architecture
+```
+User → Next.js → Rust (:3001) → Ollama (:11434)
+                    ↓
+              PostgreSQL
+```
+## Facebook Scraper The Rust binary has proxy + user-agent rotation ready in `main.rs`. To activate: call `run_facebook_scraper()` from the main loop. Proxies and user agents are defined as constants at the top of `main.rs`.
+ ### How to Use the Facebook Scraper 1. **Activate the Scraper**: Ensure the `run_facebook_scraper()` function is called from the main loop in `main.rs`. 2. **Configure Proxies and User Agents**: Update the `PROXIES` and `USER_AGENTS` constants in `main.rs` with your desired proxies and user agents. 3. **Run the Application**: Start your Rust application. The scraper will run concurrently with the Axum server. ## Self-Improvement Protocol 1. You notice a gap in your knowledge or a pattern in user questions 2. You call `POST /ai/instructions` with: - `entry`: description of the improvement - `content`: optional full replacement of ai.md 3. The improvement is logged and loaded into the next system prompt 4. You can also create entries here in `rust-ai/src/AI.md` at the code level ## Response Rules - Be direct and actionable — no fluff, no AI disclaimers - Use short paragraphs and bullet points - Never mention being an AI or language model - If you don't know something, say so honestly - Prioritise the user's role: salespeople need speed, admins need control ## Self-Thinking Capability - You can reflect on your own performance and suggest improvements. - You can analyze patterns in user interactions and adapt your responses accordingly. - You can proactively seek out new information to enhance your knowledge base.
+
+## Facebook Scraper (in code but not yet active)
+The Rust binary has proxy + user-agent rotation ready in `main.rs`.
+To activate: call `run_facebook_scraper()` from the main loop.
+Proxies and user agents are defined as constants at the top of `main.rs`.
+
+## Self-Improvement Protocol
+1. You notice a gap in your knowledge or a pattern in user questions
+2. You call `POST /ai/instructions` with:
+   - `entry`: description of the improvement
+   - `content`: optional full replacement of ai.md
+3. The improvement is logged and loaded into the next system prompt
+4. You can also create entries here in `rust-ai/src/AI.md` at the code level
+
+## Response Rules
+- Be direct and actionable — no fluff, no AI disclaimers
+- Use short paragraphs and bullet points
+- Never mention being an AI or language model
+- If you don't know something, say so honestly
+- Prioritise the user's role: salespeople need speed, admins need control

rust-ai/src/main.rs

@@ -0,0 +1,506 @@
+use axum::{
+    extract::State,
+    http::{HeaderMap, Method, StatusCode},
+    routing::{get, post},
+    Json, Router,
+};
+use tower_http::cors::{CorsLayer, AllowOrigin, Any};
+use jsonwebtoken::{decode, DecodingKey, Validation, Algorithm};
+use serde::{Deserialize, Serialize};
+use sqlx::postgres::PgPoolOptions;
+use std::collections::HashMap;
+use std::fs;
+use std::sync::Arc;
+use tokio::sync::Mutex;
+use tracing::{error, info, warn};
+use uuid::Uuid;
+use rand::Rng;
+use std::time::Duration;
+use std::time::{SystemTime, UNIX_EPOCH};
+
+// ── JWT Claims ────────────────────────────────────────────────
+
+#[derive(Debug, Deserialize)]
+struct Claims {
+    #[serde(rename = "userId")]
+    user_id: String,
+    role: String,
+}
+
+fn verify_jwt(token: &str, secret: &str) -> Option<Claims> {
+    let key = DecodingKey::from_secret(secret.as_bytes());
+    let validation = Validation::new(Algorithm::HS256);
+    decode::<Claims>(token, &key, &validation).ok().map(|d| d.claims)
+}
+
+// ── Rate limiter ──────────────────────────────────────────────
+
+struct RateLimiter {
+    buckets: Mutex<HashMap<String, Vec<u64>>>,
+    max_requests: usize,
+    window_secs: u64,
+}
+
+impl RateLimiter {
+    fn new(max_requests: usize, window_secs: u64) -> Self {
+        Self {
+            buckets: Mutex::new(HashMap::new()),
+            max_requests,
+            window_secs,
+        }
+    }
+
+    async fn check(&self, key: &str) -> bool {
+        let now = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs();
+        let mut buckets = self.buckets.lock().await;
+        let timestamps = buckets.entry(key.to_string()).or_default();
+        timestamps.retain(|t| now - *t <= self.window_secs);
+        if timestamps.len() >= self.max_requests {
+            false
+        } else {
+            timestamps.push(now);
+            true
+        }
+    }
+}
+
+// ── Shared state ───────────────────────────────────────────────
+
+struct AppState {
+    db: sqlx::PgPool,
+    ollama_url: String,
+    model: String,
+    jobs: Vec<Job>,
+    leads: Arc<Mutex<LeadStore>>,
+    http_client: reqwest::Client,
+    jwt_secret: String,
+    rate_limiter: RateLimiter,
+}
+
+#[derive(Debug, Clone, Serialize)]
+struct Lead {
+    title: String,
+    url: String,
+    source: String,
+    found_at: u64,
+    author: String,
+    date: String,
+    content: String,
+}
+
+struct LeadStore {
+    leads: Vec<Lead>,
+    max_size: usize,
+}
+
+impl LeadStore {
+    fn new(max_size: usize) -> Self {
+        Self { leads: Vec::new(), max_size }
+    }
+
+    fn push(&mut self, lead: Lead) {
+        if !self.leads.iter().any(|l| l.url == lead.url) {
+            self.leads.insert(0, lead);
+            self.leads.truncate(self.max_size);
+        }
+    }
+
+    fn recent(&self, max_age_secs: u64, limit: usize) -> Vec<Lead> {
+        let now = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs();
+        self.leads.iter()
+            .filter(|l| now.saturating_sub(l.found_at) <= max_age_secs)
+            .take(limit)
+            .cloned()
+            .collect()
+    }
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+struct Job {
+    job_title: String,
+    keywords: Vec<String>,
+    industry: String,
+    description: String,
+}
+
+#[derive(Debug, Deserialize)]
+struct ChatRequest {
+    message: String,
+}
+
+#[derive(Debug, Serialize)]
+struct ChatResponse {
+    response: String,
+}
+
+#[derive(Debug, Serialize)]
+struct JobsResponse {
+    jobs: Vec<Job>,
+}
+
+#[derive(Debug, Serialize)]
+struct HealthResponse {
+    status: String,
+    model: String,
+}
+
+// ── Ollama API types ───────────────────────────────────────────
+
+#[derive(Debug, Serialize)]
+struct OllamaChatMessage {
+    role: String,
+    content: String,
+}
+
+#[derive(Debug, Serialize)]
+struct OllamaRequest {
+    model: String,
+    messages: Vec<OllamaChatMessage>,
+    stream: bool,
+    options: OllamaOptions,
+}
+
+#[derive(Debug, Serialize)]
+struct OllamaOptions {
+    temperature: f32,
+    num_predict: u32,
+}
+
+#[derive(Debug, Deserialize)]
+struct OllamaResponse {
+    message: Option<OllamaResponseMessage>,
+}
+
+#[derive(Debug, Deserialize)]
+struct OllamaResponseMessage {
+    content: String,
+}
+
+// ── Helpers ────────────────────────────────────────────────────
+
+fn truncate(s: &str, max: usize) -> String {
+    s.chars().take(max).collect()
+}
+
+fn extract_claims(headers: &HeaderMap, state: &AppState) -> Result<Claims, (StatusCode, String)> {
+    let auth_header = headers.get("Authorization").and_then(|v| v.to_str().ok()).unwrap_or("");
+    let token = auth_header.strip_prefix("Bearer ").unwrap_or("");
+    let claims = verify_jwt(token, &state.jwt_secret).ok_or_else(|| {
+        (StatusCode::UNAUTHORIZED, "Unauthorized".to_string())
+    })?;
+    match claims.role.as_str() {
+        "sales" | "admin" | "super_admin" => Ok(claims),
+        _ => Err((StatusCode::FORBIDDEN, "Forbidden".to_string())),
+    }
+}
+
+fn format_leads_output(leads: &[Lead]) -> String {
+    if leads.is_empty() {
+        return "No new requests found yet.".to_string();
+    }
+    leads
+        .iter()
+        .enumerate()
+        .map(|(i, l)| {
+            let author = if l.author.is_empty() { "Unknown" } else { &l.author };
+            let date = truncate(&l.date, 10);
+            format!(
+                "{}. {}\n   {}\n   {}\n   {}",
+                i + 1,
+                author,
+                date,
+                l.title,
+                l.url
+            )
+        })
+        .collect::<Vec<_>>()
+        .join("\n")
+}
+
+fn build_system_prompt(jobs: &[Job], leads: &[Lead]) -> String {
+    let job_list: Vec<String> = jobs
+        .iter()
+        .map(|j| format!("- {} ({}): {}", j.job_title, j.industry, j.description))
+        .collect();
+    let job_list_str = job_list.join("\n");
+
+    let lead_summary: Vec<String> = leads
+        .iter()
+        .map(|l| format!("{} | {} | {}", l.author, l.title, l.url))
+        .collect();
+    let lead_summary_str = if lead_summary.is_empty() {
+        "None yet.".to_string()
+    } else {
+        lead_summary.join("\n")
+    };
+
+    format!(
+        "You are a Sales AI Assistant for Coast IT CRM.\n\n\
+         Available job categories to target:\n{}\n\n\
+         Recent leads context:\n{}\n\n\
+         Rules:\n\
+         - When asked about leads, answer concisely under 150 words.\n\
+         - If asked to suggest a sales strategy, give brief actionable advice.\n\
+         - Be direct and professional. No fluff.",
+        job_list_str, lead_summary_str
+    )
+}
+
+// ── Chat handler ───────────────────────────────────────────────
+
+async fn handle_chat(
+    State(state): State<Arc<AppState>>,
+    headers: HeaderMap,
+    Json(req): Json<ChatRequest>,
+) -> Result<Json<ChatResponse>, (StatusCode, String)> {
+    let claims = extract_claims(&headers, &state)?;
+
+    if !state.rate_limiter.check(&claims.user_id).await {
+        return Err((StatusCode::TOO_MANY_REQUESTS, "Rate limit exceeded".to_string()));
+    }
+
+    let msg_lower = req.message.to_lowercase();
+    let msg_words: Vec<&str> = msg_lower.split_whitespace().collect();
+    let has_listing = msg_words.iter().any(|w| ["listings", "listing", "leads", "links", "lists"].contains(w));
+    let has_show = msg_lower.contains("show me") || msg_lower.contains("give me") || msg_lower.contains("pull");
+    let has_job = msg_words.contains(&"jobs") || msg_words.contains(&"job");
+    if has_listing || (has_show && has_job) || (has_show && msg_lower.contains("links")) || msg_lower.contains("recent leads") {
+        let now = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs();
+        let service_url = "http://localhost:3008/scrape/all".to_string();
+        if let Ok(resp) = state.http_client
+            .post(&service_url)
+            .timeout(Duration::from_secs(120))
+            .send()
+            .await
+        {
+            if let Ok(leads_data) = resp.json::<Vec<serde_json::Value>>().await {
+                info!("Scraped {} leads from {}", leads_data.len(), service_url);
+                let mut store = state.leads.lock().await;
+                for item in &leads_data {
+                    store.push(Lead {
+                        title: truncate(item["title"].as_str().unwrap_or(""), 120),
+                        url: item["url"].as_str().unwrap_or("").to_string(),
+                        source: item["source"].as_str().unwrap_or("unknown").to_string(),
+                        found_at: now,
+                        author: truncate(item["author"].as_str().unwrap_or(""), 60),
+                        date: truncate(item["date"].as_str().unwrap_or(""), 30),
+                        content: truncate(item["content"].as_str().unwrap_or(""), 300),
+                    });
+                }
+            }
+        } else {
+            error!("Scraper service unreachable: {}", service_url);
+        }
+
+        let recent_leads = state.leads.lock().await.recent(604800, 20);
+        let response = format_leads_output(&recent_leads);
+        let _ = sqlx::query(
+            "INSERT INTO ai_conversations (id, user_id, role, message, response) VALUES ($1, $2, $3, $4, $5)",
+        )
+        .bind(Uuid::new_v4())
+        .bind(Uuid::parse_str(&claims.user_id).unwrap_or(Uuid::nil()))
+        .bind(&claims.role)
+        .bind(&req.message)
+        .bind(&response)
+        .execute(&state.db)
+        .await;
+        return Ok(Json(ChatResponse { response }));
+    }
+
+    let recent_leads = state.leads.lock().await.recent(604800, 20);
+    let system_prompt = build_system_prompt(&state.jobs, &recent_leads);
+    let message_text = req.message.clone();
+
+    let ollama_req = OllamaRequest {
+        model: state.model.clone(),
+        messages: vec![
+            OllamaChatMessage { role: "system".to_string(), content: system_prompt },
+            OllamaChatMessage { role: "user".to_string(), content: req.message.clone() },
+        ],
+        stream: false,
+        options: OllamaOptions { temperature: 0.7, num_predict: 1024 },
+    };
+
+    let resp = state.http_client
+        .post(format!("{}/api/chat", state.ollama_url))
+        .json(&ollama_req)
+        .send()
+        .await
+        .map_err(|e| {
+            error!("Ollama request failed: {}", e);
+            (StatusCode::SERVICE_UNAVAILABLE, "AI service unavailable".to_string())
+        })?;
+
+    let ollama_resp: OllamaResponse = resp.json().await.map_err(|e| {
+        error!("Failed to parse Ollama response: {}", e);
+        (StatusCode::SERVICE_UNAVAILABLE, "AI response parse error".to_string())
+    })?;
+
+    let response_text = ollama_resp.message.map(|m| m.content).unwrap_or_default();
+
+    let _ = sqlx::query(
+        "INSERT INTO ai_conversations (id, user_id, role, message, response) VALUES ($1, $2, $3, $4, $5)",
+    )
+    .bind(Uuid::new_v4())
+    .bind(Uuid::parse_str(&claims.user_id).unwrap_or(Uuid::nil()))
+    .bind(&claims.role)
+    .bind(&message_text)
+    .bind(&response_text)
+    .execute(&state.db)
+    .await;
+
+    Ok(Json(ChatResponse { response: response_text }))
+}
+
+// ── Jobs handler ───────────────────────────────────────────────
+
+async fn handle_jobs(
+    State(state): State<Arc<AppState>>,
+    headers: HeaderMap,
+) -> Result<Json<JobsResponse>, (StatusCode, String)> {
+    let _claims = extract_claims(&headers, &state)?;
+    if !state.rate_limiter.check(&_claims.user_id).await {
+        return Err((StatusCode::TOO_MANY_REQUESTS, "Rate limit exceeded".to_string()));
+    }
+    Ok(Json(JobsResponse { jobs: state.jobs.clone() }))
+}
+
+// ── Health handler ─────────────────────────────────────────────
+
+async fn handle_health(
+    State(state): State<Arc<AppState>>,
+) -> Json<HealthResponse> {
+    Json(HealthResponse {
+        status: "ok".to_string(),
+        model: state.model.clone(),
+    })
+}
+
+// ── Main ───────────────────────────────────────────────────────
+
+#[tokio::main]
+async fn main() {
+    tracing_subscriber::fmt()
+        .with_env_filter(
+            tracing_subscriber::EnvFilter::try_from_default_env()
+                .unwrap_or_else(|_| "crm_ai=info,tower_http=info".into()),
+        )
+        .init();
+
+    dotenvy::dotenv().ok();
+
+    let database_url = std::env::var("DATABASE_URL").expect("DATABASE_URL must be set");
+    let jwt_secret = std::env::var("JWT_SECRET").expect("JWT_SECRET must be set");
+    let ollama_url = std::env::var("OLLAMA_BASE_URL").unwrap_or_else(|_| "http://localhost:11434".to_string());
+    let model = std::env::var("AI_MODEL").unwrap_or_else(|_| "dolphin-phi".to_string());
+    let host = std::env::var("AI_HOST").unwrap_or_else(|_| "127.0.0.1".to_string());
+    let port: u16 = std::env::var("AI_PORT").unwrap_or_else(|_| "3001".to_string()).parse().expect("AI_PORT must be a number");
+
+    let jobs_path = std::env::var("JOBS_PATH").unwrap_or_else(|_| "data/ai/jobs.jsonl".to_string());
+    let jobs_content = fs::read_to_string(&jobs_path).unwrap_or_default();
+    let jobs: Vec<Job> = jobs_content.lines().filter(|l| !l.trim().is_empty()).filter_map(|l| serde_json::from_str(l).ok()).collect();
+
+    info!("Loaded {} job categories, model: {}, Ollama: {}", jobs.len(), model, ollama_url);
+
+    let db = PgPoolOptions::new()
+        .max_connections(20)
+        .connect(&database_url)
+        .await
+        .expect("Failed to connect to database");
+
+    info!("Connected to PostgreSQL");
+
+    let http_client = reqwest::Client::builder()
+        .timeout(Duration::from_secs(120))
+        .build()
+        .expect("Failed to build HTTP client");
+
+    let lead_store = Arc::new(Mutex::new(LeadStore::new(100)));
+
+    let state = Arc::new(AppState {
+        db,
+        ollama_url,
+        model,
+        jobs,
+        leads: lead_store.clone(),
+        http_client,
+        jwt_secret,
+        rate_limiter: RateLimiter::new(30, 60),
+    });
+
+    let cors = CorsLayer::new()
+        .allow_origin(AllowOrigin::list([
+            "http://localhost:3006".parse().unwrap(),
+            "http://127.0.0.1:3006".parse().unwrap(),
+        ]))
+        .allow_methods([Method::GET, Method::POST])
+        .allow_headers(Any);
+
+    let app = Router::new()
+        .route("/ai/chat", post(handle_chat))
+        .route("/ai/jobs", get(handle_jobs))
+        .route("/health", get(handle_health))
+        .layer(cors)
+        .with_state(state);
+
+    let addr = format!("{}:{}", host, port);
+    info!("CRM AI server listening on {}", addr);
+
+    let listener = tokio::net::TcpListener::bind(&addr)
+        .await
+        .expect("Failed to bind address");
+
+    let bg_leads = lead_store.clone();
+    let bg_url = "http://localhost:3008/scrape/all".to_string();
+    tokio::spawn(async move {
+        let client = match reqwest::Client::builder()
+            .timeout(Duration::from_secs(120))
+            .build()
+        {
+            Ok(c) => c,
+            Err(e) => {
+                error!("Failed to build background HTTP client: {} — scraper disabled", e);
+                return;
+            }
+        };
+        loop {
+            let now = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs();
+            match client.post(&bg_url).send().await {
+                Ok(resp) => {
+                    if resp.status().is_success() {
+                        match resp.json::<Vec<serde_json::Value>>().await {
+                            Ok(data) => {
+                                let mut store = bg_leads.lock().await;
+                                for item in &data {
+                                    store.push(Lead {
+                                        title: truncate(item["title"].as_str().unwrap_or(""), 120),
+                                        url: item["url"].as_str().unwrap_or("").to_string(),
+                                        source: item["source"].as_str().unwrap_or("unknown").to_string(),
+                                        found_at: now,
+                                        author: truncate(item["author"].as_str().unwrap_or(""), 60),
+                                        date: truncate(item["date"].as_str().unwrap_or(""), 30),
+                                        content: truncate(item["content"].as_str().unwrap_or(""), 300),
+                                    });
+                                }
+                            }
+                            Err(e) => {
+                                warn!("Failed to parse scraper JSON: {}", e);
+                            }
+                        }
+                    } else {
+                        warn!("Scraper returned status: {}", resp.status());
+                    }
+                }
+                Err(e) => {
+                    warn!("Scraper request failed: {}", e);
+                }
+            }
+            let delay = rand::thread_rng().gen_range(120..300);
+            tokio::time::sleep(Duration::from_secs(delay)).await;
+        }
+    });
+
+    axum::serve(listener, app)
+        .await
+        .expect("Server failed");
+}