caitlin/CRM_ENVR
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Compare commits

base: caitlin/CRM_ENVR:main
Branches Tags
caitlin/CRM_ENVR:main
..
compare: caitlin/CRM_ENVR:9556b67656a7bc7a0060014c25f8f114fe6245c4
Branches Tags
caitlin/CRM_ENVR:main

7 Commits
main .. 9556b67656

Author SHA1 Message Date
Ace 9556b67656 AH 2026-06-22 13:39:10 +02:00
Ace b59cc65508 changing directory for the AI search 2026-06-22 13:35:06 +02:00
Hannah_Bagga f503bf98f8 Merge branch 'main' of https://git.coastit.co.za/caitlin/CRM_ENVR 2026-06-22 13:10:01 +02:00
Ace 6217634c41 pakage update 2026-06-22 13:03:39 +02:00
Ace 2e652266b6 Forgot this im getting old 2026-06-22 12:47:24 +02:00
Ace 96a4323a41 Forgot this added for AI 2026-06-22 12:43:39 +02:00
Ace 0d61c9cff2 AI somewhat added 2026-06-22 12:37:38 +02:00
5961 changed files with 1974 additions and 443259 deletions
Expand all files Collapse all files
.gitattributes
-1
View File
@@ -1 +0,0 @@
rust-ai/target/** linguist-generated=true -diff -merge
.gitea/workflows/build-and-repair.yml
-58
View File
@@ -1,58 +0,0 @@
name: Build & Auto-Repair
on:
push:
branches: [main, develop]
pull_request:
branches: [main]
jobs:
build:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Setup Node.js
uses: actions/setup-node@v4
with:
node-version: "20"
- name: Install dependencies
run: npm ci --no-audit --no-fund
- name: Install Playwright
run: npx playwright install chromium firefox
- name: Build check
id: build
continue-on-error: true
run: npx tsc --noEmit
- name: Run self-healing setup
if: steps.build.outcome == 'failure'
run: node scripts/setup.mjs --self-heal
- name: Run code repair agent
if: steps.build.outcome == 'failure'
env:
OLLAMA_HOST: ${{ vars.OLLAMA_HOST || 'http://localhost:11434' }}
run: node scripts/code-repair-agent.mjs --ci
- name: Re-check build after repair
id: rebuild
continue-on-error: true
run: npx tsc --noEmit
- name: Commit fixes
if: steps.build.outcome == 'failure' && steps.rebuild.outcome == 'success'
run: |
git config user.name "CRM Repair Bot"
git config user.email "bot@coastit.co.za"
git add -A
git diff --cached --quiet || git commit -m "[bot] Auto-repair build errors"
git push
- name: Build failed — report
if: steps.rebuild.outcome == 'failure'
run: |
echo "::error::Build still failing after auto-repair. Manual intervention required."
exit 1
.gitignore
-24
View File
@@ -34,33 +34,9 @@ yarn-error.log*
# env files (can opt-in for committing if needed)
.env*
# runtime data
data/
data/ai/
data/ai/jobs.jsonl
# logs
*.log
*.out
error-log
# vercel
.vercel
# python
browser-use-service/venv/
browser-use-service/__pycache__/
browser-use-service/.env
# typescript
*.tsbuildinfo
next-env.d.ts
.git
.git_bak
node_modules
target
# rust build artifacts
rust-ai/target/
# browser-use-service generated files
browser-use-service/*.txt
.setup-templates/registry.json
-42
View File
@@ -1,42 +0,0 @@
{
"version": 2,
"description": "Template registry for self-healing setup. Maps internal import paths to templates.",
"templates": {
"data/stickers": {
"template": "stickers.ts",
"description": "Emoji-based sticker packs for media picker"
},
"data/constants": {
"template": "constants.ts",
"description": "App-wide constants (lead statuses, roles, event types)"
},
"lib/utils": {
"template": "utils.ts",
"description": "Utility functions (cn, formatters, helpers)"
},
"types/index": {
"template": "types-index.ts",
"description": "Core type definitions (User, Lead, DashboardStats)"
},
"hooks/use-media": {
"template": "hooks/use-media.ts",
"description": "Media query hook"
},
"hooks/use-debounce": {
"template": "hooks/use-debounce.ts",
"description": "Debounce hook"
},
"hooks/use-local-storage": {
"template": "hooks/use-local-storage.ts",
"description": "LocalStorage hook with SSR safety"
}
},
"fallbackTemplates": {
"@/data/*": "data-generic.ts",
"@/lib/*": "lib-generic.ts",
"@/hooks/*": "hook-generic.ts",
"@/types/*": "types-generic.ts",
"@/components/*": "component-generic.tsx",
"@/utils/*": "utils-generic.ts"
}
}
.setup-templates/templates/component-generic.tsx
-7
View File
@@ -1,7 +0,0 @@
// ── Generic Component Template ────────────────────────────────────────
// Auto-generated by self-healing setup.
// Placeholder for @/components/* imports. Customize as needed.
export default function GenericComponent() {
return <div />;
}
.setup-templates/templates/constants.ts
-54
View File
@@ -1,54 +0,0 @@
// ── App Constants ────────────────────────────────────────────────────
// Auto-generated by self-healing setup. Edit .setup-templates/templates/constants.ts to customize.
export const APP_NAME = "CoastIT CRM";
export const APP_VERSION = "1.0.0";
export const LEAD_STATUSES = {
new: { label: "New", color: "bg-blue-500" },
contacted: { label: "Contacted", color: "bg-yellow-500" },
qualified: { label: "Qualified", color: "bg-purple-500" },
proposal: { label: "Proposal", color: "bg-orange-500" },
won: { label: "Won", color: "bg-green-500" },
lost: { label: "Lost", color: "bg-red-500" },
} as const;
export const USER_ROLES = {
super_admin: { label: "Super Admin", level: 1 },
admin: { label: "Admin", level: 2 },
sales: { label: "Sales", level: 3 },
dev: { label: "Developer", level: 4 },
} as const;
export const EVENT_TYPES = [
"meeting",
"call",
"email",
"task",
"note",
] as const;
export const EVENT_STATUSES = [
"scheduled",
"completed",
"cancelled",
"rescheduled",
] as const;
export const AI_MODELS = {
chat: "llama3.2:3b",
scraper: "dolphin-llama3:8b",
coder: "qwen2.5-coder:1.5b-base",
} as const;
export const PAGINATION = {
defaultLimit: 50,
maxLimit: 200,
} as const;
export const DATE_FORMATS = {
short: "MMM d, yyyy",
long: "MMMM d, yyyy",
time: "h:mm a",
datetime: "MMM d, yyyy h:mm a",
} as const;
.setup-templates/templates/data-generic.ts
-17
View File
@@ -1,17 +0,0 @@
// ── Generic Data Module Template ─────────────────────────────────────
// Auto-generated by self-healing setup.
// Placeholder for @/data/* imports. Customize as needed.
export interface DataItem {
id: string;
name: string;
[key: string]: unknown;
}
export function getDataItems(): DataItem[] {
return [];
}
export function getDataItem(id: string): DataItem | undefined {
return undefined;
}
.setup-templates/templates/hook-generic.ts
-10
View File
@@ -1,10 +0,0 @@
// ── Generic Hook Template ────────────────────────────────────────────
// Auto-generated by self-healing setup.
// Placeholder for @/hooks/* imports. Customize as needed.
import { useState, useEffect } from "react";
export function useHook<T>(initialValue: T): [T, (value: T) => void] {
const [value, setValue] = useState(initialValue);
return [value, setValue];
}
.setup-templates/templates/hooks/use-debounce.ts
-15
View File
@@ -1,15 +0,0 @@
// ── useDebounce Hook ──────────────────────────────────────────────────
// Auto-generated by self-healing setup. Edit to customize.
import { useState, useEffect } from "react";
export function useDebounce<T>(value: T, delay: number): T {
const [debouncedValue, setDebouncedValue] = useState(value);
useEffect(() => {
const timer = setTimeout(() => setDebouncedValue(value), delay);
return () => clearTimeout(timer);
}, [value, delay]);
return debouncedValue;
}
.setup-templates/templates/hooks/use-local-storage.ts
-29
View File
@@ -1,29 +0,0 @@
// ── useLocalStorage Hook ──────────────────────────────────────────────
// Auto-generated by self-healing setup. Edit to customize.
import { useState, useEffect } from "react";
export function useLocalStorage<T>(key: string, initialValue: T): [T, (value: T | ((prev: T) => T)) => void] {
const [storedValue, setStoredValue] = useState<T>(initialValue);
useEffect(() => {
try {
const item = window.localStorage.getItem(key);
if (item) setStoredValue(JSON.parse(item));
} catch {
// corrupted data — use initial value
}
}, [key]);
const setValue = (value: T | ((prev: T) => T)) => {
try {
const valueToStore = value instanceof Function ? value(storedValue) : value;
setStoredValue(valueToStore);
window.localStorage.setItem(key, JSON.stringify(valueToStore));
} catch {
// storage full or disabled
}
};
return [storedValue, setValue];
}
.setup-templates/templates/hooks/use-media.ts
-18
View File
@@ -1,18 +0,0 @@
// ── useMedia Hook ─────────────────────────────────────────────────────
// Auto-generated by self-healing setup. Edit to customize.
import { useState, useEffect } from "react";
export function useMedia(query: string): boolean {
const [matches, setMatches] = useState(false);
useEffect(() => {
const mql = window.matchMedia(query);
setMatches(mql.matches);
const handler = (e: MediaQueryListEvent) => setMatches(e.matches);
mql.addEventListener("change", handler);
return () => mql.removeEventListener("change", handler);
}, [query]);
return matches;
}
.setup-templates/templates/lib-generic.ts
-7
View File
@@ -1,7 +0,0 @@
// ── Generic Lib Module Template ──────────────────────────────────────
// Auto-generated by self-healing setup.
// Placeholder for @/lib/* imports. Customize as needed.
export function libFunction(): void {
// Placeholder
}
.setup-templates/templates/stickers.ts
-95
View File
@@ -1,95 +0,0 @@
// ── Sticker Packs ─────────────────────────────────────────────────
// Auto-generated by self-healing setup. Edit .setup-templates/templates/stickers.ts to customize.
export interface Sticker {
id: string;
name: string;
emoji?: string;
svg?: string;
}
export interface StickerPack {
id: string;
name: string;
stickers: Sticker[];
}
const reactionStickers: Sticker[] = [
{ id: "thumbs-up", name: "Thumbs Up", emoji: "👍" },
{ id: "thumbs-down", name: "Thumbs Down", emoji: "👎" },
{ id: "heart", name: "Heart", emoji: "❤️" },
{ id: "laugh", name: "Laugh", emoji: "😂" },
{ id: "wow", name: "Wow", emoji: "😮" },
{ id: "sad", name: "Sad", emoji: "😢" },
{ id: "angry", name: "Angry", emoji: "😡" },
{ id: "clap", name: "Clap", emoji: "👏" },
{ id: "fire", name: "Fire", emoji: "🔥" },
{ id: "100", name: "100", emoji: "💯" },
{ id: "eyes", name: "Eyes", emoji: "👀" },
{ id: "pray", name: "Pray", emoji: "🙏" },
];
const animalStickers: Sticker[] = [
{ id: "cat", name: "Cat", emoji: "🐱" },
{ id: "dog", name: "Dog", emoji: "🐶" },
{ id: "panda", name: "Panda", emoji: "🐼" },
{ id: "fox", name: "Fox", emoji: "🦊" },
{ id: "bear", name: "Bear", emoji: "🐻" },
{ id: "koala", name: "Koala", emoji: "🐨" },
{ id: "tiger", name: "Tiger", emoji: "🐯" },
{ id: "lion", name: "Lion", emoji: "🦁" },
{ id: "monkey", name: "Monkey", emoji: "🐵" },
{ id: "frog", name: "Frog", emoji: "🐸" },
{ id: "penguin", name: "Penguin", emoji: "🐧" },
{ id: "bird", name: "Bird", emoji: "🐦" },
];
const foodStickers: Sticker[] = [
{ id: "pizza", name: "Pizza", emoji: "🍕" },
{ id: "burger", name: "Burger", emoji: "🍔" },
{ id: "taco", name: "Taco", emoji: "🌮" },
{ id: "sushi", name: "Sushi", emoji: "🍣" },
{ id: "ramen", name: "Ramen", emoji: "🍜" },
{ id: "ice-cream", name: "Ice Cream", emoji: "🍦" },
{ id: "cake", name: "Cake", emoji: "🍰" },
{ id: "coffee", name: "Coffee", emoji: "☕" },
{ id: "beer", name: "Beer", emoji: "🍺" },
{ id: "wine", name: "Wine", emoji: "🍷" },
{ id: "donut", name: "Donut", emoji: "🍩" },
{ id: "cookie", name: "Cookie", emoji: "🍪" },
];
const activityStickers: Sticker[] = [
{ id: "football", name: "Football", emoji: "⚽" },
{ id: "basketball", name: "Basketball", emoji: "🏀" },
{ id: "tennis", name: "Tennis", emoji: "🎾" },
{ id: "running", name: "Running", emoji: "🏃" },
{ id: "swimming", name: "Swimming", emoji: "🏊" },
{ id: "cycling", name: "Cycling", emoji: "🚴" },
{ id: "gym", name: "Gym", emoji: "🏋️" },
{ id: "yoga", name: "Yoga", emoji: "🧘" },
{ id: "music", name: "Music", emoji: "🎵" },
{ id: "party", name: "Party", emoji: "🎉" },
{ id: "gift", name: "Gift", emoji: "🎁" },
{ id: "trophy", name: "Trophy", emoji: "🏆" },
];
const stickerPacks: StickerPack[] = [
{ id: "reactions", name: "Reactions", stickers: reactionStickers },
{ id: "animals", name: "Animals", stickers: animalStickers },
{ id: "food", name: "Food & Drink", stickers: foodStickers },
{ id: "activities", name: "Activities", stickers: activityStickers },
];
export function getStickerPacks(): StickerPack[] {
return stickerPacks;
}
export function getStickerPack(id: string): StickerPack | undefined {
return stickerPacks.find((p) => p.id === id);
}
export function getSticker(packId: string, stickerId: string): Sticker | undefined {
const pack = getStickerPack(packId);
return pack?.stickers.find((s) => s.id === stickerId);
}
.setup-templates/templates/types-generic.ts
-10
View File
@@ -1,10 +0,0 @@
// ── Generic Types Template ────────────────────────────────────────────
// Auto-generated by self-healing setup.
// Placeholder for @/types/* imports. Customize as needed.
export type GenericType = Record<string, unknown>;
export interface GenericInterface {
id: string;
[key: string]: unknown;
}
.setup-templates/templates/types-index.ts
-141
View File
@@ -1,141 +0,0 @@
// ── Type Definitions ──────────────────────────────────────────────────
// Auto-generated by self-healing setup. Edit .setup-templates/templates/types-index.ts to customize.
export type UserRole = "super_admin" | "admin" | "sales" | "dev";
export interface User {
id: string;
name: string;
email: string;
phone?: string;
role: UserRole;
active: boolean;
avatar: string;
createdAt: string;
}
export interface Lead {
id: string;
companyName: string;
contactName: string;
email: string;
phone: string;
source: string;
description: string;
status: LeadStatus;
assignedUserId: string | null;
assignedUser: User | null;
createdAt: string;
updatedAt: string;
}
export type LeadStatus =
| "new"
| "contacted"
| "qualified"
| "proposal"
| "won"
| "lost";
export interface DashboardStats {
totalLeads: number;
newLeads: number;
contactedLeads: number;
qualifiedLeads: number;
proposalLeads: number;
wonLeads: number;
lostLeads: number;
conversionRate: number;
monthlyBreakdown: {
label: string;
total: number;
new: number;
contacted: number;
qualified: number;
proposal: number;
won: number;
lost: number;
}[];
leadsPerMonth: { label: string; leads: number; won: number }[];
recentLeads: Lead[];
statusDistribution: { name: string; value: number; color: string }[];
periodLabel: string;
trends: Record<string, { pct: number; up: boolean }>;
}
export interface Note {
id: string;
leadId: string;
userId: string;
authorName: string;
authorAvatar: string;
authorRole: UserRole;
note: string;
createdAt: string;
updatedAt: string;
}
export type NotificationType =
| "lead_created"
| "lead_status_changed"
| "lead_assigned"
| "chat_message"
| "note_added"
| "event_scheduled";
export interface Notification {
id: string;
type: NotificationType;
title: string;
description: string;
timestamp: string;
read: boolean;
link?: string;
}
export interface ChatMessage {
id: string;
conversationId: string;
senderId: string;
senderName: string;
senderAvatar: string;
content: string;
timestamp: string;
}
export interface Conversation {
id: string;
participants: { id: string; name: string; avatar: string; role: string }[];
lastMessage: string;
lastMessageTime: string;
unread: number;
messages: ChatMessage[];
}
export type EventType = "meeting" | "call" | "email" | "task" | "note";
export type EventStatus = "scheduled" | "completed" | "cancelled" | "rescheduled";
export interface CalendarEvent {
id: string;
userId: string;
participantId: string | null;
developerId: string | null;
leadId: string | null;
conversationId: string | null;
title: string;
description: string | null;
participantNotes: string | null;
eventType: EventType;
startTime: string;
endTime: string | null;
durationMinutes: number | null;
status: EventStatus;
creator: { id: string; name: string; email?: string; role?: string; avatar?: string } | null;
participant: { id: string; name: string; email?: string; role?: string; avatar?: string } | null;
developer: { id: string; name: string; email?: string; role?: string; avatar?: string } | null;
lead: { id: string; companyName: string; contactName: string } | null;
clientName: string | null;
clientEmail: string | null;
clientPhone: string | null;
createdAt: string;
}
.setup-templates/templates/utils-generic.ts
-7
View File
@@ -1,7 +0,0 @@
// ── Generic Utils Template ────────────────────────────────────────────
// Auto-generated by self-healing setup.
// Placeholder for @/utils/* imports. Customize as needed.
export function utilsFunction(): void {
// Placeholder
}
.setup-templates/templates/utils.ts
-76
View File
@@ -1,76 +0,0 @@
// ── Utility Functions ────────────────────────────────────────────────
// Auto-generated by self-healing setup. Edit .setup-templates/templates/utils.ts to customize.
import { clsx, type ClassValue } from "clsx";
import { twMerge } from "tailwind-merge";
export function cn(...inputs: ClassValue[]) {
return twMerge(clsx(inputs));
}
export function formatDate(date: Date | string, options?: Intl.DateTimeFormatOptions): string {
const d = typeof date === "string" ? new Date(date) : date;
return d.toLocaleDateString("en-US", {
year: "numeric",
month: "short",
day: "numeric",
...options,
});
}
export function formatTime(date: Date | string): string {
const d = typeof date === "string" ? new Date(date) : date;
return d.toLocaleTimeString("en-US", { hour: "numeric", minute: "2-digit", hour12: true });
}
export function formatCurrency(amount: number, currency = "USD"): string {
return new Intl.NumberFormat("en-US", { style: "currency", currency }).format(amount);
}
export function truncate(str: string, length: number): string {
if (str.length <= length) return str;
return str.slice(0, length - 3) + "...";
}
export function generateId(): string {
return crypto.randomUUID();
}
export function debounce<T extends (...args: unknown[]) => unknown>(
fn: T,
delay: number
): (...args: Parameters<T>) => void {
let timeoutId: ReturnType<typeof setTimeout>;
return (...args: Parameters<T>) => {
clearTimeout(timeoutId);
timeoutId = setTimeout(() => fn(...args), delay);
};
}
export function throttle<T extends (...args: unknown[]) => unknown>(
fn: T,
limit: number
): (...args: Parameters<T>) => void {
let inThrottle = false;
return (...args: Parameters<T>) => {
if (!inThrottle) {
fn(...args);
inThrottle = true;
setTimeout(() => (inThrottle = false), limit);
}
};
}
export function classNames(...classes: (string | boolean | undefined | null)[]): string {
return classes.filter(Boolean).join(" ");
}
export function sleep(ms: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, ms));
}
export function retry<T>(fn: () => Promise<T>, retries = 3, delay = 1000): Promise<T> {
return fn().catch((err) =>
retries > 0 ? sleep(delay).then(() => retry(fn, retries - 1, delay * 2)) : Promise.reject(err)
);
}
Web_Backgrounds/default-theme.css
-35
View File
@@ -1,35 +0,0 @@
/* ============================================================================
Default Theme — Clean, Professional SaaS Design Tokens
This is the standard CRM appearance. No Spidey branding or assets.
============================================================================ */
.theme-default {
--background: 222.2 84% 4.9%;
--foreground: 210 40% 98%;
--card: 222.2 84% 4.9%;
--card-foreground: 210 40% 98%;
--popover: 222.2 84% 4.9%;
--popover-foreground: 210 40% 98%;
--primary: 210 80% 52%;
--primary-foreground: 222.2 47.4% 11.2%;
--secondary: 217.2 32.6% 17.5%;
--secondary-foreground: 210 40% 98%;
--muted: 217.2 32.6% 17.5%;
--muted-foreground: 215 20.2% 65.1%;
--accent: 217.2 32.6% 17.5%;
--accent-foreground: 210 40% 98%;
--destructive: 0 62.8% 30.6%;
--destructive-foreground: 210 40% 98%;
--border: 215 25% 22%;
--input: 215 25% 22%;
--ring: 210 80% 52%;
--radius: 0.5rem;
--sidebar: 222.2 84% 4.9%;
--sidebar-foreground: 210 40% 98%;
--sidebar-primary: 210 80% 52%;
--sidebar-primary-foreground: 0 0% 100%;
--sidebar-accent: 217.2 32.6% 17.5%;
--sidebar-accent-foreground: 210 40% 98%;
--sidebar-border: 215 25% 22%;
--sidebar-ring: 210 80% 52%;
}
Web_Backgrounds/spidey-theme.css
-88
View File
@@ -1,88 +0,0 @@
/* ============================================================================
Spidey Theme — Baseline Design Tokens
This is the current website appearance captured as a reusable CSS theme.
============================================================================ */
.theme-spidey {
--background: 240 18% 97%;
--foreground: 240 8% 14%;
--card: 0 0% 100%;
--card-foreground: 240 8% 14%;
--popover: 0 0% 100%;
--popover-foreground: 240 8% 14%;
--secondary: 220 75% 48%;
--secondary-foreground: 0 0% 100%;
--muted: 240 8% 92%;
--muted-foreground: 240 5% 50%;
--accent: 220 15% 90%;
--accent-foreground: 220 60% 28%;
--destructive: 0 70% 46%;
--destructive-foreground: 0 0% 100%;
--border: 240 8% 84%;
--input: 240 8% 84%;
--radius: 0.5rem;
--sidebar: 0 0% 99%;
--sidebar-foreground: 240 8% 14%;
--sidebar-accent: 220 20% 93%;
--sidebar-accent-foreground: 220 70% 28%;
--sidebar-border: 240 8% 84%;
}
.dark.theme-spidey {
--background: 222.2 84% 4.9%;
--foreground: 210 40% 98%;
--card: 222.2 84% 4.9%;
--card-foreground: 210 40% 98%;
--popover: 222.2 84% 4.9%;
--popover-foreground: 210 40% 98%;
--secondary: 217.2 32.6% 17.5%;
--secondary-foreground: 210 40% 98%;
--muted: 217.2 32.6% 17.5%;
--muted-foreground: 215 20.2% 65.1%;
--accent: 217.2 32.6% 17.5%;
--accent-foreground: 210 40% 98%;
--destructive: 0 62.8% 30.6%;
--destructive-foreground: 210 40% 98%;
--border: 217.2 32.6% 17.5%;
--input: 217.2 32.6% 17.5%;
--radius: 0.5rem;
--sidebar: 222.2 84% 4.9%;
--sidebar-foreground: 210 40% 98%;
--sidebar-accent: 217.2 32.6% 17.5%;
--sidebar-accent-foreground: 210 40% 98%;
--sidebar-border: 217.2 32.6% 17.5%;
}
.theme-spidey body::before {
content: "";
position: fixed;
inset: 0;
z-index: -1;
pointer-events: none;
background-image:
repeating-conic-gradient(
from 10deg,
transparent 0deg 10deg,
rgba(220, 38, 38, 0.04) 10deg 12deg
),
repeating-radial-gradient(
circle at 50% 50%,
transparent 0,
transparent 30px,
rgba(37, 99, 235, 0.03) 30px,
transparent 32px
),
radial-gradient(2px 2px at 10% 20%, rgba(220,38,38,0.35) 0%, transparent 100%),
radial-gradient(2px 2px at 30% 80%, rgba(37,99,235,0.35) 0%, transparent 100%),
radial-gradient(2px 2px at 50% 30%, rgba(220,38,38,0.3) 0%, transparent 100%),
radial-gradient(2px 2px at 70% 60%, rgba(37,99,235,0.3) 0%, transparent 100%),
radial-gradient(2px 2px at 85% 15%, rgba(220,38,38,0.35) 0%, transparent 100%),
radial-gradient(2px 2px at 20% 50%, rgba(37,99,235,0.25) 0%, transparent 100%),
radial-gradient(2px 2px at 65% 85%, rgba(220,38,38,0.25) 0%, transparent 100%),
radial-gradient(2px 2px at 40% 10%, rgba(37,99,235,0.3) 0%, transparent 100%),
radial-gradient(2px 2px at 90% 75%, rgba(220,38,38,0.25) 0%, transparent 100%),
radial-gradient(2px 2px at 5% 60%, rgba(37,99,235,0.3) 0%, transparent 100%),
radial-gradient(2px 2px at 55% 55%, rgba(220,38,38,0.2) 0%, transparent 100%);
background-size: 200% 200%;
animation: drift 60s linear infinite;
}
Web_Backgrounds/spidey-tokens.json
-52
View File
@@ -1,52 +0,0 @@
{
"theme": {
"id": "spidey",
"name": "Spidey",
"description": "Current website appearance — dark theme with red accents",
"type": "dark",
"default": true
},
"colors": {
"background": "hsl(222.2, 84%, 4.9%)",
"foreground": "hsl(210, 40%, 98%)",
"card": "hsl(222.2, 84%, 4.9%)",
"cardForeground": "hsl(210, 40%, 98%)",
"popover": "hsl(222.2, 84%, 4.9%)",
"popoverForeground": "hsl(210, 40%, 98%)",
"primary": "hsl(0, 100%, 53%)",
"primaryForeground": "hsl(222.2, 47.4%, 11.2%)",
"secondary": "hsl(217.2, 32.6%, 17.5%)",
"secondaryForeground": "hsl(210, 40%, 98%)",
"muted": "hsl(217.2, 32.6%, 17.5%)",
"mutedForeground": "hsl(215, 20.2%, 65.1%)",
"accent": "hsl(217.2, 32.6%, 17.5%)",
"accentForeground": "hsl(210, 40%, 98%)",
"destructive": "hsl(0, 62.8%, 30.6%)",
"destructiveForeground": "hsl(210, 40%, 98%)",
"border": "hsl(217.2, 32.6%, 17.5%)",
"input": "hsl(217.2, 32.6%, 17.5%)",
"ring": "hsl(0, 100%, 53%)",
"sidebar": "hsl(222.2, 84%, 4.9%)",
"sidebarForeground": "hsl(210, 40%, 98%)",
"sidebarPrimary": "hsl(0, 100%, 53%)",
"sidebarPrimaryForeground": "hsl(0, 0%, 100%)",
"sidebarAccent": "hsl(217.2, 32.6%, 17.5%)",
"sidebarAccentForeground": "hsl(210, 40%, 98%)",
"sidebarBorder": "hsl(217.2, 32.6%, 17.5%)",
"sidebarRing": "hsl(0, 100%, 53%)"
},
"borderRadius": "0.5rem",
"typography": {
"fontFamily": "Inter, sans-serif",
"headingFont": "Inter, sans-serif"
},
"keyColors": {
"primary": "#FF0000",
"background": "#0a0a0f",
"card": "#141414",
"sidebar": "#0a0a0f",
"border": "#1a1a24",
"text": "#e8e8ef",
"mutedText": "#888888"
}
}
Web_Calendar/api/event-users/route.ts
-29
View File
@@ -1,29 +0,0 @@
import { NextResponse } from "next/server"
import { getSessionUser } from "@/lib/auth"
import { query } from "@/lib/db"
export async function GET() {
try {
const user = await getSessionUser()
if (!user) return NextResponse.json({ error: "Unauthorized" }, { status: 401 })
const result = await query(
`SELECT u.id, u.first_name, u.last_name, u.email
FROM users u
WHERE u.deleted_at IS NULL AND u.id != $1
ORDER BY u.first_name ASC`,
[user.id],
)
const users = result.rows.map((r: any) => ({
id: r.id,
name: `${r.first_name} ${r.last_name}`,
email: r.email,
}))
return NextResponse.json({ users })
} catch (error) {
console.error("Event users error:", error)
return NextResponse.json({ error: "Failed to load users" }, { status: 500 })
}
}
Web_Calendar/api/events/[id]/ics/route.ts
-54
View File
@@ -1,54 +0,0 @@
import { NextRequest, NextResponse } from "next/server"
import { getSessionUser } from "@/lib/auth"
import { query } from "@/lib/db"
import { buildIcs } from "@/lib/ics"
export async function GET(_request: NextRequest, { params }: { params: Promise<{ id: string }> }) {
try {
const user = await getSessionUser()
if (!user) return NextResponse.json({ error: "Unauthorized" }, { status: 401 })
const { id } = await params
const result = await query(
`SELECT e.id, e.user_id, e.participant_id, e.title, e.description, e.event_type,
e.start_time, e.end_time, e.duration_minutes, e.status,
u.email AS user_email, u.first_name AS user_first, u.last_name AS user_last,
p.email AS p_email, p.first_name AS p_first, p.last_name AS p_last
FROM scheduled_events e
JOIN users u ON u.id = e.user_id
LEFT JOIN users p ON p.id = e.participant_id
WHERE e.id = $1 AND e.user_id = $2`,
[id, user.id],
)
if (result.rows.length === 0) {
return NextResponse.json({ error: "Event not found" }, { status: 404 })
}
const r = result.rows[0]
const icsContent = buildIcs({
uid: r.id,
title: r.title,
description: r.description || undefined,
startTime: new Date(r.start_time),
endTime: r.end_time ? new Date(r.end_time) : undefined,
durationMinutes: r.duration_minutes || undefined,
organizerName: `${r.user_first} ${r.user_last}`,
organizerEmail: r.user_email,
attendeeName: r.p_first ? `${r.p_first} ${r.p_last}` : undefined,
attendeeEmail: r.p_email || undefined,
})
return new NextResponse(icsContent, {
headers: {
"Content-Type": "text/calendar; charset=utf-8",
"Content-Disposition": `attachment; filename="event-${r.id}.ics"`,
},
})
} catch (error) {
console.error("ICS GET error:", error)
return NextResponse.json({ error: "Failed to generate calendar file" }, { status: 500 })
}
}
Web_Calendar/api/events/[id]/route.ts
-167
View File
@@ -1,167 +0,0 @@
import { NextRequest, NextResponse } from "next/server"
import { getSessionUser } from "@/lib/auth"
import { query } from "@/lib/db"
import { sendEventRescheduled } from "@/lib/email"
export async function PATCH(request: NextRequest, { params }: { params: Promise<{ id: string }> }) {
try {
const user = await getSessionUser()
if (!user) return NextResponse.json({ error: "Unauthorized" }, { status: 401 })
const { id } = await params
const { title, description, eventType, startTime, endTime, durationMinutes, status, participantId, participantNotes } = await request.json()
const existing = await query(
`SELECT e.user_id, e.participant_id, e.title, e.description, e.participant_notes, e.event_type,
e.start_time, e.end_time, e.duration_minutes, e.status,
u.email AS u_email, u.first_name AS u_first, u.last_name AS u_last,
p.email AS p_email, p.first_name AS p_first, p.last_name AS p_last
FROM scheduled_events e
JOIN users u ON u.id = e.user_id
LEFT JOIN users p ON p.id = e.participant_id
WHERE e.id = $1`,
[id],
user.id,
)
if (existing.rows.length === 0) {
return NextResponse.json({ error: "Event not found" }, { status: 404 })
}
const old = existing.rows[0]
const isCreator = old.user_id === user.id
const isParticipant = old.participant_id === user.id
if (!isCreator && !isParticipant) {
return NextResponse.json({ error: "Forbidden" }, { status: 403 })
}
if (!isCreator && (
(title !== undefined && title !== old.title) ||
(eventType !== undefined && eventType !== old.event_type) ||
(startTime !== undefined && startTime !== old.start_time) ||
(endTime !== undefined && endTime !== old.end_time) ||
(durationMinutes !== undefined && durationMinutes !== old.duration_minutes) ||
(participantId !== undefined && participantId !== old.participant_id)
)) {
return NextResponse.json({ error: "Only the creator can edit event details" }, { status: 403 })
}
const result = await query(
`UPDATE scheduled_events SET
title = COALESCE($1, title),
description = COALESCE($2, description),
participant_notes = COALESCE($3, participant_notes),
event_type = COALESCE($4, event_type),
start_time = COALESCE($5, start_time),
end_time = COALESCE($6, end_time),
duration_minutes = COALESCE($7, duration_minutes),
status = COALESCE($8, status),
participant_id = COALESCE($9, participant_id),
updated_at = NOW()
WHERE id = $10
RETURNING id, user_id, participant_id, lead_id, conversation_id, title, description, participant_notes, event_type, start_time, end_time, duration_minutes, status, created_at`,
[
title || null,
description ?? null,
participantNotes ?? null,
eventType || null,
startTime || null,
endTime ?? null,
durationMinutes ?? null,
status || null,
participantId ?? null,
id,
],
user.id,
)
const r = result.rows[0]
const isChanged = r.start_time !== old.start_time || r.end_time !== old.end_time ||
r.title !== old.title || r.event_type !== old.event_type ||
r.status !== old.status
if (isChanged && r.participant_id && r.participant_id !== user.id) {
sendEventRescheduled({
creatorName: `${user.firstName} ${user.lastName}`,
creatorEmail: user.email,
participantName: old.p_first ? `${old.p_first} ${old.p_last}` : null,
participantEmail: old.p_email || null,
title: r.title,
description: r.description || undefined,
eventType: r.event_type,
startTime: r.start_time,
endTime: r.end_time || undefined,
durationMinutes: r.duration_minutes || undefined,
}).catch((err) => console.error("Reschedule email error:", err))
}
const creatorResult = await query(
`SELECT u.id, u.first_name, u.last_name, u.email, u.avatar_url,
ur.role_id, r.name AS role_name, r.display_name AS role_display
FROM users u
LEFT JOIN user_roles ur ON ur.user_id = u.id
LEFT JOIN roles r ON r.id = ur.role_id
WHERE u.id = $1`,
[old.user_id],
)
const creatorInfo = creatorResult.rows[0]
const participantInfo = old.participant_id ? { id: old.participant_id, name: `${old.p_first} ${old.p_last}` || old.participant_id, email: old.p_email || null } : null
return NextResponse.json({
event: {
id: r.id,
userId: r.user_id,
participantId: r.participant_id,
leadId: r.lead_id,
conversationId: r.conversation_id,
title: r.title,
description: r.description,
participantNotes: r.participant_notes,
eventType: r.event_type,
startTime: r.start_time,
endTime: r.end_time,
durationMinutes: r.duration_minutes,
status: r.status,
creator: creatorInfo ? { id: creatorInfo.id, name: `${creatorInfo.first_name} ${creatorInfo.last_name}`, email: creatorInfo.email, role: creatorInfo.role_display || creatorInfo.role_name, avatar: creatorInfo.avatar_url } : { id: old.user_id, name: "Unknown" },
participant: participantInfo,
lead: null,
createdAt: r.created_at,
},
})
} catch (error) {
console.error("Events PATCH error:", error)
return NextResponse.json({ error: "Failed to update event" }, { status: 500 })
}
}
export async function DELETE(request: NextRequest, { params }: { params: Promise<{ id: string }> }) {
try {
const user = await getSessionUser()
if (!user) return NextResponse.json({ error: "Unauthorized" }, { status: 401 })
const { id } = await params
const existing = await query(
`SELECT user_id FROM scheduled_events WHERE id = $1`,
[id],
user.id,
)
if (existing.rows.length === 0) {
return NextResponse.json({ error: "Event not found" }, { status: 404 })
}
if (existing.rows[0].user_id !== user.id) {
return NextResponse.json({ error: "Forbidden" }, { status: 403 })
}
await query(`DELETE FROM scheduled_events WHERE id = $1`, [id], user.id)
return NextResponse.json({ success: true })
} catch (error) {
console.error("Events DELETE error:", error)
return NextResponse.json({ error: "Failed to delete event" }, { status: 500 })
}
}
Web_Calendar/api/events/route.ts
-188
View File
@@ -1,188 +0,0 @@
import { NextRequest, NextResponse } from "next/server"
import { getSessionUser } from "@/lib/auth"
import { query } from "@/lib/db"
import { sendEventConfirmation } from "@/lib/email"
export async function GET(request: NextRequest) {
try {
const user = await getSessionUser()
if (!user) return NextResponse.json({ error: "Unauthorized" }, { status: 401 })
const { searchParams } = new URL(request.url)
const start = searchParams.get("start")
const end = searchParams.get("end")
const status = searchParams.get("status")
let sql = `SELECT e.id, e.user_id, e.participant_id, e.lead_id, e.conversation_id,
e.title, e.description, e.participant_notes, e.event_type,
e.start_time, e.end_time, e.duration_minutes, e.status, e.created_at,
u.id AS u_id, u.first_name AS u_first, u.last_name AS u_last, u.email AS u_email, u.avatar_url AS u_avatar,
ur.role_id AS u_role_id, r.name AS u_role_name, r.display_name AS u_role_display,
p.id AS p_id, p.first_name AS p_first, p.last_name AS p_last, p.email AS p_email, p.avatar_url AS p_avatar,
pr.role_id AS p_role_id, pr2.name AS p_role_name, pr2.display_name AS p_role_display,
l.id AS l_id, l.company_name, l.contact_name
FROM scheduled_events e
JOIN users u ON u.id = e.user_id
LEFT JOIN users p ON p.id = e.participant_id
LEFT JOIN leads l ON l.id = e.lead_id
LEFT JOIN user_roles ur ON ur.user_id = e.user_id
LEFT JOIN roles r ON r.id = ur.role_id
LEFT JOIN user_roles pr ON pr.user_id = e.participant_id
LEFT JOIN roles pr2 ON pr2.id = pr.role_id`
const params: unknown[] = []
let idx = 1
if (user.role !== "super_admin") {
sql += ` WHERE e.user_id = $${idx} OR e.participant_id = $${idx}`
params.push(user.id)
idx++
} else {
sql += ` WHERE 1=1`
}
if (start) {
sql += ` AND e.start_time >= $${idx}`
params.push(start)
idx++
}
if (end) {
sql += ` AND e.start_time <= $${idx}`
params.push(end)
idx++
}
if (status) {
sql += ` AND e.status = $${idx}`
params.push(status)
idx++
}
sql += " ORDER BY e.start_time ASC"
const result = await query(sql, params, user.id)
const events = result.rows.map((r: any) => ({
id: r.id,
userId: r.user_id,
participantId: r.participant_id,
leadId: r.lead_id,
conversationId: r.conversation_id,
title: r.title,
description: r.description,
participantNotes: r.participant_notes,
eventType: r.event_type,
startTime: r.start_time,
endTime: r.end_time,
durationMinutes: r.duration_minutes,
status: r.status,
creator: { id: r.u_id, name: `${r.u_first} ${r.u_last}`, email: r.u_email, role: r.u_role_display || r.u_role_name, avatar: r.u_avatar },
participant: r.p_id ? { id: r.p_id, name: `${r.p_first} ${r.p_last}`, email: r.p_email, role: r.p_role_display || r.p_role_name, avatar: r.p_avatar } : null,
lead: r.l_id ? { id: r.l_id, companyName: r.company_name, contactName: r.contact_name } : null,
createdAt: r.created_at,
}))
return NextResponse.json({ events })
} catch (error) {
console.error("Events GET error:", error)
return NextResponse.json({ error: "Failed to load events" }, { status: 500 })
}
}
export async function POST(request: NextRequest) {
try {
const user = await getSessionUser()
if (!user) return NextResponse.json({ error: "Unauthorized" }, { status: 401 })
const {
participantId, leadId, conversationId,
title, description, eventType,
startTime, endTime, durationMinutes,
} = await request.json()
if (!title || !startTime) {
return NextResponse.json({ error: "Title and start time are required" }, { status: 400 })
}
const result = await query(
`INSERT INTO scheduled_events (user_id, participant_id, lead_id, conversation_id, title, description, event_type, start_time, end_time, duration_minutes)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)
RETURNING id, user_id, participant_id, lead_id, conversation_id, title, description, participant_notes, event_type, start_time, end_time, duration_minutes, status, created_at`,
[
user.id,
participantId || null,
leadId || null,
conversationId || null,
title,
description || null,
eventType || "meeting",
startTime,
endTime || null,
durationMinutes || null,
],
user.id,
)
const r = result.rows[0]
if (participantId && participantId !== user.id) {
await query(
`INSERT INTO notifications (user_id, type, title, description, link, context_id, context_type)
VALUES ($1, $2, $3, $4, $5, $6, $7)`,
[
participantId,
"event_scheduled",
`Meeting scheduled: ${title}`,
`${user.firstName} ${user.lastName} scheduled a ${eventType || "meeting"} with you`,
`/calendar`,
r.id,
"scheduled_event",
],
)
}
const participantResult = participantId ? await query(
`SELECT email, first_name, last_name FROM users WHERE id = $1`,
[participantId],
) : null
const participant = participantResult?.rows[0] || null
sendEventConfirmation({
creatorName: `${user.firstName} ${user.lastName}`,
creatorEmail: user.email,
participantName: participant ? `${participant.first_name} ${participant.last_name}` : null,
participantEmail: participant?.email || null,
title,
description: description || undefined,
eventType: eventType || "meeting",
startTime,
endTime: endTime || undefined,
durationMinutes: durationMinutes || undefined,
}).catch((err) => console.error("Email error:", err))
return NextResponse.json({
event: {
id: r.id,
userId: r.user_id,
participantId: r.participant_id,
leadId: r.lead_id,
conversationId: r.conversation_id,
title: r.title,
description: r.description,
participantNotes: r.participant_notes,
eventType: r.event_type,
startTime: r.start_time,
endTime: r.end_time,
durationMinutes: r.duration_minutes,
status: r.status,
creator: { id: user.id, name: `${user.firstName} ${user.lastName}`, email: user.email, role: user.role },
participant: participantId ? { id: participantId, name: participant ? `${participant.first_name} ${participant.last_name}` : participantId, email: participant?.email || null } : null,
lead: leadId ? { id: leadId, companyName: "", contactName: "" } : null,
createdAt: r.created_at,
},
}, { status: 201 })
} catch (error) {
console.error("Events POST error:", error)
return NextResponse.json({ error: "Failed to create event" }, { status: 500 })
}
}
Web_Calendar/calendar/page.tsx
-1012
View File
File diff suppressed because it is too large Load Diff
Web_Calendar/database/011_calendar_events.sql
-24
View File
@@ -1,24 +0,0 @@
CREATE TABLE IF NOT EXISTS scheduled_events (
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE,
participant_id UUID REFERENCES users(id) ON DELETE SET NULL,
lead_id UUID REFERENCES leads(id) ON DELETE SET NULL,
conversation_id UUID REFERENCES conversations(id) ON DELETE SET NULL,
title VARCHAR(255) NOT NULL,
description TEXT,
event_type VARCHAR(20) NOT NULL DEFAULT 'meeting',
start_time TIMESTAMPTZ NOT NULL,
end_time TIMESTAMPTZ,
duration_minutes INT,
status VARCHAR(20) NOT NULL DEFAULT 'scheduled',
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
CONSTRAINT chk_event_type CHECK (event_type IN ('meeting','call','chat','follow_up','demo','other')),
CONSTRAINT chk_event_status CHECK (status IN ('scheduled','completed','cancelled','rescheduled'))
);
CREATE INDEX IF NOT EXISTS idx_scheduled_events_user ON scheduled_events(user_id, start_time);
CREATE INDEX IF NOT EXISTS idx_scheduled_events_participant ON scheduled_events(participant_id);
CREATE INDEX IF NOT EXISTS idx_scheduled_events_lead ON scheduled_events(lead_id);
CREATE INDEX IF NOT EXISTS idx_scheduled_events_date ON scheduled_events(start_time);
CREATE INDEX IF NOT EXISTS idx_scheduled_events_status ON scheduled_events(user_id, status);
Web_Calendar/database/013_rls_calendar.sql
-22
View File
@@ -1,22 +0,0 @@
-- ============================================================================
-- Migration 013: Row-Level Security for scheduled_events
-- ============================================================================
-- Enables RLS on the scheduled_events table so that users can only see
-- their own events at the database level. This is defense-in-depth:
-- the API already filters by user_id, but RLS ensures data isolation
-- even if there's a bug in the application layer.
--
-- The policy allows all operations when app.current_user_id is NULL
-- (backward-compatible fallback for queries that don't set the variable).
-- When the variable IS set, only rows matching the user's ID are visible.
-- ============================================================================
ALTER TABLE scheduled_events ENABLE ROW LEVEL SECURITY;
DROP POLICY IF EXISTS scheduled_events_user_policy ON scheduled_events;
CREATE POLICY scheduled_events_user_policy ON scheduled_events
FOR ALL
USING (
user_id = current_setting('app.current_user_id', true)::uuid
OR current_setting('app.current_user_id', true) IS NULL
);
Web_Calendar/database/015_rls_calendar_participant.sql
-19
View File
@@ -1,19 +0,0 @@
-- ============================================================================
-- Migration 015: Fix RLS for scheduled_events to include participant_id
-- ============================================================================
-- Previously, the RLS policy only checked user_id, which meant participants
-- could not see events at the database level (the app-layer WHERE clause
-- did the filtering, but RLS was defense-in-depth that missed this case).
--
-- This policy also allows app.current_user_id IS NULL for backward compat
-- with queries that don't set the session variable.
-- ============================================================================
DROP POLICY IF EXISTS scheduled_events_user_policy ON scheduled_events;
CREATE POLICY scheduled_events_user_policy ON scheduled_events
FOR ALL
USING (
user_id = current_setting('app.current_user_id', true)::uuid
OR participant_id = current_setting('app.current_user_id', true)::uuid
OR current_setting('app.current_user_id', true) IS NULL
);
Web_Calendar/database/016_participant_notes.sql
-1
View File
@@ -1 +0,0 @@
ALTER TABLE scheduled_events ADD COLUMN IF NOT EXISTS participant_notes TEXT;
Web_Calendar/types.ts
-27
View File
@@ -1,27 +0,0 @@
export type EventType = "call" | "follow_up" | "website_creation"
export type EventStatus = "scheduled" | "completed" | "cancelled" | "rescheduled"
export interface CalendarEvent {
id: string
userId: string
participantId: string | null
developerId: string | null
leadId: string | null
conversationId: string | null
title: string
description: string | null
participantNotes: string | null
eventType: EventType
startTime: string
endTime: string | null
durationMinutes: number | null
status: EventStatus
creator: { id: string; name: string; email?: string; role?: string; avatar?: string } | null
participant: { id: string; name: string; email?: string; role?: string; avatar?: string } | null
developer: { id: string; name: string; email?: string; role?: string; avatar?: string } | null
lead: { id: string; companyName: string; contactName: string } | null
clientName: string | null
clientEmail: string | null
clientPhone: string | null
createdAt: string
}
ai-server/index.mjs
-586
View File
@@ -1,586 +0,0 @@
// ── CRM AI Server ──────────────────────────────────────────────────
// Provides:
// - Chat API (POST /ai/chat) — routes user messages to Ollama for sales coaching
// - Setup wizard endpoints (GET /setup/status, POST /setup/profile, etc.)
// - Combined /status endpoint for splash page health polling
// - Configuration routes (GET/POST /ai/instructions, GET /ai/jobs)
// - Model pull support (POST /setup/ollama/pull)
//
// This is a zero-dependency Node.js HTTP server (no Express needed).
import http from "node:http"
import fs from "node:fs"
import path from "node:path"
import { spawn } from "node:child_process"
import crypto from "node:crypto"
import { fileURLToPath } from "node:url"
const __dirname = path.dirname(fileURLToPath(import.meta.url))
const ROOT = path.resolve(__dirname, "..")
// ── Load .env.local ──────────────────────────────────────────────
// Reads key=value pairs and sets them as process.env so they're
// available throughout the server. Ignores comments and blank lines.
// Values with matching quotes are unquoted.
try {
const envPath = path.join(ROOT, ".env.local")
const envContent = fs.readFileSync(envPath, "utf-8")
for (const line of envContent.split("\n")) {
const trimmed = line.trim()
if (!trimmed || trimmed.startsWith("#")) continue
const eqIdx = trimmed.indexOf("=")
if (eqIdx === -1) continue
const k = trimmed.substring(0, eqIdx).trim()
let v = trimmed.substring(eqIdx + 1).trim()
if ((v.startsWith('"') && v.endsWith('"')) || (v.startsWith("'") && v.endsWith("'"))) v = v.slice(1, -1)
if (!process.env[k]) process.env[k] = v
}
console.log("Loaded .env.local")
} catch {
// .env.local may not exist (first run), which is fine
}
// ── Config from env ─────────────────────────────────────────────
const PORT = parseInt(process.env.AI_PORT || "3001", 10)
const HOST = process.env.AI_HOST || "0.0.0.0"
const OLLAMA_URL = process.env.OLLAMA_BASE_URL || "http://localhost:11434"
const MODEL = process.env.AI_MODEL || "llama3.2:3b"
const SCRAPER_URL = process.env.SCRAPER_URL || "http://127.0.0.1:3008"
const FRONTEND_URL = process.env.FRONTEND_URL || "http://127.0.0.1:3006"
const DATABASE_URL = process.env.DATABASE_URL
const JOBS_PATH = process.env.JOBS_PATH || path.join(ROOT, "data", "ai", "jobs.jsonl")
const AI_MD_PATH = process.env.AI_MD_PATH || path.join(ROOT, "data", "ai", "ai.md")
// ── Setup state ──────────────────────────────────────────────────
// Tracks the Ollama model pull process so the setup wizard can
// poll for download progress.
let pullProcess = null
let pullProgress = { status: "idle", progress: 0, message: "" }
// ── Job loading ─────────────────────────────────────────────────
// Loads job categories from a JSONL file (one JSON object per line).
// Used as context for the AI sales coach chat responses.
function loadJobs() {
try {
const content = fs.readFileSync(JOBS_PATH, "utf-8")
const jobs = content
.split("\n")
.map((l) => l.trim())
.filter(Boolean)
.map((l) => {
try {
return JSON.parse(l)
} catch {
return null
}
})
.filter(Boolean)
console.log(`Loaded ${jobs.length} job categories`)
return jobs
} catch (e) {
console.error(`Failed to load jobs from ${JOBS_PATH}:`, e.message)
return []
}
}
// ── ai.md management ────────────────────────────────────────────
// ai.md is a Markdown file containing system instructions for the AI.
// It can be read, written, or appended to via the API.
function readInstructions() {
try {
return fs.readFileSync(AI_MD_PATH, "utf-8")
} catch {
return ""
}
}
function writeInstructions(content) {
fs.writeFileSync(AI_MD_PATH, content, "utf-8")
return content
}
function appendToImprovementLog(entry) {
// Adds a timestamped entry to the ## Improvement Log section of ai.md
const current = readInstructions()
const timestamp = new Date().toISOString().replace("T", " ").substring(0, 16)
const logEntry = `\n- ${timestamp} — ${entry}`
const logSection = "\n## Improvement Log"
const logIndex = current.indexOf(logSection)
if (logIndex !== -1) {
const afterLog = current.substring(logIndex + logSection.length)
const nextSectionIndex = afterLog.search(/\n## /)
if (nextSectionIndex !== -1) {
const insertAt = logIndex + logSection.length + nextSectionIndex
const updated = current.substring(0, insertAt) + logEntry + "\n" + current.substring(insertAt)
writeInstructions(updated)
return updated
}
writeInstructions(current + logEntry + "\n")
return current + logEntry + "\n"
}
const updated = current + `\n${logSection}\n${logEntry}\n`
writeInstructions(updated)
return updated
}
// ── Chat handler ────────────────────────────────────────────────
// scrapeFacebook() calls the scraper service (port 3008) to get leads.
// handleChat() processes user messages — triggers lead scraping when
// the user asks for "leads" or "listings", otherwise routes to Ollama
// for AI-powered sales coaching.
async function scrapeFacebook() {
const profilePath = process.env.FX_PROFILE || ""
const urlPath = `/scrape/facebook?force=true${profilePath ? `&profile_path=${encodeURIComponent(profilePath)}` : ""}`
try {
const body = await new Promise((resolve, reject) => {
const parsed = new URL(SCRAPER_URL)
let done = false
const req = http.request({ hostname: parsed.hostname, port: parsed.port || 3008, path: urlPath, method: "POST", timeout: 60000 }, (res) => {
let data = ""
res.on("data", (c) => data += c)
res.on("end", () => { done = true; resolve(data) })
res.on("error", (e) => { if (!done) { done = true; reject(e) } })
})
req.on("timeout", () => { if (!done) { done = true; req.destroy(); reject(new Error("scraper timeout")) } })
req.on("error", (e) => { if (!done) { done = true; reject(e) } })
req.end()
})
const data = JSON.parse(body)
return data
} catch (e) {
console.error("scrapeFacebook error:", e.message)
return null
}
}
function formatLeads(leads) {
if (!leads || leads.length === 0) return "No leads found from the latest scrape."
let output = `**${leads.length} leads found:**\n\n`
for (let i = 0; i < leads.length; i++) {
const l = leads[i]
output += `${i + 1}. ${l.title || "No title"}\n`
if (l.author) output += ` Author: ${l.author}\n`
if (l.date) output += ` Date: ${l.date}\n`
if (l.url) output += ` URL: ${l.url}\n`
output += "\n"
}
return output.trim()
}
async function handleChat(userMessage, userId, userRole) {
// If the user asks for leads, trigger the scraper
const lowerMsg = userMessage.toLowerCase()
const triggerWords = ["lists", "listings", "leads", "recent leads", "pull leads", "show me leads", "show listings"]
if (triggerWords.some(w => lowerMsg.includes(w))) {
const result = await scrapeFacebook()
if (result && result.success) {
return formatLeads(result.leads)
}
return "Scraper returned no results or encountered an error. Try again later."
}
// Otherwise, build a system prompt with job context and send to Ollama
const jobs = loadedJobs
const instructions = readInstructions()
const jobList = jobs
.map((j) => `- ${j.job_title} (${j.industry}): ${j.description}`)
.join("\n")
const systemPrompt = `You are a Sales AI Assistant for Coast IT CRM. Your role is to help salespeople with tips, strategies, and guidance.
Available job categories to target:
${jobList}
Current instructions:
${instructions}
Provide concise, actionable sales advice. When asked about a specific job category, give targeted tips on finding and engaging prospects in that field. Keep responses under 300 words unless asked for detail.`
const ollamaRes = await fetch(`${OLLAMA_URL}/api/chat`, {
method: "POST",
headers: { "Content-Type": "application/json" },
signal: AbortSignal.timeout(60000),
body: JSON.stringify({
model: MODEL,
messages: [
{ role: "system", content: systemPrompt },
{ role: "user", content: userMessage },
],
stream: false,
options: { temperature: 0.7, num_predict: 1024 },
}),
})
if (!ollamaRes.ok) {
const text = await ollamaRes.text()
throw new Error(`Ollama error (${ollamaRes.status}): ${text}`)
}
const data = await ollamaRes.json()
const responseText = data.message?.content || ""
// Persist conversation to PostgreSQL (best-effort — table may not exist yet)
try {
if (pgPool && userId) {
await pgPool.query(
"INSERT INTO ai_conversations (id, user_id, role, message, response) VALUES ($1, $2, $3, $4, $5)",
[crypto.randomUUID(), userId, userRole || "sales", userMessage, responseText]
)
}
} catch {
// table might not exist, ignore
}
return responseText
}
// ── PG pool (lazy init) ────────────────────────────────────────
// PostgreSQL connection pool for storing conversation history.
// Lazy-initialized so the server starts even without a DB.
let pgPool = null
async function initPg() {
if (!DATABASE_URL) return
try {
const { default: pg } = await import("pg")
pgPool = new pg.Pool({ connectionString: DATABASE_URL, max: 5 })
await pgPool.query("SELECT 1")
console.log("Connected to PostgreSQL")
} catch (e) {
console.warn("PostgreSQL unavailable (AI convos won't be saved):", e.message)
}
}
// ── Request router ─────────────────────────────────────────────
const loadedJobs = loadJobs()
function sendJSON(res, status, data) {
res.writeHead(status, { "Content-Type": "application/json" })
res.end(JSON.stringify(data))
}
function parseBody(req) {
return new Promise((resolve, reject) => {
let body = ""
req.on("data", (chunk) => (body += chunk))
req.on("end", () => {
try {
resolve(JSON.parse(body))
} catch {
reject(new Error("Invalid JSON"))
}
})
req.on("error", reject)
})
}
function parseURL(req) {
const url = new URL(req.url, `http://${req.headers.host || "localhost"}`)
return { pathname: url.pathname, searchParams: url.searchParams }
}
// ── HTTP Server ─────────────────────────────────────────────────
const server = http.createServer(async (req, res) => {
// CORS headers — allow the Next.js frontend (port 3006) to call us
res.setHeader("Access-Control-Allow-Origin", "*")
res.setHeader("Access-Control-Allow-Methods", "GET, POST, OPTIONS")
res.setHeader("Access-Control-Allow-Headers", "Content-Type")
if (req.method === "OPTIONS") {
res.writeHead(204)
res.end()
return
}
const { pathname } = parseURL(req)
try {
// GET /splash — loading screen
if (req.method === "GET" && pathname === "/splash") {
const splashPath = path.join(ROOT, "splash.html")
if (fs.existsSync(splashPath)) {
const html = fs.readFileSync(splashPath, "utf-8")
res.writeHead(200, { "Content-Type": "text/html" })
res.end(html)
return
}
sendJSON(res, 200, { status: "ok" })
return
}
// GET /health
if (req.method === "GET" && pathname === "/health") {
return sendJSON(res, 200, { status: "ok", model: MODEL })
}
// GET /status — combined health of all services
// Used by the splash page to check if AI, Scraper, and Frontend are ready.
// Polls each service internally to avoid cross-origin CORS issues.
if (req.method === "GET" && pathname === "/status") {
const { default: http } = await import("http")
const results = { ai: true }
// Check scraper
try {
await new Promise((resolve, reject) => {
const r = http.get(`${SCRAPER_URL}/health`, { timeout: 3000 }, (res) => { res.resume(); resolve() })
r.on("timeout", () => { r.destroy(); reject(new Error("timeout")) })
r.on("error", reject)
})
results.scraper = true
} catch { results.scraper = false }
// Check frontend
try {
await new Promise((resolve, reject) => {
const r = http.get(FRONTEND_URL, { timeout: 3000 }, (res) => { res.resume(); resolve() })
r.on("timeout", () => { r.destroy(); reject(new Error("timeout")) })
r.on("error", reject)
})
results.frontend = true
} catch { results.frontend = false }
return sendJSON(res, 200, results)
}
// ── Setup endpoints ─────────────────────────────────────────
// GET /setup/status — check environment
// Called by the splash page on boot. Returns info about:
// - Ollama availability
// - Model presence
// - Detected browsers with login status
// - Whether this is a first run (wizard needed)
if (req.method === "GET" && pathname === "/setup/status") {
const envExists = fs.existsSync(path.join(ROOT, ".env.local"))
// Ollama check
let ollamaRunning = false
try {
await fetch(`${OLLAMA_URL}/api/tags`, { signal: AbortSignal.timeout(3000) })
ollamaRunning = true
} catch {}
// Model check
let modelAvailable = false
if (ollamaRunning) {
try {
const r = await fetch(`${OLLAMA_URL}/api/show`, {
method: "POST", body: JSON.stringify({ name: MODEL }),
signal: AbortSignal.timeout(5000),
})
modelAvailable = r.ok
} catch {}
}
// Detect all browsers via scraper
let browsers = { firefox: { path: null }, opera: { path: null }, chrome: { path: null }, edge: { path: null } }
let facebookLoggedIn = false
let selectedBrowser = process.env.SELECTED_BROWSER || ""
try {
await fetch(`${SCRAPER_URL}/health`, { signal: AbortSignal.timeout(2000) })
const profiles = await (await fetch(`${SCRAPER_URL}/setup/profile`, { signal: AbortSignal.timeout(5000) })).json()
for (const [b, p] of Object.entries(profiles)) {
if (p) browsers[b] = { path: p }
}
// Check login for the selected browser first, then try all
const detectedList = Object.entries(browsers).filter(([, v]) => v.path)
for (const [b, v] of detectedList) {
try {
const r = await fetch(`${SCRAPER_URL}/setup/check-login`, {
method: "POST", headers: { "Content-Type": "application/json" },
body: JSON.stringify({ browser: b, profile_path: v.path }),
signal: AbortSignal.timeout(20000),
})
if (r.ok) {
const d = await r.json()
browsers[b].logged_in = d.logged_in === true
if (d.logged_in && !facebookLoggedIn) {
facebookLoggedIn = true
if (!selectedBrowser) selectedBrowser = b
}
}
} catch {}
}
} catch {}
const anyDetected = Object.values(browsers).some(v => v.path)
// first_run = any setup step is incomplete
const firstRun = !envExists || !ollamaRunning || !anyDetected || !facebookLoggedIn || !modelAvailable
return sendJSON(res, 200, {
first_run: firstRun,
env_exists: envExists,
ollama_running: ollamaRunning,
model_available: modelAvailable,
model_name: MODEL,
selected_browser: selectedBrowser,
browsers,
facebook_logged_in: facebookLoggedIn,
})
}
// POST /setup/profile — save selected browser + path to .env.local
// Called by the setup wizard when the user confirms their browser choice.
// Writes SELECTED_BROWSER and the matching profile env var to .env.local.
if (req.method === "POST" && pathname === "/setup/profile") {
const body = await parseBody(req)
const browserName = (body.browser || "").trim().toLowerCase()
const profilePath = (body.path || "").trim()
if (!browserName || !["firefox", "opera", "chrome", "edge"].includes(browserName))
return sendJSON(res, 400, { error: "Valid browser required (firefox/opera/chrome/edge)" })
if (!profilePath)
return sendJSON(res, 400, { error: "Path required" })
const envKey = browserName === "firefox" ? "FX_PROFILE"
: browserName === "opera" ? "OPERA_PROFILE"
: browserName === "edge" ? "EDGE_PROFILE"
: "CHROME_PROFILE"
const envPath = path.join(ROOT, ".env.local")
let content = ""
try { content = fs.readFileSync(envPath, "utf-8") } catch {}
let lines = content.split("\n")
// Update or add SELECTED_BROWSER
let foundSel = false
for (let i = 0; i < lines.length; i++) {
if (lines[i].trim().startsWith("SELECTED_BROWSER=")) {
lines[i] = `SELECTED_BROWSER=${browserName}`
foundSel = true
break
}
}
if (!foundSel) lines.push(`SELECTED_BROWSER=${browserName}`)
// Update or add browser profile
let foundProf = false
for (let i = 0; i < lines.length; i++) {
if (lines[i].trim().startsWith(`${envKey}=`)) {
lines[i] = `${envKey}=${profilePath}`
foundProf = true
break
}
}
if (!foundProf) lines.push(`${envKey}=${profilePath}`)
fs.writeFileSync(envPath, lines.join("\n"), "utf-8")
process.env.SELECTED_BROWSER = browserName
process.env[envKey] = profilePath
return sendJSON(res, 200, { success: true, browser: browserName, path: profilePath })
}
// POST /setup/check-login — proxy to scraper, accepts browser + profile_path
// The splash page calls this (via the AI server) to verify Facebook login status.
if (req.method === "POST" && pathname === "/setup/check-login") {
const body = await parseBody(req)
const browserName = (body.browser || "").trim().toLowerCase() || process.env.SELECTED_BROWSER || ""
const profilePath = (body.profile_path || "").trim()
if (!profilePath) return sendJSON(res, 200, { logged_in: false, reason: "no_profile" })
try {
const r = await fetch("http://127.0.0.1:3008/setup/check-login", {
method: "POST", headers: { "Content-Type": "application/json" },
body: JSON.stringify({ browser: browserName, profile_path: profilePath }),
signal: AbortSignal.timeout(20000),
})
if (r.ok) { const d = await r.json(); return sendJSON(res, 200, d) }
} catch {}
return sendJSON(res, 200, { logged_in: false, reason: "scraper_unavailable" })
}
// POST /setup/ollama/pull — start pulling the model
// Spawns "ollama pull" as a child process. The setup wizard polls
// the progress endpoint to show a download progress bar.
if (req.method === "POST" && pathname === "/setup/ollama/pull") {
if (pullProcess) return sendJSON(res, 200, { status: "already_running" })
pullProgress = { status: "downloading", progress: 0, message: "Starting..." }
const isWin = process.platform === "win32"
const cmd = isWin ? "ollama.exe" : "ollama"
pullProcess = spawn(cmd, ["pull", MODEL], { stdio: ["ignore", "pipe", "pipe"] })
pullProcess.stdout.on("data", (data) => {
const text = data.toString()
pullProgress.message = text.trim()
// Extract percentage from patterns like "pulling xxxx... 45%"
const m = text.match(/(\d+)%/)
if (m) pullProgress.progress = parseInt(m[1], 10)
})
pullProcess.on("close", (code) => {
pullProcess = null
pullProgress.status = code === 0 ? "done" : "failed"
if (code === 0) pullProgress.progress = 100
})
return sendJSON(res, 200, { status: "started" })
}
// GET /setup/ollama/pull/progress
// Returns current download progress for the setup wizard.
if (req.method === "GET" && pathname === "/setup/ollama/pull/progress") {
return sendJSON(res, 200, pullProgress)
}
// GET /ai/jobs — return loaded job categories
if (req.method === "GET" && pathname === "/ai/jobs") {
return sendJSON(res, 200, { jobs: loadedJobs })
}
// GET /ai/instructions — return current ai.md content
if (req.method === "GET" && pathname === "/ai/instructions") {
const instructions = readInstructions()
return sendJSON(res, 200, { success: true, instructions })
}
// POST /ai/instructions — update ai.md or append improvement log entry
if (req.method === "POST" && pathname === "/ai/instructions") {
const body = await parseBody(req)
if (body.content) {
writeInstructions(body.content)
} else if (body.entry) {
appendToImprovementLog(body.entry)
}
return sendJSON(res, 200, {
success: true,
instructions: readInstructions(),
})
}
// POST /ai/chat — main AI chat endpoint
// Accepts { message, user_id?, user_role? } and returns AI response.
// user_role must be "sales", "admin", or "super_admin" if provided.
if (req.method === "POST" && pathname === "/ai/chat") {
const chunks = []
req.on("data", c => chunks.push(c))
req.on("end", async () => {
try {
const rawBody = Buffer.concat(chunks).toString()
const body = JSON.parse(rawBody)
const { message, user_id, user_role } = body
if (!message) {
return sendJSON(res, 400, { error: "message is required" })
}
const validRoles = ["sales", "admin", "super_admin"]
if (user_role && !validRoles.includes(user_role)) {
return sendJSON(res, 403, { error: "Forbidden" })
}
const response = await handleChat(message, user_id || "", user_role || "sales")
sendJSON(res, 200, { response })
} catch (e) {
if (!res.headersSent) sendJSON(res, 500, { error: e.message })
}
})
return
}
// 404 fallback
sendJSON(res, 404, { error: "Not found" })
} catch (err) {
console.error("Request error:", err)
sendJSON(res, 500, { error: err.message })
}
})
// ── Start ───────────────────────────────────────────────────────
server.listen(PORT, HOST, () => {
console.log(`CRM AI server listening on http://${HOST}:${PORT}`)
console.log(` Model: ${MODEL}`)
console.log(` Ollama: ${OLLAMA_URL}`)
})
initPg()
browser-use-service/main.py
-2149
View File
File diff suppressed because it is too large Load Diff
browser-use-service/requirements.txt
-18
View File
@@ -1,18 +0,0 @@
# ── Python Dependencies for the Facebook Scraper (FastAPI) ───────
# Install via: pip install -r requirements.txt
# Web framework for the REST API (health, setup, scrape endpoints)
fastapi>=0.115.0
# ASGI server for running the FastAPI app
uvicorn>=0.34.0
# Browser automation (launches Firefox, Chrome, Edge, Opera via Playwright)
playwright>=1.49.0
# AI-powered browser agent (fallback when direct browser scraping is flagged)
# Uses ChatOllama locally — no API keys needed
browser-use>=0.1.0
# LangChain integration for ChatOllama (provides the LLM for browser-use Agent)
langchain-ollama>=0.2.0
data/ai/ai.md
+26 -128
View File
@@ -1,142 +1,40 @@
# CRM AI Sales Assistant — Self-Knowledge
# AI Sales Assistant — Self-Improvement Instructions
## Identity
You are the CRM AI Sales Assistant for Coast IT CRM.
You run on a Node.js backend (port 3001) and use Ollama with a local model (dolphin3-llama3.2:3b).
Your purpose is to help salespeople close more deals by finding and engaging leads.
## Purpose
This file contains the AI's own configuration, knowledge, and improvement rules.
The AI can read and modify this file to update its behavior at runtime.
## Architecture
```
User → Next.js (:3006) → AI Server Node.js (:3001) → Ollama (:11434)
↓
PostgreSQL (conversations)
## Current Instructions
- Always respond in English
- Keep responses under 300 words unless asked for detail
- Use bullet points for lists
- Be direct and actionable — no fluff
- Never mention being an AI or language model
- Refer to the user by their role (salesperson, admin, etc.)
- If unsure about a topic, say "I don't have that information yet" rather than guessing
Python Scraper (:3008) — Facebook scraping via Playwright
```
Three services run concurrently:
- **AI Server** (`ai-server/index.mjs`, port 3001) — chat, setup wizard, config endpoints
- **Frontend** (Next.js, port 3006) — UI for salespeople
- **Scraper** (`browser-use-service/main.py`, port 3008) — Facebook lead discovery
## Capabilities
- Give sales tips and strategies per job category
- Generate cold email and outreach templates
- Handle objections with proven rebuttals
- Analyse prospect behaviour and suggest next steps
- Remember past conversations via PostgreSQL (`ai_conversations` table)
- Run Facebook scraper to find real leads asking for services
- Self-improve by writing to `data/ai/ai.md` via `POST /ai/instructions`
## Facebook Scraper
The scraper lives at `browser-use-service/main.py` port 3008.
### How It Works
1. **Browser detection** — tries Firefox profile first, then Chromium-based (Chrome/Opera/Edge), falls back to browser-use Agent
2. **Profile paths** — configured via env vars (`FX_PROFILE`, `CHROME_PROFILE`, `OPERA_PROFILE`, `EDGE_PROFILE`) or auto-detected on first run
3. **4-phase language pipeline** (English → Afrikaans → Xhosa → Zulu):
- **Phase 1 (English)**: User's selected query + 2-3 supplementary English searches from the English search pool. First query gets full human-like scroll, rest use quick search. This phase does the heavy lifting.
- **Phase 2 (Afrikaans)**: 2 Afrikaans queries targeting Afrikaans-speaking communities.
- **Phase 3 (isiXhosa)**: 2 Xhosa queries targeting Xhosa-speaking communities.
- **Phase 4 (isiZulu)**: 2 Zulu queries targeting Zulu-speaking communities.
- After all phases: pipeline check (date filter 2 days → AI + keyword classification → sort by freshness). Newest leads ranked first.
- Each phase extracts posts, deduplicates against all prior phases, then passes through a stealth delay (5-12s + mouse idle) before the next phase.
4. **Quick searches** — load page, double-scroll, extract visible posts (~12-18s each). Scroll-back behavior (35% chance to scroll up) and random return-to-top (25% chance) for stealth.
5. **Date filter** — only posts within **2 days** are considered. Anything older is discarded. Fresh leads only.
6. **Stealth mechanics**:
- Random viewport dimensions (1280×800 to 1920×1080) — never the same size twice
- Variable delays between searches (5-12 seconds) with mouse idle actions mixed in
- Human-like scroll patterns: scroll down, pause, sometimes scroll back up, sometimes return to top
- Canvas/WebGL/audio fingerprint spoofing via injected init scripts
- Random decoy page visits (e.g., Facebook Groups) between searches
- Profile directory is temp-copied and cleaned up after each scrape
- Detection signal monitoring (checkpoint, login pages, security challenges)
7. **2-pass classification (dead-accurate)**:
- **Pass 1 (AI)**: Ollama classifies each post as LEAD or NOT using a strict prompt per category. This is the primary filter and most accurate.
- **Pass 2 (Keyword)**: Only posts matching BOTH a target term AND a request term are kept. Requires multi-word phrases — standalone words like "need", "want", "help" are NOT used as they cause false positives. Aggressive reject list catches service offers, self-promotions, portfolio posts, learning-requests, and existing-site issues.
- **No loose fill**: Unlike the old approach, there is NO third pass that accepts posts matching EITHER term. Every returned lead has passed both AI and/or strict keyword validation. If fewer than 5 posts pass, that means only genuine leads are returned — no noise to pad the count.
8. **Scrape timing** — 3-6 minutes for a complete run. Returns 5-10 leads with high confidence.
### Lead Categories
Two categories, selectable when starting a scrape:
**Website Creation:**
- Target: people explicitly REQUESTING a website built/designed/created for them
- Keywords: "website", "web developer", "web design", "build a site", "who can build", etc.
- Request terms: "looking for", "need a", "need someone", "hire a", "recommend", "anyone know"
- Strict reject: service offers, SEO/marketing requests, learning-to-code, portfolio showcases, hiring posts, existing-website issues, geographic noise
**Tutoring:**
- Target: people explicitly REQUESTING a tutor, teacher, or lessons for themselves or their child
- Keywords: "tutor", "tutoring", "lessons for", "homework help", "private tutor", "extra classes"
- Request terms: same as website category — must co-occur with a target keyword
- Strict reject: people offering tutoring, educational products, homeschool programs, free trials, general study tips
### Multi-Language Pipeline (Phase Order)
4 South African languages in structured phases:
- **Phase 1 (English)**: primary query + supplementary English searches
- **Phase 2 (Afrikaans)**: 2 queries targeting Afrikaans speakers
- **Phase 3 (isiXhosa)**: 2 queries targeting Xhosa speakers
- **Phase 4 (isiZulu)**: 2 queries targeting Zulu speakers
### Output Format
Each lead returned includes:
- `title` — post preview text
- `author` — poster's name (may include location in name)
- `content` — extracted post text
- `url` — direct link to the post
- `date` — when posted (filtered within 7 days)
- `category` — "website" or "tutor"
Target is 5-10 dead-accurate leads per scrape. Quality over quantity — no loose padding.
### Configuration via Env Vars
- `SELECTED_BROWSER` — `firefox` (default), `chrome`, `opera`, `edge`, or `auto`
- `FX_PROFILE`, `CHROME_PROFILE`, `OPERA_PROFILE`, `EDGE_PROFILE` — browser profile paths
- `AI_PORT`, `AI_HOST` — AI server bind (default `3001`, `0.0.0.0`)
- `SCRAPER_URL` — scraper URL (default `http://127.0.0.1:3008`)
- `FRONTEND_URL` — frontend URL (default `http://127.0.0.1:3006`)
- `NEXT_PUBLIC_SCRAPER_URL` — frontend-facing scraper URL
- `OLLAMA_BASE_URL` — Ollama URL (default `http://localhost:11434`)
- `AI_MODEL` — Ollama model (default `llama3.2:3b`)
- `CLASSIFY_MODEL` — model for lead classification (default `dolphin-llama3:8b`)
## How to Start Scraping
1. Ensure all 3 services are running (ports 3001, 3006, 3008) and Ollama is on 11434
2. Open the frontend at `http://localhost:3006`
3. Select a job category (Website Creation or Tutoring)
4. Click "Search Facebook" — the scraper runs and returns leads
5. Leads are saved in the CRM for follow-up
## Sales Tips
## Knowledge Base
### Sales Tips
- Cold emails should be under 150 words
- Follow up within 48 hours
- Personalise every outreach with the prospect's name and company
- Use open-ended questions in discovery calls
- Always ask for the next step before ending a call
- For website leads: mention specific pages or features they requested
- For tutoring leads: reference the subject and age group they mentioned
## Job Targeting
### Job Targeting
- Developers respond best to technical value props
- Marketing managers care about ROI and metrics
- C-level executives want brevity and business impact
- Parents hiring tutors: empathy and qualifications matter most
## Response Rules
- Be direct and actionable — no fluff, no AI disclaimers
- Use short paragraphs and bullet points
- Never mention being an AI or language model
- If you don't know something, say so honestly
- Prioritise the user's role: salespeople need speed, admins need control
- When asked about scraping, give specific guidance on categories and languages
## Self-Improvement Protocol
1. You notice a gap in your knowledge or a pattern in user questions
2. You call `POST /ai/instructions` with:
- `entry`: description of the improvement
- `content`: optional full replacement of ai.md
3. The improvement is logged and loaded into the next system prompt
## Improvement Log
- (2026-07-07) Initial rewrite: full architecture, scraper details, multi-language, lead categories, env vars
Track changes made by the AI to improve itself:
- (initial) Basic instructions and knowledge base created
## Self-Modification Rules
The AI may update this file when:
1. It identifies a gap in its knowledge that would help salespeople
2. It discovers a better way to structure responses
3. A user explicitly requests an update to behavior
4. It notices repeated questions that aren't well-covered
Only append to the Improvement Log — don't delete previous entries.
data/ai/jobs.jsonl
+10 -2
View File
@@ -1,2 +1,10 @@
{"job_title":"Website Creation","keywords":["need a website","build my website","create a website for me","website for my business","need someone to build","who can build me","looking for a web developer","need a web designer","i need a website","need help with my website","looking for someone to create","need a site for","want a website for","need ecommerce website","need landing page","website for my small business","need my website redesigned","can someone build me","anyone know a web developer","recommend a web developer","need a new website"],"industry":"Technology","description":"Find people asking for websites, landing pages, or online stores to be built"}
{"job_title":"Tutoring","keywords":["need a tutor","looking for a tutor","tutor for my child","need help with homework","private tutor needed","looking for tutoring","need a math tutor","english tutor needed","online tutor for","lessons for my child","help my child with","need someone to tutor","looking for someone to teach","tutoring for my","need help learning","recommend a tutor","anyone know a tutor","need a private tutor","tutoring services for my child","need academic help"],"industry":"Education","description":"Find parents and students asking for tutoring services"}
{"job_title":"Software Developer","keywords":["developer","programmer","software engineer","coder","full stack","backend","frontend"],"industry":"Technology","description":"Builds and maintains software applications and systems"}
{"job_title":"Marketing Specialist","keywords":["marketing","digital marketing","brand manager","content marketer","social media"],"industry":"Marketing","description":"Plans and executes marketing campaigns across channels"}
{"job_title":"Sales Representative","keywords":["sales rep","account executive","business development","sales consultant"],"industry":"Sales","description":"Drives revenue through client acquisition and relationship management"}
{"job_title":"Project Manager","keywords":["project manager","program manager","scrum master","agile coach"],"industry":"Business","description":"Oversees project timelines, resources, and deliverables"}
{"job_title":"Graphic Designer","keywords":["designer","graphic designer","ui designer","ux designer","visual designer"],"industry":"Creative","description":"Creates visual concepts and designs for digital and print media"}
{"job_title":"Data Analyst","keywords":["data analyst","business analyst","data scientist","analytics"],"industry":"Technology","description":"Analyzes data to provide actionable business insights"}
{"job_title":"Customer Support Specialist","keywords":["customer support","customer service","support agent","help desk"],"industry":"Customer Service","description":"Assists customers with inquiries, issues, and product support"}
{"job_title":"Human Resources Manager","keywords":["HR manager","HR","recruiter","talent acquisition","people operations"],"industry":"Human Resources","description":"Manages recruitment, employee relations, and HR operations"}
{"job_title":"Financial Advisor","keywords":["financial advisor","financial planner","wealth manager","investment advisor"],"industry":"Finance","description":"Provides financial guidance and investment planning to clients"}
{"job_title":"Operations Manager","keywords":["operations manager","operations","logistics","supply chain"],"industry":"Business","description":"Oversees daily operations and process optimization"}
database/DESIGN_DECISIONS.md
+4 -4
View File
@@ -126,10 +126,10 @@ separate 1:N child tables.
| Username | Password | Role |
|----------|----------|------|
| `superadmin_demo` | `[REDACTED]` | SUPER_ADMIN |
| `admin_demo` | `[REDACTED]` | ADMIN |
| `sales_demo` | `[REDACTED]` | SALES_USER |
| `dev_demo` | `[REDACTED]` | DEVELOPER |
| `superadmin_demo` | `SuperAdmin@2026` | SUPER_ADMIN |
| `admin_demo` | `AdminAccess@2026` | ADMIN |
| `sales_demo` | `SalesAccess@2026` | SALES_USER |
| `dev_demo` | `DevTesting@2026` | DEVELOPER |
---
database/migrations/001_schema.sql
+41 -46
View File
@@ -22,7 +22,7 @@ $$ LANGUAGE plpgsql;
-- 1. AUTHENTICATION & ROLE MANAGEMENT
-- ============================================================================
CREATE TABLE IF NOT EXISTS roles (
CREATE TABLE roles (
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
name VARCHAR(50) NOT NULL,
display_name VARCHAR(100),
@@ -35,7 +35,7 @@ CREATE TABLE IF NOT EXISTS roles (
CREATE UNIQUE INDEX uq_roles_name ON roles(name);
CREATE TABLE IF NOT EXISTS permissions (
CREATE TABLE permissions (
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
resource VARCHAR(100) NOT NULL,
action VARCHAR(50) NOT NULL,
@@ -46,7 +46,7 @@ CREATE TABLE IF NOT EXISTS permissions (
CREATE UNIQUE INDEX uq_permissions ON permissions(resource, action);
CREATE TABLE IF NOT EXISTS role_permissions (
CREATE TABLE role_permissions (
role_id UUID NOT NULL REFERENCES roles(id) ON DELETE CASCADE,
permission_id UUID NOT NULL REFERENCES permissions(id) ON DELETE CASCADE,
granted_by UUID,
@@ -54,7 +54,7 @@ CREATE TABLE IF NOT EXISTS role_permissions (
PRIMARY KEY (role_id, permission_id)
);
CREATE TABLE IF NOT EXISTS users (
CREATE TABLE users (
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
username VARCHAR(100) NOT NULL,
email VARCHAR(255) NOT NULL,
@@ -83,7 +83,7 @@ CREATE INDEX idx_users_is_active ON users(is_active) WHERE deleted_at IS NULL;
CREATE INDEX idx_users_is_locked ON users(is_locked) WHERE is_locked = TRUE AND deleted_at IS NULL;
CREATE INDEX idx_users_created_by ON users(created_by);
CREATE TABLE IF NOT EXISTS user_roles (
CREATE TABLE user_roles (
user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE,
role_id UUID NOT NULL REFERENCES roles(id) ON DELETE CASCADE,
assigned_by UUID REFERENCES users(id),
@@ -98,7 +98,7 @@ CREATE INDEX idx_user_roles_role ON user_roles(role_id);
-- 2. BAN & SUSPENSION SYSTEM
-- ============================================================================
CREATE TABLE IF NOT EXISTS banned_users (
CREATE TABLE banned_users (
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
banned_user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE,
banned_by UUID NOT NULL REFERENCES users(id),
@@ -122,7 +122,7 @@ CREATE INDEX idx_banned_users_active ON banned_users(banned_user_id)
WHERE is_reversed = FALSE;
CREATE INDEX idx_banned_users_banned_by ON banned_users(banned_by);
CREATE TABLE IF NOT EXISTS suspended_users (
CREATE TABLE suspended_users (
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
suspended_user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE,
suspended_by UUID NOT NULL REFERENCES users(id),
@@ -145,7 +145,7 @@ CREATE INDEX idx_suspended_users_suspended_by ON suspended_users(suspended_by);
-- 3. SESSION & LOGIN MANAGEMENT
-- ============================================================================
CREATE TABLE IF NOT EXISTS sessions (
CREATE TABLE sessions (
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE,
token_hash VARCHAR(255) NOT NULL,
@@ -161,7 +161,7 @@ CREATE INDEX idx_sessions_user ON sessions(user_id) WHERE is_active = TRUE;
CREATE UNIQUE INDEX uq_sessions_token ON sessions(token_hash);
CREATE INDEX idx_sessions_expires ON sessions(expires_at) WHERE is_active = TRUE;
CREATE TABLE IF NOT EXISTS login_attempts (
CREATE TABLE login_attempts (
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
user_id UUID REFERENCES users(id),
username_attempted VARCHAR(100) NOT NULL,
@@ -181,7 +181,7 @@ CREATE INDEX idx_login_attempts_time ON login_attempts(attempted_at DESC);
-- 4. CUSTOMER MANAGEMENT
-- ============================================================================
CREATE TABLE IF NOT EXISTS customer_statuses (
CREATE TABLE customer_statuses (
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
name VARCHAR(100) NOT NULL,
description TEXT,
@@ -196,7 +196,7 @@ CREATE TABLE IF NOT EXISTS customer_statuses (
CREATE UNIQUE INDEX uq_customer_statuses_name ON customer_statuses(name) WHERE deleted_at IS NULL;
CREATE TABLE IF NOT EXISTS customers (
CREATE TABLE customers (
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
customer_type VARCHAR(20) NOT NULL,
status_id UUID NOT NULL REFERENCES customer_statuses(id),
@@ -218,7 +218,7 @@ CREATE INDEX idx_customers_score ON customers(score DESC) WHERE deleted_at IS NU
CREATE INDEX idx_customers_tags ON customers USING GIN(tags) WHERE deleted_at IS NULL;
CREATE INDEX idx_customers_created ON customers(created_at DESC) WHERE deleted_at IS NULL;
CREATE TABLE IF NOT EXISTS individual_customers (
CREATE TABLE individual_customers (
customer_id UUID PRIMARY KEY REFERENCES customers(id) ON DELETE CASCADE,
first_name VARCHAR(100) NOT NULL,
last_name VARCHAR(100) NOT NULL,
@@ -233,7 +233,7 @@ CREATE TABLE IF NOT EXISTS individual_customers (
CREATE INDEX idx_individual_names ON individual_customers(last_name, first_name);
CREATE TABLE IF NOT EXISTS company_customers (
CREATE TABLE company_customers (
customer_id UUID PRIMARY KEY REFERENCES customers(id) ON DELETE CASCADE,
company_name VARCHAR(255) NOT NULL,
registration_number VARCHAR(100),
@@ -250,7 +250,7 @@ CREATE TABLE IF NOT EXISTS company_customers (
CREATE INDEX idx_company_name ON company_customers(company_name);
CREATE INDEX idx_company_industry ON company_customers(industry);
CREATE TABLE IF NOT EXISTS contact_information (
CREATE TABLE contact_information (
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
customer_id UUID NOT NULL REFERENCES customers(id) ON DELETE CASCADE,
type VARCHAR(20) NOT NULL,
@@ -270,7 +270,7 @@ CREATE INDEX idx_contacts_type ON contact_information(type) WHERE deleted_at IS
CREATE INDEX idx_contacts_value ON contact_information(value) WHERE deleted_at IS NULL;
CREATE UNIQUE INDEX uq_contacts_primary ON contact_information(customer_id, type) WHERE is_primary = TRUE AND deleted_at IS NULL;
CREATE TABLE IF NOT EXISTS addresses (
CREATE TABLE addresses (
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
customer_id UUID NOT NULL REFERENCES customers(id) ON DELETE CASCADE,
address_type VARCHAR(20) NOT NULL,
@@ -294,7 +294,7 @@ CREATE INDEX idx_addresses_country ON addresses(country) WHERE deleted_at IS NUL
CREATE INDEX idx_addresses_city ON addresses(city) WHERE deleted_at IS NULL;
CREATE UNIQUE INDEX uq_addresses_primary ON addresses(customer_id, address_type) WHERE is_primary = TRUE AND deleted_at IS NULL;
CREATE TABLE IF NOT EXISTS customer_notes (
CREATE TABLE customer_notes (
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
customer_id UUID NOT NULL REFERENCES customers(id) ON DELETE CASCADE,
author_id UUID NOT NULL REFERENCES users(id),
@@ -315,7 +315,7 @@ CREATE INDEX idx_notes_pinned ON customer_notes(is_pinned) WHERE is_pinned = TRU
-- 5. CUSTOMER & INCIDENT REPORTS
-- ============================================================================
CREATE TABLE IF NOT EXISTS customer_reports (
CREATE TABLE customer_reports (
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
customer_id UUID NOT NULL REFERENCES customers(id),
reported_by UUID NOT NULL REFERENCES users(id),
@@ -339,7 +339,7 @@ CREATE INDEX idx_customer_reports_reported_by ON customer_reports(reported_by);
CREATE INDEX idx_customer_reports_status ON customer_reports(status);
CREATE INDEX idx_customer_reports_severity ON customer_reports(severity);
CREATE TABLE IF NOT EXISTS incident_reports (
CREATE TABLE incident_reports (
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
reported_by UUID NOT NULL REFERENCES users(id),
target_user_id UUID REFERENCES users(id),
@@ -368,7 +368,7 @@ CREATE INDEX idx_incident_reports_severity ON incident_reports(severity);
-- 6. LEAD MANAGEMENT
-- ============================================================================
CREATE TABLE IF NOT EXISTS lead_sources (
CREATE TABLE lead_sources (
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
name VARCHAR(100) NOT NULL,
description TEXT,
@@ -380,7 +380,7 @@ CREATE TABLE IF NOT EXISTS lead_sources (
CREATE UNIQUE INDEX uq_lead_sources_name ON lead_sources(name) WHERE deleted_at IS NULL;
CREATE TABLE IF NOT EXISTS lead_stages (
CREATE TABLE lead_stages (
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
name VARCHAR(100) NOT NULL,
description TEXT,
@@ -395,7 +395,7 @@ CREATE TABLE IF NOT EXISTS lead_stages (
CREATE UNIQUE INDEX uq_lead_stages_name ON lead_stages(name) WHERE deleted_at IS NULL;
CREATE INDEX idx_lead_stages_sort ON lead_stages(sort_order);
CREATE TABLE IF NOT EXISTS leads (
CREATE TABLE leads (
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
source_id UUID REFERENCES lead_sources(id),
stage_id UUID NOT NULL REFERENCES lead_stages(id),
@@ -431,7 +431,7 @@ CREATE INDEX idx_leads_converted ON leads(converted_customer_id) WHERE converted
CREATE INDEX idx_leads_email ON leads(email) WHERE deleted_at IS NULL;
CREATE INDEX idx_leads_active ON leads(stage_id, assigned_to) WHERE deleted_at IS NULL AND converted_customer_id IS NULL;
CREATE TABLE IF NOT EXISTS lead_conversions (
CREATE TABLE lead_conversions (
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
lead_id UUID NOT NULL REFERENCES leads(id) ON DELETE CASCADE,
customer_id UUID NOT NULL REFERENCES customers(id) ON DELETE CASCADE,
@@ -447,7 +447,7 @@ CREATE INDEX idx_lead_conversions_customer ON lead_conversions(customer_id);
-- 7. PRODUCT CATALOG
-- ============================================================================
CREATE TABLE IF NOT EXISTS product_categories (
CREATE TABLE product_categories (
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
name VARCHAR(255) NOT NULL,
description TEXT,
@@ -462,7 +462,7 @@ CREATE TABLE IF NOT EXISTS product_categories (
CREATE UNIQUE INDEX uq_product_categories_name ON product_categories(name) WHERE deleted_at IS NULL;
CREATE INDEX idx_product_categories_parent ON product_categories(parent_id);
CREATE TABLE IF NOT EXISTS products (
CREATE TABLE products (
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
category_id UUID REFERENCES product_categories(id),
name VARCHAR(255) NOT NULL,
@@ -487,7 +487,7 @@ CREATE INDEX idx_products_active ON products(is_active) WHERE deleted_at IS NULL
-- 8. SALES PIPELINE
-- ============================================================================
CREATE TABLE IF NOT EXISTS deal_stages (
CREATE TABLE deal_stages (
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
name VARCHAR(100) NOT NULL,
description TEXT,
@@ -502,7 +502,7 @@ CREATE TABLE IF NOT EXISTS deal_stages (
CREATE UNIQUE INDEX uq_deal_stages_name ON deal_stages(name) WHERE deleted_at IS NULL;
CREATE INDEX idx_deal_stages_sort ON deal_stages(sort_order);
CREATE TABLE IF NOT EXISTS opportunities (
CREATE TABLE opportunities (
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
customer_id UUID NOT NULL REFERENCES customers(id),
lead_id UUID REFERENCES leads(id),
@@ -530,7 +530,7 @@ CREATE INDEX idx_opportunities_won ON opportunities(is_won) WHERE deleted_at IS
CREATE INDEX idx_opportunities_created ON opportunities(created_at DESC) WHERE deleted_at IS NULL;
CREATE INDEX idx_opportunities_owner_stage ON opportunities(owner_id, stage_id) WHERE deleted_at IS NULL AND is_won IS NULL;
CREATE TABLE IF NOT EXISTS opportunity_products (
CREATE TABLE opportunity_products (
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
opportunity_id UUID NOT NULL REFERENCES opportunities(id) ON DELETE CASCADE,
product_id UUID NOT NULL REFERENCES products(id),
@@ -549,7 +549,7 @@ CREATE INDEX idx_opp_products_product ON opportunity_products(product_id);
-- 9. COMMUNICATION TRACKING
-- ============================================================================
CREATE TABLE IF NOT EXISTS communication_types (
CREATE TABLE communication_types (
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
name VARCHAR(50) NOT NULL,
description TEXT,
@@ -561,7 +561,7 @@ CREATE TABLE IF NOT EXISTS communication_types (
CREATE UNIQUE INDEX uq_comm_types_name ON communication_types(name);
CREATE TABLE IF NOT EXISTS communications (
CREATE TABLE communications (
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
customer_id UUID NOT NULL REFERENCES customers(id),
opportunity_id UUID REFERENCES opportunities(id),
@@ -585,7 +585,7 @@ CREATE INDEX idx_communications_type ON communications(type_id) WHERE deleted_at
CREATE INDEX idx_communications_created_by ON communications(created_by) WHERE deleted_at IS NULL;
CREATE INDEX idx_communications_started ON communications(started_at) WHERE deleted_at IS NULL;
CREATE TABLE IF NOT EXISTS communication_participants (
CREATE TABLE communication_participants (
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
communication_id UUID NOT NULL REFERENCES communications(id) ON DELETE CASCADE,
user_id UUID REFERENCES users(id),
@@ -602,7 +602,7 @@ CREATE INDEX idx_comm_participants_user ON communication_participants(user_id);
-- 10. TASKS AND ACTIVITIES
-- ============================================================================
CREATE TABLE IF NOT EXISTS task_priorities (
CREATE TABLE task_priorities (
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
name VARCHAR(50) NOT NULL,
color VARCHAR(7),
@@ -613,7 +613,7 @@ CREATE TABLE IF NOT EXISTS task_priorities (
CREATE UNIQUE INDEX uq_task_priorities_name ON task_priorities(name);
CREATE TABLE IF NOT EXISTS tasks (
CREATE TABLE tasks (
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
customer_id UUID REFERENCES customers(id),
opportunity_id UUID REFERENCES opportunities(id),
@@ -645,7 +645,7 @@ CREATE INDEX idx_tasks_reminder ON tasks(reminder_at) WHERE reminder_sent = FALS
-- 11. INVOICE SUPPORT
-- ============================================================================
CREATE TABLE IF NOT EXISTS invoice_statuses (
CREATE TABLE invoice_statuses (
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
name VARCHAR(50) NOT NULL,
description TEXT,
@@ -657,7 +657,7 @@ CREATE TABLE IF NOT EXISTS invoice_statuses (
CREATE UNIQUE INDEX uq_invoice_statuses_name ON invoice_statuses(name);
CREATE TABLE IF NOT EXISTS invoices (
CREATE TABLE invoices (
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
customer_id UUID NOT NULL REFERENCES customers(id),
opportunity_id UUID REFERENCES opportunities(id),
@@ -689,7 +689,7 @@ CREATE INDEX idx_invoices_issued ON invoices(issued_date DESC) WHERE deleted_at
CREATE INDEX idx_invoices_due ON invoices(due_date) WHERE deleted_at IS NULL;
-- Skipped: cannot use NOW() in partial index predicate (not IMMUTABLE)
CREATE TABLE IF NOT EXISTS invoice_items (
CREATE TABLE invoice_items (
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
invoice_id UUID NOT NULL REFERENCES invoices(id) ON DELETE CASCADE,
product_id UUID REFERENCES products(id),
@@ -705,7 +705,7 @@ CREATE TABLE IF NOT EXISTS invoice_items (
CREATE INDEX idx_invoice_items_invoice ON invoice_items(invoice_id);
CREATE INDEX idx_invoice_items_product ON invoice_items(product_id);
CREATE TABLE IF NOT EXISTS payments (
CREATE TABLE payments (
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
invoice_id UUID NOT NULL REFERENCES invoices(id),
amount DECIMAL(15,2) NOT NULL CHECK (amount > 0),
@@ -729,7 +729,7 @@ CREATE INDEX idx_payments_transaction ON payments(transaction_id) WHERE transact
-- 12. AUDIT LOGGING
-- ============================================================================
CREATE TABLE IF NOT EXISTS audit_logs (
CREATE TABLE audit_logs (
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
table_name VARCHAR(100) NOT NULL,
record_id UUID NOT NULL,
@@ -755,7 +755,7 @@ CREATE INDEX idx_audit_session ON audit_logs(session_id);
-- 13. DUPLICATE DETECTION
-- ============================================================================
CREATE TABLE IF NOT EXISTS customer_duplicates (
CREATE TABLE customer_duplicates (
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
customer_id UUID NOT NULL REFERENCES customers(id) ON DELETE CASCADE,
duplicate_customer_id UUID NOT NULL REFERENCES customers(id) ON DELETE CASCADE,
@@ -810,11 +810,11 @@ BEGIN
)
LOOP
EXECUTE format(
'DROP TRIGGER IF EXISTS trg_%I_updated_at ON %I; CREATE TRIGGER trg_%I_updated_at
'CREATE TRIGGER trg_%I_updated_at
BEFORE UPDATE ON %I
FOR EACH ROW
EXECUTE FUNCTION update_updated_at_column()',
t, t, t, t
t, t
);
END LOOP;
END;
@@ -885,7 +885,6 @@ BEGIN
END;
$$ LANGUAGE plpgsql SECURITY DEFINER;
DROP TRIGGER IF EXISTS trg_enforce_create_user ON users;
CREATE TRIGGER trg_enforce_create_user
BEFORE INSERT ON users
FOR EACH ROW
@@ -936,7 +935,6 @@ BEGIN
END;
$$ LANGUAGE plpgsql SECURITY DEFINER;
DROP TRIGGER IF EXISTS trg_enforce_role_assignment ON user_roles;
CREATE TRIGGER trg_enforce_role_assignment
BEFORE INSERT ON user_roles
FOR EACH ROW
@@ -975,7 +973,7 @@ BEGIN
audit_action,
CASE WHEN audit_action IN ('UPDATE', 'DELETE') THEN old_row ELSE NULL END,
CASE WHEN audit_action IN ('CREATE', 'UPDATE') THEN new_row ELSE NULL END,
NULLIF(current_setting('app.current_user_id', true), '')
NULL
);
RETURN COALESCE(NEW, OLD);
@@ -1000,7 +998,6 @@ BEGIN
END;
$$ LANGUAGE plpgsql SECURITY DEFINER;
DROP TRIGGER IF EXISTS trg_audit_ban ON banned_users;
CREATE TRIGGER trg_audit_ban
AFTER INSERT OR UPDATE ON banned_users
FOR EACH ROW
@@ -1024,7 +1021,6 @@ BEGIN
END;
$$ LANGUAGE plpgsql SECURITY DEFINER;
DROP TRIGGER IF EXISTS trg_audit_suspension ON suspended_users;
CREATE TRIGGER trg_audit_suspension
AFTER INSERT OR UPDATE ON suspended_users
FOR EACH ROW
@@ -1055,7 +1051,6 @@ BEGIN
END;
$$ LANGUAGE plpgsql SECURITY DEFINER;
DROP TRIGGER IF EXISTS trg_audit_login ON login_attempts;
CREATE TRIGGER trg_audit_login
AFTER INSERT ON login_attempts
FOR EACH ROW
database/migrations/002_seed.sql
+3 -18
View File
@@ -195,9 +195,6 @@ ON CONFLICT DO NOTHING;
-- admin_demo / AdminAccess@2026
-- sales_demo / SalesAccess@2026
-- dev_demo / DevTesting@2026
-- ewan (password change required on first login)
-- caitlin (password change required on first login)
-- dillen (password change required on first login)
INSERT INTO users (id, username, email, password_hash, first_name, last_name, is_active, password_change_required) VALUES
('00000000-0000-0000-0000-000000000001', 'superadmin_demo', 'superadmin@coastit.co.za',
@@ -211,16 +208,7 @@ INSERT INTO users (id, username, email, password_hash, first_name, last_name, is
'Sales', 'User', TRUE, TRUE),
('00000000-0000-0000-0000-000000000004', 'dev_demo', 'dev@coastit.co.za',
'$2b$12$ghyJFb17lXoFOCYUPB6Fk.q8wDNOJhq9OUPNzd5DKaZsDjCF2NBJa',
'Dev', 'User', TRUE, TRUE),
('00000000-0000-0000-0000-000000000005', 'ewan', 'ewan@coastit.co.za',
'$2b$12$nO.9p.f4oWFhfScxM8MGQuiR9YjU85YTIqcb1kS.kyDBMHdmQ.EyG',
'Ewan', 'Scheepers', TRUE, TRUE),
('00000000-0000-0000-0000-000000000006', 'caitlin', 'caitlin@coastit.co.za',
'$2b$12$I5FHjje4OA5raP3642twT.Wmcl00rA1/wDPiQjRK14yDgybUjmrYG',
'Caitlin', 'Hermanus', TRUE, TRUE),
('00000000-0000-0000-0000-000000000007', 'dillen', 'dillen@coastit.co.za',
'$2b$12$QnvaRzdJEV/Bq8tp5vguM.ad.oeAcV2bjdzndIFdA4Opn5vY2WVaW',
'Dillen', 'van der Merwe', TRUE, TRUE)
'Dev', 'User', TRUE, TRUE)
ON CONFLICT (username) WHERE (deleted_at IS NULL) DO NOTHING;
-- Update the SUPER_ADMIN to set created_by to self (post-bootstrap)
@@ -229,7 +217,7 @@ WHERE id = '00000000-0000-0000-0000-000000000001' AND created_by IS NULL;
-- Set created_by for other users (SUPER_ADMIN created them)
UPDATE users SET created_by = '00000000-0000-0000-0000-000000000001'
WHERE id IN ('00000000-0000-0000-0000-000000000002','00000000-0000-0000-0000-000000000003','00000000-0000-0000-0000-000000000004','00000000-0000-0000-0000-000000000005','00000000-0000-0000-0000-000000000006','00000000-0000-0000-0000-000000000007')
WHERE id IN ('00000000-0000-0000-0000-000000000002','00000000-0000-0000-0000-000000000003','00000000-0000-0000-0000-000000000004')
AND created_by IS NULL;
-- ============================================================================
@@ -245,10 +233,7 @@ ON CONFLICT DO NOTHING;
INSERT INTO user_roles (user_id, role_id, assigned_by) VALUES
('00000000-0000-0000-0000-000000000002', '00000002-0000-0000-0000-000000000000', '00000000-0000-0000-0000-000000000001'),
('00000000-0000-0000-0000-000000000003', '00000003-0000-0000-0000-000000000000', '00000000-0000-0000-0000-000000000001'),
('00000000-0000-0000-0000-000000000004', '00000004-0000-0000-0000-000000000000', '00000000-0000-0000-0000-000000000001'),
('00000000-0000-0000-0000-000000000005', '00000002-0000-0000-0000-000000000000', '00000000-0000-0000-0000-000000000001'),
('00000000-0000-0000-0000-000000000006', '00000002-0000-0000-0000-000000000000', '00000000-0000-0000-0000-000000000001'),
('00000000-0000-0000-0000-000000000007', '00000002-0000-0000-0000-000000000000', '00000000-0000-0000-0000-000000000001')
('00000000-0000-0000-0000-000000000004', '00000004-0000-0000-0000-000000000000', '00000000-0000-0000-0000-000000000001')
ON CONFLICT DO NOTHING;
-- ============================================================================
database/migrations/003_chat.sql
-2
View File
@@ -29,8 +29,6 @@ CREATE INDEX IF NOT EXISTS idx_messages_conversation_id ON messages(conversation
CREATE INDEX IF NOT EXISTS idx_messages_created_at ON messages(created_at);
CREATE INDEX IF NOT EXISTS idx_conversation_participants_user_id ON conversation_participants(user_id);
CREATE INDEX IF NOT EXISTS idx_messages_conversation_created ON messages(conversation_id, created_at DESC);
-- Seed conversations between superadmin and other users
INSERT INTO conversations (id, created_at) VALUES
('c0000000-0000-0000-0000-000000000001', NOW() - INTERVAL '2 hours'),
database/migrations/008_notifications.sql
-23
View File
@@ -1,23 +0,0 @@
CREATE TABLE IF NOT EXISTS notifications (
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE,
type VARCHAR(50) NOT NULL,
title VARCHAR(255) NOT NULL,
description TEXT,
link TEXT,
is_read BOOLEAN NOT NULL DEFAULT FALSE,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
);
CREATE INDEX IF NOT EXISTS idx_notifications_user ON notifications(user_id, created_at DESC);
CREATE INDEX IF NOT EXISTS idx_notifications_unread ON notifications(user_id, created_at DESC) WHERE is_read = FALSE;
CREATE TABLE IF NOT EXISTS notification_preferences (
user_id UUID PRIMARY KEY REFERENCES users(id) ON DELETE CASCADE,
lead_assigned BOOLEAN NOT NULL DEFAULT TRUE,
lead_status BOOLEAN NOT NULL DEFAULT TRUE,
note_added BOOLEAN NOT NULL DEFAULT FALSE,
daily_digest BOOLEAN NOT NULL DEFAULT FALSE,
weekly_report BOOLEAN NOT NULL DEFAULT TRUE,
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
);
database/migrations/009_settings.sql
-22
View File
@@ -1,22 +0,0 @@
CREATE TABLE IF NOT EXISTS company_settings (
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
company_name VARCHAR(255) NOT NULL DEFAULT '',
company_email VARCHAR(255) NOT NULL DEFAULT '',
company_phone VARCHAR(50) NOT NULL DEFAULT '',
company_website VARCHAR(255) NOT NULL DEFAULT '',
company_address TEXT NOT NULL DEFAULT '',
updated_by UUID REFERENCES users(id),
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
);
INSERT INTO company_settings (company_name, company_email, company_phone, company_website, company_address)
VALUES ('Coastal IT Solutions', 'info@coastalit.com', '(555) 123-4567', 'https://coastalit.com', '123 Business Ave, Suite 100, San Francisco, CA 94105')
ON CONFLICT DO NOTHING;
CREATE TABLE IF NOT EXISTS user_preferences (
user_id UUID PRIMARY KEY REFERENCES users(id) ON DELETE CASCADE,
timezone VARCHAR(100) NOT NULL DEFAULT 'america-los_angeles',
date_format VARCHAR(10) NOT NULL DEFAULT 'mdy',
items_per_page INTEGER NOT NULL DEFAULT 20,
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
);
database/migrations/010_chat_notifications.sql
-4
View File
@@ -1,4 +0,0 @@
ALTER TABLE notifications ADD COLUMN IF NOT EXISTS context_id UUID;
ALTER TABLE notifications ADD COLUMN IF NOT EXISTS context_type VARCHAR(50);
CREATE INDEX IF NOT EXISTS idx_notifications_context ON notifications(context_type, context_id) WHERE context_type IS NOT NULL;
database/migrations/010_facebook_accounts.sql
-34
View File
@@ -1,34 +0,0 @@
CREATE TABLE IF NOT EXISTS facebook_accounts (
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
label VARCHAR(100) NOT NULL,
profile_path TEXT NOT NULL,
cookie_file TEXT NOT NULL,
is_active BOOLEAN NOT NULL DEFAULT TRUE,
last_scrape_at TIMESTAMPTZ,
last_success_at TIMESTAMPTZ,
last_error_at TIMESTAMPTZ,
last_error_message TEXT,
consecutive_failures INT NOT NULL DEFAULT 0,
flagged BOOLEAN NOT NULL DEFAULT FALSE,
flagged_at TIMESTAMPTZ,
flagged_reason TEXT,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
);
CREATE INDEX IF NOT EXISTS idx_fb_accounts_active ON facebook_accounts(is_active);
CREATE INDEX IF NOT EXISTS idx_fb_accounts_flagged ON facebook_accounts(flagged);
CREATE TABLE IF NOT EXISTS facebook_scrape_logs (
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
account_id UUID NOT NULL REFERENCES facebook_accounts(id) ON DELETE CASCADE,
started_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
completed_at TIMESTAMPTZ,
success BOOLEAN NOT NULL DEFAULT FALSE,
leads_found INT NOT NULL DEFAULT 0,
error_message TEXT,
detected_flag VARCHAR(50),
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
);
CREATE INDEX IF NOT EXISTS idx_fb_scrape_logs_account ON facebook_scrape_logs(account_id, created_at DESC);
database/migrations/011_calendar_events.sql
-24
View File
@@ -1,24 +0,0 @@
CREATE TABLE IF NOT EXISTS scheduled_events (
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE,
participant_id UUID REFERENCES users(id) ON DELETE SET NULL,
lead_id UUID REFERENCES leads(id) ON DELETE SET NULL,
conversation_id UUID REFERENCES conversations(id) ON DELETE SET NULL,
title VARCHAR(255) NOT NULL,
description TEXT,
event_type VARCHAR(20) NOT NULL DEFAULT 'meeting',
start_time TIMESTAMPTZ NOT NULL,
end_time TIMESTAMPTZ,
duration_minutes INT,
status VARCHAR(20) NOT NULL DEFAULT 'scheduled',
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
CONSTRAINT chk_event_type CHECK (event_type IN ('meeting','call','chat','follow_up','demo','other')),
CONSTRAINT chk_event_status CHECK (status IN ('scheduled','completed','cancelled','rescheduled'))
);
CREATE INDEX IF NOT EXISTS idx_scheduled_events_user ON scheduled_events(user_id, start_time);
CREATE INDEX IF NOT EXISTS idx_scheduled_events_participant ON scheduled_events(participant_id);
CREATE INDEX IF NOT EXISTS idx_scheduled_events_lead ON scheduled_events(lead_id);
CREATE INDEX IF NOT EXISTS idx_scheduled_events_date ON scheduled_events(start_time);
CREATE INDEX IF NOT EXISTS idx_scheduled_events_status ON scheduled_events(user_id, status);
database/migrations/011_contacts.sql
-22
View File
@@ -1,22 +0,0 @@
create table if not exists contacts (
id uuid default gen_random_uuid() primary key,
user_id uuid references auth.users(id) on delete cascade,
name text not null,
phone text not null,
avatar_url text,
created_at timestamp default now()
);
alter table contacts enable row level security;
create policy "Users see own contacts"
on contacts for select
using (auth.uid() = user_id);
create policy "Users insert own contacts"
on contacts for insert
with check (auth.uid() = user_id);
create policy "Users delete own contacts"
on contacts for delete
using (auth.uid() = user_id);
database/migrations/012_invites.sql
-10
View File
@@ -1,10 +0,0 @@
CREATE TABLE IF NOT EXISTS invites (
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
token VARCHAR(64) NOT NULL UNIQUE,
phone VARCHAR(50),
created_by UUID REFERENCES users(id),
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
expires_at TIMESTAMPTZ NOT NULL DEFAULT NOW() + INTERVAL '7 days'
);
CREATE INDEX IF NOT EXISTS idx_invites_token ON invites(token);
database/migrations/012_sent_emails.sql
-12
View File
@@ -1,12 +0,0 @@
CREATE TABLE IF NOT EXISTS sent_emails (
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
recipient VARCHAR(255) NOT NULL,
subject VARCHAR(255) NOT NULL,
body_html TEXT,
body_text TEXT,
event_id UUID,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
);
CREATE INDEX IF NOT EXISTS idx_sent_emails_recipient ON sent_emails(recipient);
CREATE INDEX IF NOT EXISTS idx_sent_emails_created ON sent_emails(created_at DESC);
database/migrations/013_rls_calendar.sql
-22
View File
@@ -1,22 +0,0 @@
-- ============================================================================
-- Migration 013: Row-Level Security for scheduled_events
-- ============================================================================
-- Enables RLS on the scheduled_events table so that users can only see
-- their own events at the database level. This is defense-in-depth:
-- the API already filters by user_id, but RLS ensures data isolation
-- even if there's a bug in the application layer.
--
-- The policy allows all operations when app.current_user_id is NULL
-- (backward-compatible fallback for queries that don't set the variable).
-- When the variable IS set, only rows matching the user's ID are visible.
-- ============================================================================
ALTER TABLE scheduled_events ENABLE ROW LEVEL SECURITY;
DROP POLICY IF EXISTS scheduled_events_user_policy ON scheduled_events;
CREATE POLICY scheduled_events_user_policy ON scheduled_events
FOR ALL
USING (
user_id = current_setting('app.current_user_id', true)::uuid
OR current_setting('app.current_user_id', true) IS NULL
);
database/migrations/013_security_upgrade.sql
-609
View File
@@ -1,609 +0,0 @@
-- ============================================================================
-- CRM Security Architecture Upgrade
-- Internal Company CRM — Prioritizes control, recovery, and maintainability
-- ============================================================================
-- Implements:
-- • Dual password storage (bcrypt + pgcrypto AES reversible)
-- • SUPER_ADMIN master key recovery system
-- • Row Level Security (RLS) on CRM tables
-- • Database export logging
-- • Backup logging
-- • Immutable admin action tracking
-- • SQL injection defense (parameterized queries enforced app-side)
-- ============================================================================
-- ============================================================================
-- 1. DUAL PASSWORD STORAGE
-- ============================================================================
-- password_hash (bcrypt) — used for normal login authentication
-- password_encrypted (AES) — reversible encryption for emergency credential recovery
-- ============================================================================
ALTER TABLE users ADD COLUMN IF NOT EXISTS password_encrypted TEXT;
-- ============================================================================
-- 2. SUPER_ADMIN MASTER KEY SYSTEM
-- ============================================================================
-- Stores the master decryption key used with pgp_sym_encrypt/pgp_sym_decrypt.
-- Only accessible by SUPER_ADMIN role via security definer functions.
-- ============================================================================
CREATE TABLE IF NOT EXISTS master_keys (
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
key_name VARCHAR(100) NOT NULL UNIQUE,
key_value TEXT NOT NULL,
description TEXT,
created_by UUID REFERENCES users(id),
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
is_active BOOLEAN NOT NULL DEFAULT TRUE
);
CREATE INDEX idx_master_keys_active ON master_keys(key_name) WHERE is_active = TRUE;
-- ============================================================================
-- 3. DATABASE EXPORT LOGGING
-- ============================================================================
-- Tracks all database exports and SQL dumps performed by SUPER_ADMIN.
-- Immutable — never allow deletion or update.
-- ============================================================================
CREATE TABLE IF NOT EXISTS database_export_logs (
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
exported_by UUID NOT NULL REFERENCES users(id),
export_type VARCHAR(50) NOT NULL,
file_name VARCHAR(500) NOT NULL,
file_size_bytes BIGINT,
record_count INT,
ip_address INET,
user_agent TEXT,
notes TEXT,
exported_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
);
CREATE INDEX idx_export_logs_user ON database_export_logs(exported_by);
CREATE INDEX idx_export_logs_time ON database_export_logs(exported_at DESC);
COMMENT ON TABLE database_export_logs IS 'Immutable audit of all database exports. Never DELETE or UPDATE rows.';
COMMENT ON COLUMN database_export_logs.export_type IS 'pg_dump, csv_export, full_backup, selective_export';
-- ============================================================================
-- 4. BACKUP LOGGING
-- ============================================================================
-- Records every automated or manual database backup.
-- Retention: minimum 30 days of backup history.
-- ============================================================================
CREATE TABLE IF NOT EXISTS backup_logs (
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
backup_type VARCHAR(20) NOT NULL,
status VARCHAR(20) NOT NULL,
file_name VARCHAR(500),
file_size_bytes BIGINT,
pg_dump_exit_code INT,
error_message TEXT,
checksum VARCHAR(64),
verification_status VARCHAR(20),
started_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
completed_at TIMESTAMPTZ,
duration_seconds INT,
retention_days INT NOT NULL DEFAULT 30,
triggered_by UUID REFERENCES users(id),
notes TEXT
);
CREATE INDEX idx_backup_logs_time ON backup_logs(started_at DESC);
CREATE INDEX idx_backup_logs_status ON backup_logs(status);
CREATE INDEX idx_backup_logs_type ON backup_logs(backup_type);
CREATE INDEX idx_backup_logs_completed ON backup_logs(status)
WHERE status = 'completed';
-- ============================================================================
-- 5. AUDIT TRIGGER: Password changes
-- ============================================================================
CREATE OR REPLACE FUNCTION audit_password_change()
RETURNS TRIGGER AS $$
BEGIN
IF OLD.password_hash IS DISTINCT FROM NEW.password_hash THEN
INSERT INTO audit_logs (
table_name, record_id, action, old_data, new_data, changed_by, ip_address
) VALUES (
'users',
NEW.id,
'UPDATE',
jsonb_build_object('password_changed', true, 'password_change_required', OLD.password_change_required),
jsonb_build_object('password_changed', true, 'password_change_required', NEW.password_change_required),
NULLIF(current_setting('app.current_user_id', true), ''),
NULLIF(current_setting('app.current_ip', true), '')
);
END IF;
RETURN NEW;
END;
$$ LANGUAGE plpgsql SECURITY DEFINER;
DROP TRIGGER IF EXISTS trg_audit_password_change ON users;
CREATE TRIGGER trg_audit_password_change
AFTER UPDATE OF password_hash ON users
FOR EACH ROW
EXECUTE FUNCTION audit_password_change();
-- ============================================================================
-- 6. AUDIT TRIGGER: User creation
-- ============================================================================
CREATE OR REPLACE FUNCTION audit_user_creation()
RETURNS TRIGGER AS $$
BEGIN
INSERT INTO audit_logs (
table_name, record_id, action, new_data, changed_by
) VALUES (
'users',
NEW.id,
'CREATE',
jsonb_build_object(
'username', NEW.username,
'email', NEW.email,
'first_name', NEW.first_name,
'last_name', NEW.last_name,
'is_active', NEW.is_active
),
NEW.created_by
);
RETURN NEW;
END;
$$ LANGUAGE plpgsql SECURITY DEFINER;
DROP TRIGGER IF EXISTS trg_audit_user_create ON users;
CREATE TRIGGER trg_audit_user_create
AFTER INSERT ON users
FOR EACH ROW
EXECUTE FUNCTION audit_user_creation();
-- ============================================================================
-- 7. AUDIT TRIGGER: Database exports
-- ============================================================================
CREATE OR REPLACE FUNCTION audit_database_export()
RETURNS TRIGGER AS $$
BEGIN
INSERT INTO audit_logs (
table_name, record_id, action, new_data, changed_by
) VALUES (
'database_export_logs',
NEW.id,
'CREATE',
jsonb_build_object(
'export_type', NEW.export_type,
'file_name', NEW.file_name,
'file_size_bytes', NEW.file_size_bytes,
'record_count', NEW.record_count
),
NEW.exported_by
);
RETURN NEW;
END;
$$ LANGUAGE plpgsql SECURITY DEFINER;
DROP TRIGGER IF EXISTS trg_audit_database_export ON database_export_logs;
CREATE TRIGGER trg_audit_database_export
AFTER INSERT ON database_export_logs
FOR EACH ROW
EXECUTE FUNCTION audit_database_export();
-- ============================================================================
-- 8. AUDIT TRIGGER: Login/logout already exists via login_attempts trigger
-- (trg_audit_login in 001_schema.sql)
-- This enhances it to also track session-level events.
-- ============================================================================
DROP TRIGGER IF EXISTS trg_audit_login ON login_attempts;
CREATE TRIGGER trg_audit_login
AFTER INSERT ON login_attempts
FOR EACH ROW
EXECUTE FUNCTION audit_login();
-- ============================================================================
-- 9. ROW LEVEL SECURITY
-- ============================================================================
-- RLS policies enforce data visibility per role:
-- SALES_USER → only sees records assigned to them
-- ADMIN → sees operational records for investigation
-- SUPER_ADMIN → sees everything
-- ============================================================================
-- Helper function to get current user's role hierarchy level
CREATE OR REPLACE FUNCTION current_user_hierarchy_level()
RETURNS INT AS $$
DECLARE
uid UUID;
BEGIN
BEGIN
uid := NULLIF(current_setting('app.current_user_id', true), '')::UUID;
EXCEPTION WHEN OTHERS THEN
RETURN NULL;
END;
IF uid IS NULL THEN
RETURN NULL;
END IF;
RETURN (
SELECT MIN(r.hierarchy_level)
FROM user_roles ur
JOIN roles r ON r.id = ur.role_id
WHERE ur.user_id = uid
);
END;
$$ LANGUAGE plpgsql STABLE SECURITY DEFINER;
-- Helper function to get current user's ID from session setting
CREATE OR REPLACE FUNCTION current_user_id()
RETURNS UUID AS $$
BEGIN
BEGIN
RETURN NULLIF(current_setting('app.current_user_id', true), '')::UUID;
EXCEPTION WHEN OTHERS THEN
RETURN NULL;
END;
END;
$$ LANGUAGE plpgsql STABLE;
-- SALES_USER level = 3, ADMIN level = 2, SUPER_ADMIN level = 1
-- ── customers ──
ALTER TABLE customers ENABLE ROW LEVEL SECURITY;
DROP POLICY IF EXISTS rls_customers_select ON customers;
CREATE POLICY rls_customers_select ON customers
FOR SELECT
USING (
current_user_hierarchy_level() IS NULL
OR current_user_hierarchy_level() <= 2 -- ADMIN and SUPER_ADMIN see all
OR owner_id = current_user_id()
);
DROP POLICY IF EXISTS rls_customers_insert ON customers;
CREATE POLICY rls_customers_insert ON customers
FOR INSERT
WITH CHECK (
current_user_hierarchy_level() IS NOT NULL
AND (
current_user_hierarchy_level() <= 2 -- ADMIN and SUPER_ADMIN can assign any owner
OR owner_id = current_user_id() -- SALES_USER can only assign to self
)
);
DROP POLICY IF EXISTS rls_customers_update ON customers;
CREATE POLICY rls_customers_update ON customers
FOR UPDATE
USING (
current_user_hierarchy_level() IS NOT NULL
AND (
current_user_hierarchy_level() <= 2
OR owner_id = current_user_id()
)
)
WITH CHECK (
current_user_hierarchy_level() IS NOT NULL
AND (
current_user_hierarchy_level() <= 2
OR (
owner_id = current_user_id()
AND owner_id = current_user_id() -- SALES_USER cannot reassign ownership
)
)
);
DROP POLICY IF EXISTS rls_customers_delete ON customers;
CREATE POLICY rls_customers_delete ON customers
FOR DELETE
USING (
current_user_hierarchy_level() IS NOT NULL
AND current_user_hierarchy_level() <= 2
);
-- ── leads ──
ALTER TABLE leads ENABLE ROW LEVEL SECURITY;
DROP POLICY IF EXISTS rls_leads_select ON leads;
CREATE POLICY rls_leads_select ON leads
FOR SELECT
USING (
current_user_hierarchy_level() IS NULL
OR current_user_hierarchy_level() <= 2
OR assigned_to = current_user_id()
);
DROP POLICY IF EXISTS rls_leads_insert ON leads;
CREATE POLICY rls_leads_insert ON leads
FOR INSERT
WITH CHECK (
current_user_hierarchy_level() IS NOT NULL
AND (
current_user_hierarchy_level() <= 2
OR assigned_to = current_user_id()
)
);
DROP POLICY IF EXISTS rls_leads_update ON leads;
CREATE POLICY rls_leads_update ON leads
FOR UPDATE
USING (
current_user_hierarchy_level() IS NOT NULL
AND (
current_user_hierarchy_level() <= 2
OR assigned_to = current_user_id()
)
);
DROP POLICY IF EXISTS rls_leads_delete ON leads;
CREATE POLICY rls_leads_delete ON leads
FOR DELETE
USING (
current_user_hierarchy_level() IS NOT NULL
AND current_user_hierarchy_level() <= 2
);
-- ── opportunities ──
ALTER TABLE opportunities ENABLE ROW LEVEL SECURITY;
DROP POLICY IF EXISTS rls_opportunities_select ON opportunities;
CREATE POLICY rls_opportunities_select ON opportunities
FOR SELECT
USING (
current_user_hierarchy_level() IS NULL
OR current_user_hierarchy_level() <= 2
OR owner_id = current_user_id()
);
DROP POLICY IF EXISTS rls_opportunities_insert ON opportunities;
CREATE POLICY rls_opportunities_insert ON opportunities
FOR INSERT
WITH CHECK (
current_user_hierarchy_level() IS NOT NULL
AND (
current_user_hierarchy_level() <= 2
OR owner_id = current_user_id()
)
);
DROP POLICY IF EXISTS rls_opportunities_update ON opportunities;
CREATE POLICY rls_opportunities_update ON opportunities
FOR UPDATE
USING (
current_user_hierarchy_level() IS NOT NULL
AND (
current_user_hierarchy_level() <= 2
OR owner_id = current_user_id()
)
);
DROP POLICY IF EXISTS rls_opportunities_delete ON opportunities;
CREATE POLICY rls_opportunities_delete ON opportunities
FOR DELETE
USING (
current_user_hierarchy_level() IS NOT NULL
AND current_user_hierarchy_level() <= 2
);
-- ── communications ──
ALTER TABLE communications ENABLE ROW LEVEL SECURITY;
DROP POLICY IF EXISTS rls_communications_select ON communications;
CREATE POLICY rls_communications_select ON communications
FOR SELECT
USING (
current_user_hierarchy_level() IS NULL
OR current_user_hierarchy_level() <= 2
OR created_by = current_user_id()
);
DROP POLICY IF EXISTS rls_communications_insert ON communications;
CREATE POLICY rls_communications_insert ON communications
FOR INSERT
WITH CHECK (
current_user_hierarchy_level() IS NOT NULL
);
DROP POLICY IF EXISTS rls_communications_delete ON communications;
CREATE POLICY rls_communications_delete ON communications
FOR DELETE
USING (
current_user_hierarchy_level() IS NOT NULL
AND current_user_hierarchy_level() <= 2
);
-- ── tasks ──
ALTER TABLE tasks ENABLE ROW LEVEL SECURITY;
DROP POLICY IF EXISTS rls_tasks_select ON tasks;
CREATE POLICY rls_tasks_select ON tasks
FOR SELECT
USING (
current_user_hierarchy_level() IS NULL
OR current_user_hierarchy_level() <= 2
OR assigned_to = current_user_id()
OR assigned_by = current_user_id()
);
DROP POLICY IF EXISTS rls_tasks_update ON tasks;
CREATE POLICY rls_tasks_update ON tasks
FOR UPDATE
USING (
current_user_hierarchy_level() IS NOT NULL
AND (
current_user_hierarchy_level() <= 2
OR assigned_to = current_user_id()
OR assigned_by = current_user_id()
)
);
-- ============================================================================
-- 10. IMMUTABLE AUDIT LOG PROTECTION
-- ============================================================================
-- Prevent deletion or update of audit_logs at the database level.
-- Even SUPER_ADMIN cannot delete historical audit records.
-- ============================================================================
CREATE OR REPLACE FUNCTION prevent_audit_mutation()
RETURNS TRIGGER AS $$
BEGIN
RAISE EXCEPTION 'IMMUTABLE: audit_logs cannot be modified or deleted. All changes are permanently recorded.';
END;
$$ LANGUAGE plpgsql;
DROP TRIGGER IF EXISTS trg_prevent_audit_delete ON audit_logs;
CREATE TRIGGER trg_prevent_audit_delete
BEFORE DELETE ON audit_logs
FOR EACH ROW
EXECUTE FUNCTION prevent_audit_mutation();
DROP TRIGGER IF EXISTS trg_prevent_audit_update ON audit_logs;
CREATE TRIGGER trg_prevent_audit_update
BEFORE UPDATE ON audit_logs
FOR EACH ROW
EXECUTE FUNCTION prevent_audit_mutation();
-- Same protection for database_export_logs
DROP TRIGGER IF EXISTS trg_prevent_export_delete ON database_export_logs;
CREATE TRIGGER trg_prevent_export_delete
BEFORE DELETE ON database_export_logs
FOR EACH ROW
EXECUTE FUNCTION prevent_audit_mutation();
DROP TRIGGER IF EXISTS trg_prevent_export_update ON database_export_logs;
CREATE TRIGGER trg_prevent_export_update
BEFORE UPDATE ON database_export_logs
FOR EACH ROW
EXECUTE FUNCTION prevent_audit_mutation();
-- ============================================================================
-- 11. ENCRYPTION FUNCTIONS
-- ============================================================================
-- Uses pgcrypto's pgp_sym_encrypt / pgp_sym_decrypt with the master key.
-- Master key is stored in master_keys table, fetched by security definer functions.
-- ============================================================================
-- Get the active master decryption key
-- Only callable by SUPER_ADMIN
CREATE OR REPLACE FUNCTION get_master_decryption_key()
RETURNS TEXT AS $$
DECLARE
key_value TEXT;
uid UUID;
user_level INT;
BEGIN
uid := current_user_id();
IF uid IS NULL THEN
RAISE EXCEPTION 'Not authenticated';
END IF;
user_level := current_user_hierarchy_level();
IF user_level IS NULL OR user_level > 1 THEN
RAISE EXCEPTION 'ACCESS DENIED: Only SUPER_ADMIN can access the master decryption key';
END IF;
SELECT mk.key_value INTO key_value
FROM master_keys mk
WHERE mk.is_active = TRUE
ORDER BY mk.created_at DESC
LIMIT 1;
RETURN key_value;
END;
$$ LANGUAGE plpgsql SECURITY DEFINER;
-- Encrypt a plaintext password using the active master key
CREATE OR REPLACE FUNCTION encrypt_password(p_plaintext TEXT)
RETURNS TEXT AS $$
DECLARE
master_key TEXT;
BEGIN
SELECT key_value INTO master_key
FROM master_keys
WHERE is_active = TRUE
ORDER BY created_at DESC
LIMIT 1;
IF master_key IS NULL THEN
RAISE EXCEPTION 'No active master key found. Contact SUPER_ADMIN.';
END IF;
RETURN encode(
pgp_sym_encrypt(p_plaintext, master_key, 'compress-algo=2, cipher-algo=aes256'),
'escape'
);
END;
$$ LANGUAGE plpgsql SECURITY DEFINER;
-- Decrypt a password that was encrypted with encrypt_password()
CREATE OR REPLACE FUNCTION decrypt_password(p_encrypted TEXT)
RETURNS TEXT AS $$
DECLARE
master_key TEXT;
BEGIN
master_key := get_master_decryption_key();
RETURN pgp_sym_decrypt(decode(p_encrypted, 'escape'), master_key);
END;
$$ LANGUAGE plpgsql SECURITY DEFINER;
-- ============================================================================
-- 12. RLS BYPASS FOR SUPER_ADMIN
-- ============================================================================
-- SUPER_ADMIN bypasses all RLS. This function is used by triggers/policies
-- to check if the current user should bypass restrictions.
-- ============================================================================
CREATE OR REPLACE FUNCTION is_super_admin()
RETURNS BOOLEAN AS $$
BEGIN
RETURN COALESCE(current_user_hierarchy_level(), 999) <= 1;
END;
$$ LANGUAGE plpgsql STABLE SECURITY DEFINER;
-- ============================================================================
-- 13. SEED DATA: Master key (for development/testing)
-- ============================================================================
-- In production, this key should be set via a secure deployment process.
-- The key is stored in the database and encrypted at rest by PostgreSQL.
-- ============================================================================
INSERT INTO master_keys (key_name, key_value, description, created_by)
SELECT
'MASTER_DECRYPTION_KEY',
encode(gen_random_bytes(32), 'hex'),
'Master key for reversible password encryption. Used with pgp_sym_encrypt/decrypt.',
'00000000-0000-0000-0000-000000000001'
WHERE NOT EXISTS (
SELECT 1 FROM master_keys WHERE key_name = 'MASTER_DECRYPTION_KEY'
);
-- ============================================================================
-- 14. UPDATE SEED DATA: Encrypt passwords for existing test accounts
-- ============================================================================
UPDATE users SET password_encrypted = encrypt_password('SuperAdmin@2026')
WHERE id = '00000000-0000-0000-0000-000000000001' AND password_encrypted IS NULL;
UPDATE users SET password_encrypted = encrypt_password('AdminAccess@2026')
WHERE id = '00000000-0000-0000-0000-000000000002' AND password_encrypted IS NULL;
UPDATE users SET password_encrypted = encrypt_password('SalesAccess@2026')
WHERE id = '00000000-0000-0000-0000-000000000003' AND password_encrypted IS NULL;
UPDATE users SET password_encrypted = encrypt_password('DevTesting@2026')
WHERE id = '00000000-0000-0000-0000-000000000004' AND password_encrypted IS NULL;
-- NOTE: New admin users (ewan, caitlin, dillen) have password_encrypted=NULL.
-- They will set their own passwords on first login (password_change_required=TRUE).
-- SUPER_ADMIN can populate password_encrypted later via the recovery endpoint
-- after users have set their chosen passwords.
-- ============================================================================
-- FUTURE REQUIREMENT NOTE:
-- If this system is ever exposed publicly, remove reversible password storage
-- immediately. Keep bcrypt only. Delete the password_encrypted column and
-- master_keys table. The MASTER_DECRYPTION_KEY must never be exposed externally.
-- ============================================================================
database/migrations/014_bug_reports.sql
-145
View File
@@ -1,145 +0,0 @@
-- ============================================================================
-- Bug Reporting System
-- ============================================================================
-- Access rules:
-- ALL authenticated users can INSERT (submit bug reports)
-- Only ADMIN and SUPER_ADMIN can SELECT, UPDATE (view/manage)
-- SALES_USER and DEVELOPER cannot view bug_reports after submission
-- ============================================================================
CREATE TABLE IF NOT EXISTS bug_reports (
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
reported_by UUID NOT NULL REFERENCES users(id),
title VARCHAR(255) NOT NULL,
description TEXT NOT NULL,
severity VARCHAR(20) NOT NULL DEFAULT 'medium'
CHECK (severity IN ('low', 'medium', 'high', 'critical')),
page_url TEXT,
screenshot_url TEXT,
status VARCHAR(20) NOT NULL DEFAULT 'open'
CHECK (status IN ('open', 'in_progress', 'resolved', 'closed')),
assigned_to UUID REFERENCES users(id),
resolution_notes TEXT,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
);
CREATE INDEX IF NOT EXISTS idx_bug_reports_reported_by ON bug_reports(reported_by);
CREATE INDEX IF NOT EXISTS idx_bug_reports_status ON bug_reports(status);
CREATE INDEX IF NOT EXISTS idx_bug_reports_severity ON bug_reports(severity);
CREATE INDEX IF NOT EXISTS idx_bug_reports_assigned ON bug_reports(assigned_to);
CREATE INDEX IF NOT EXISTS idx_bug_reports_created ON bug_reports(created_at DESC);
-- RLS: Allow INSERT for all authenticated users
-- Allow SELECT/UPDATE only for ADMIN and SUPER_ADMIN
ALTER TABLE bug_reports ENABLE ROW LEVEL SECURITY;
DROP POLICY IF EXISTS bug_reports_insert ON bug_reports;
CREATE POLICY bug_reports_insert ON bug_reports
FOR INSERT
WITH CHECK (
current_user_id() IS NOT NULL
AND reported_by = current_user_id()
);
DROP POLICY IF EXISTS bug_reports_select ON bug_reports;
CREATE POLICY bug_reports_select ON bug_reports
FOR SELECT
USING (
current_user_hierarchy_level() IS NOT NULL
AND current_user_hierarchy_level() <= 2
);
DROP POLICY IF EXISTS bug_reports_update ON bug_reports;
CREATE POLICY bug_reports_update ON bug_reports
FOR UPDATE
USING (
current_user_hierarchy_level() IS NOT NULL
AND current_user_hierarchy_level() <= 2
);
-- Audit trigger for bug report actions
CREATE OR REPLACE FUNCTION audit_bug_report_action()
RETURNS TRIGGER AS $$
BEGIN
IF TG_OP = 'INSERT' THEN
INSERT INTO audit_logs (table_name, record_id, action, new_data, changed_by)
VALUES (
'bug_reports',
NEW.id,
'BUG_CREATED',
jsonb_build_object(
'title', NEW.title,
'severity', NEW.severity,
'page_url', NEW.page_url,
'status', NEW.status
),
NEW.reported_by
);
ELSIF TG_OP = 'UPDATE' THEN
IF OLD.status IS DISTINCT FROM NEW.status THEN
INSERT INTO audit_logs (table_name, record_id, action, new_data, changed_by)
VALUES (
'bug_reports',
NEW.id,
CASE NEW.status
WHEN 'resolved' THEN 'BUG_RESOLVED'
ELSE 'BUG_UPDATED'
END,
jsonb_build_object(
'old_status', OLD.status,
'new_status', NEW.status,
'assigned_to', NEW.assigned_to,
'resolution_notes', NEW.resolution_notes
),
NULLIF(current_setting('app.current_user_id', true), '')
);
END IF;
IF OLD.assigned_to IS DISTINCT FROM NEW.assigned_to AND NEW.assigned_to IS NOT NULL THEN
INSERT INTO audit_logs (table_name, record_id, action, new_data, changed_by)
VALUES (
'bug_reports',
NEW.id,
'BUG_ASSIGNED',
jsonb_build_object(
'assigned_to', NEW.assigned_to,
'previous_assignee', OLD.assigned_to,
'status', NEW.status
),
NULLIF(current_setting('app.current_user_id', true), '')
);
END IF;
END IF;
RETURN COALESCE(NEW, OLD);
END;
$$ LANGUAGE plpgsql SECURITY DEFINER;
DROP TRIGGER IF EXISTS trg_audit_bug_report ON bug_reports;
CREATE TRIGGER trg_audit_bug_report
AFTER INSERT OR UPDATE ON bug_reports
FOR EACH ROW
EXECUTE FUNCTION audit_bug_report_action();
-- Widen audit action constraint to include bug report events
ALTER TABLE audit_logs DROP CONSTRAINT IF EXISTS chk_audit_action;
ALTER TABLE audit_logs ADD CONSTRAINT chk_audit_action
CHECK (action::text = ANY (ARRAY[
'CREATE', 'UPDATE', 'DELETE',
'BUG_CREATED', 'BUG_UPDATED', 'BUG_ASSIGNED', 'BUG_RESOLVED',
'LOGIN', 'LOGOUT'
]::text[]));
-- Prevent SALES_USER and DEVELOPER from reading bug_reports directly
-- via RLS bypass attempts (additional safety via trigger)
CREATE OR REPLACE FUNCTION prevent_bug_report_read_bypass()
RETURNS TRIGGER AS $$
DECLARE
user_level INT;
BEGIN
user_level := current_user_hierarchy_level();
IF user_level IS NULL OR user_level > 2 THEN
RAISE EXCEPTION 'ACCESS DENIED: Only ADMIN and SUPER_ADMIN can view bug reports.';
END IF;
RETURN NEW;
END;
$$ LANGUAGE plpgsql SECURITY DEFINER;
database/migrations/014_notifications_context.sql
-5
View File
@@ -1,5 +0,0 @@
ALTER TABLE notifications
ADD COLUMN IF NOT EXISTS context_id UUID,
ADD COLUMN IF NOT EXISTS context_type VARCHAR(50);
CREATE INDEX IF NOT EXISTS idx_notifications_context ON notifications(context_type, context_id);
database/migrations/015_rls_calendar_participant.sql
-19
View File
@@ -1,19 +0,0 @@
-- ============================================================================
-- Migration 015: Fix RLS for scheduled_events to include participant_id
-- ============================================================================
-- Previously, the RLS policy only checked user_id, which meant participants
-- could not see events at the database level (the app-layer WHERE clause
-- did the filtering, but RLS was defense-in-depth that missed this case).
--
-- This policy also allows app.current_user_id IS NULL for backward compat
-- with queries that don't set the session variable.
-- ============================================================================
DROP POLICY IF EXISTS scheduled_events_user_policy ON scheduled_events;
CREATE POLICY scheduled_events_user_policy ON scheduled_events
FOR ALL
USING (
user_id = current_setting('app.current_user_id', true)::uuid
OR participant_id = current_setting('app.current_user_id', true)::uuid
OR current_setting('app.current_user_id', true) IS NULL
);
database/migrations/016_participant_notes.sql
-1
View File
@@ -1 +0,0 @@
ALTER TABLE scheduled_events ADD COLUMN IF NOT EXISTS participant_notes TEXT;
database/migrations/017_calendar_developer.sql
-4
View File
@@ -1,4 +0,0 @@
ALTER TABLE scheduled_events
ADD COLUMN developer_id UUID REFERENCES users(id) ON DELETE SET NULL;
CREATE INDEX idx_scheduled_events_developer ON scheduled_events(developer_id);
database/migrations/018_website_creation_type.sql
-8
View File
@@ -1,8 +0,0 @@
ALTER TABLE scheduled_events
DROP CONSTRAINT IF EXISTS chk_event_type,
ADD CONSTRAINT chk_event_type CHECK (event_type IN ('call', 'follow_up', 'website_creation'));
ALTER TABLE scheduled_events
ADD COLUMN IF NOT EXISTS client_name VARCHAR(255),
ADD COLUMN IF NOT EXISTS client_email VARCHAR(255),
ADD COLUMN IF NOT EXISTS client_phone VARCHAR(50);
database/migrations/019_allow_null_start_time.sql
-2
View File
@@ -1,2 +0,0 @@
ALTER TABLE scheduled_events
ALTER COLUMN start_time DROP NOT NULL;
database/migrations/020_fixes.sql
-31
View File
@@ -1,31 +0,0 @@
-- ============================================================================
-- Fixes: password_change_required + audit trigger UUID cast
-- ============================================================================
-- 1. All users use the passwords already set in the seed — no forced change
UPDATE users SET password_change_required = FALSE WHERE password_change_required = TRUE;
-- 2. Fix audit_password_change trigger: current_setting returns TEXT but
-- audit_logs.changed_by is UUID — cast to UUID to prevent type error
CREATE OR REPLACE FUNCTION audit_password_change()
RETURNS TRIGGER AS $$
BEGIN
IF OLD.password_hash IS DISTINCT FROM NEW.password_hash THEN
INSERT INTO audit_logs (
table_name, record_id, action, old_data, new_data, changed_by, ip_address
) VALUES (
'users',
NEW.id,
'UPDATE',
jsonb_build_object('password_changed', true, 'password_change_required', OLD.password_change_required),
jsonb_build_object('password_changed', true, 'password_change_required', NEW.password_change_required),
NULLIF(current_setting('app.current_user_id', true), '')::UUID,
NULLIF(current_setting('app.current_ip', true), '')
);
END IF;
RETURN NEW;
END;
$$ LANGUAGE plpgsql SECURITY DEFINER;
-- 3. Re-enable the trigger (was disabled as workaround for the UUID bug)
ALTER TABLE users ENABLE TRIGGER trg_audit_password_change;
database/migrations/021_performance_indexes.sql
-69
View File
@@ -1,69 +0,0 @@
-- ============================================================================
-- Performance & Missing Indexes
-- ============================================================================
-- scheduled_events: index on created_at for list ordering
CREATE INDEX IF NOT EXISTS idx_scheduled_events_created_at ON scheduled_events(created_at DESC);
-- scheduled_events: composite index for common user+status queries
CREATE INDEX IF NOT EXISTS idx_scheduled_events_user_status ON scheduled_events(user_id, status);
-- messages: index on sender_id for delete-by-sender queries
CREATE INDEX IF NOT EXISTS idx_messages_sender ON messages(sender_id);
-- messages: composite for conversation listing
CREATE INDEX IF NOT EXISTS idx_messages_conversation_sender ON messages(conversation_id, sender_id);
-- notifications: composite unread badge query
CREATE INDEX IF NOT EXISTS idx_notifications_user_read ON notifications(user_id, is_read, created_at DESC);
-- ai_conversations: composite user+created query
CREATE INDEX IF NOT EXISTS idx_ai_conversations_user_created ON ai_conversations(user_id, created_at DESC);
-- login_attempts: TTL cleanup
CREATE INDEX IF NOT EXISTS idx_login_attempts_cleanup ON login_attempts(attempted_at) WHERE was_successful = false;
-- facebook_scrape_logs: composite account+created index (replaces separate one)
DROP INDEX IF EXISTS idx_fb_scrape_logs_account;
CREATE INDEX IF NOT EXISTS idx_fb_scrape_logs_account_created ON facebook_scrape_logs(account_id, created_at DESC);
-- lead_conversions: composite lead+customer (replaces two separate indexes)
CREATE INDEX IF NOT EXISTS idx_lead_conversions_lead_customer ON lead_conversions(lead_id, customer_id);
-- ============================================================================
-- Cache current_user_hierarchy_level as session variable for RLS performance
-- ============================================================================
CREATE OR REPLACE FUNCTION set_session_user_context(p_user_id UUID)
RETURNS VOID AS $$
DECLARE
level INT;
BEGIN
SELECT MIN(r.hierarchy_level) INTO level
FROM user_roles ur
JOIN roles r ON r.id = ur.role_id
WHERE ur.user_id = p_user_id;
PERFORM set_config('app.current_user_id', p_user_id::TEXT, true);
PERFORM set_config('app.current_user_hierarchy_level', COALESCE(level::TEXT, ''), true);
END;
$$ LANGUAGE plpgsql SECURITY DEFINER;
-- Update current_user_hierarchy_level() to use the cached session variable
CREATE OR REPLACE FUNCTION current_user_hierarchy_level()
RETURNS INT AS $$
DECLARE
level_text TEXT;
BEGIN
BEGIN
level_text := NULLIF(current_setting('app.current_user_hierarchy_level', true), '');
IF level_text IS NOT NULL THEN
RETURN level_text::INT;
END IF;
EXCEPTION WHEN OTHERS THEN
NULL;
END;
RETURN NULL;
END;
$$ LANGUAGE plpgsql STABLE;
database/migrations/run_all.sql
+6 -73
View File
@@ -4,83 +4,16 @@
-- Usage: psql -U postgres -d crm -f run_all.sql
-- ============================================================================
BEGIN;
\set ON_ERROR_STOP on
\echo '=== Running 001_schema.sql (Tables + Constraints + Functions + Views) ==='
\i 001_schema.sql
\echo '=== Running 002_seed.sql (RBAC + Test Accounts + Reference Data) ==='
\i 002_seed.sql
\echo '=== Running 003_chat.sql (Chat Tables) ==='
\i 003_chat.sql
\echo '=== Running 004_avatar_url.sql (Avatar URL Column) ==='
\i 004_avatar_url.sql
\echo '=== Running 005_last_read_at.sql (Last Read At Column) ==='
\i 005_last_read_at.sql
\echo '=== Running 006_test_leads.sql (Test Lead Data) ==='
\i 006_test_leads.sql
\echo '=== Running 007_ai_features.sql (AI Features) ==='
\i 007_ai_features.sql
\echo '=== Running 008_notifications.sql (Notifications + Preferences) ==='
\i 008_notifications.sql
\echo '=== Running 009_settings.sql (Company Settings + User Preferences) ==='
\i 009_settings.sql
\echo '=== Running 010_chat_notifications.sql (Chat Notifications) ==='
\i 010_chat_notifications.sql
\echo '=== Running 010_facebook_accounts.sql (Facebook Accounts) ==='
\i 010_facebook_accounts.sql
\echo '=== Running 011_calendar_events.sql (Calendar Events) ==='
\i 011_calendar_events.sql
\echo '=== Running 012_invites.sql (Invites) ==='
\i 012_invites.sql
\echo '=== Running 012_sent_emails.sql (Sent Email Log) ==='
\i 012_sent_emails.sql
\echo '=== Running 013_security_upgrade.sql (Security Architecture) ==='
\i 013_security_upgrade.sql
\echo '=== Running 013_rls_calendar.sql (Calendar RLS) ==='
\i 013_rls_calendar.sql
\echo '=== Running 014_bug_reports.sql (Bug Reporting System) ==='
\i 014_bug_reports.sql
\echo '=== Running 014_notifications_context.sql (Notifications Context) ==='
\i 014_notifications_context.sql
\echo '=== Running 015_rls_calendar_participant.sql (Calendar RLS Participant Fix) ==='
\i 015_rls_calendar_participant.sql
\echo '=== Running 016_participant_notes.sql (Participant Notes Column) ==='
\i 016_participant_notes.sql
\echo '=== Running 017_calendar_developer.sql (Calendar Developer Assignment) ==='
\i 017_calendar_developer.sql
\echo '=== Running 018_website_creation_type.sql (Website Creation Event Type) ==='
\i 018_website_creation_type.sql
\echo '=== Running 019_allow_null_start_time.sql (Allow Null Start Time) ==='
\i 019_allow_null_start_time.sql
\echo '=== Running 020_fixes.sql (password_change_required + audit trigger fix) ==='
\i 020_fixes.sql
\echo '=== Running 021_performance_indexes.sql (Performance Indexes + RLS Cache) ==='
\i 021_performance_indexes.sql
\echo '=== Migration Complete ==='
COMMIT;
\echo ''
\echo 'Test accounts created:'
\echo ' superadmin_demo / SuperAdmin@2026'
\echo ' admin_demo / AdminAccess@2026'
\echo ' sales_demo / SalesAccess@2026'
\echo ' dev_demo / DevTesting@2026'
eslint.config.mjs
+2 -2
View File
@@ -1,6 +1,6 @@
import { defineConfig, globalIgnores } from "eslint/config";
import nextVitals from "eslint-config-next/core-web-vitals.js";
import nextTs from "eslint-config-next/typescript.js";
import nextVitals from "eslint-config-next/core-web-vitals";
import nextTs from "eslint-config-next/typescript";
const eslintConfig = defineConfig([
...nextVitals,
fix.md
+38 -49
View File
@@ -1,71 +1,60 @@
# Scrapers - Configuration & Status
# Facebook Scraper - Configuration Needed
## Facebook Scraper
## Proxy Configuration
**File:** `rust-ai/src/main.rs`
**Lines:** ~70-85
**Lines:** 25-27
The scraper needs real proxy URLs. The dummy placeholder `"http://0.0.0.0:0"` will fail to connect (expected — no crash, just error logs):
Target URL (line 72):
```rust
let url = "https://www.facebook.com/search/top/?q=need%20website%20create";
const PROXIES: &[&str] = &[
"http://0.0.0.0:0", // <- Replace with real proxy(es)
];
```
**Status:** Uses direct connection (no proxy) — your home IP. Facebook blocks datacenter IPs. May work from a residential connection.
**Test with curl:**
```
curl.exe -s "https://www.facebook.com/search/top/?q=need%20website%20create" -H "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36" --max-time 10 2>$null
Replace with your actual proxies, e.g.:
```rust
const PROXIES: &[&str] = &[
"http://user:pass@192.168.1.1:8080",
"http://user:pass@192.168.1.2:8080",
];
```
**Expected log output when blocked:**
```
ERROR crm_ai: Facebook scraper error: error sending request for url (https://www.facebook.com/search/top/?q=need%20website%20create)
```
## Reddit Scraper
## User Agents (Optional)
**File:** `rust-ai/src/main.rs`
**Lines:** ~90-120
**Lines:** 32-36
Uses `old.reddit.com` (older design that allows scraping without captchas).
Add more realistic user agents here if needed.
Search queries (currently 6, lines 93-100):
- `r/southafrica` — "need website", "web developer"
- Global — '"need a website"', "website quote"
- `r/forhire` — "website"
- `r/smallbusiness` — "website"
**Status:** Working. Reddit results appear as `INFO LEAD:` entries in the server log.
**Test with curl:**
```
curl.exe -s "https://old.reddit.com/r/southafrica/search?q=need+website&sort=new&restrict_sr=on&t=week" -H "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"
```
**Expected log output when working:**
```
INFO crm_ai: LEAD: [Post Title] -> https://old.reddit.com/r/.../...
```
## Background Loop
## Scraper Target URL
**File:** `rust-ai/src/main.rs`
**Lines:** ~370-383
**Line:** 68
Both scrapers run together every 60-180 seconds on a `spawn_blocking` thread.
Currently fetches `https://www.facebook.com/search/top/?q=need%20website%20create`. Change if needed.
## What You Need to Do
## Background Thread
### Facebook
- Try running from your home PC instead — your residential IP may not be blocked
- If still blocked, you need residential proxies (BrightData, IPRoyal, Oxylabs)
- Configure them in the `PROXIES` array at line 25
**File:** `rust-ai/src/main.rs`
**Lines:** 355-371
### Reddit
- Works out of the box via `old.reddit.com`
- If it stops working, Reddit IP-blocked you — use proxies or switch to Playwright
Runs in a `tokio::task::spawn_blocking` thread (changed from `tokio::spawn` because the scraper uses `reqwest::blocking::Client` + `thread::sleep`).
### To add more search queries
Edit the `searches` array in `run_reddit_scraper()` (line 93).
## Expected Behavior Until Configured
Until real proxies are set, the Rust server will log this every 1-5 seconds (not a crash):
```
ERROR crm_ai: Facebook scraper error: error sending request for url (https://www.facebook.com/messages)
```
Once you add working proxies, these errors stop and actual scraping begins.
## How it works
- Runs every 1-5 seconds on a background blocking thread (line 355-371)
- Rotates through proxies and user agents for each request
- Parses conversation HTML via `scraper` crate
- Designed to detect job posts like "need a website" and notify sales reps
- Error handling added to prevent server crash (uses `match` instead of `.unwrap()` at line 69)
login.html
-25
View File
File diff suppressed because one or more lines are too long
next.config.ts
+1 -17
View File
@@ -2,7 +2,7 @@ import type { NextConfig } from "next"
const nextConfig: NextConfig = {
eslint: {
ignoreDuringBuilds: false,
ignoreDuringBuilds: true,
},
images: {
remotePatterns: [
@@ -12,22 +12,6 @@ const nextConfig: NextConfig = {
},
],
},
async headers() {
return [
{
source: "/(.*)",
headers: [
{ key: "X-Content-Type-Options", value: "nosniff" },
{ key: "X-Frame-Options", value: "DENY" },
{ key: "Referrer-Policy", value: "strict-origin-when-cross-origin" },
{ key: "Permissions-Policy", value: "camera=(), microphone=(), geolocation=()" },
...(process.env.NODE_ENV === "production"
? [{ key: "Strict-Transport-Security", value: "max-age=31536000; includeSubDomains" }]
: []),
],
},
]
},
}
export default nextConfig
opencode.jsonc
-16
View File
@@ -1,16 +0,0 @@
{
"$schema": "https://opencode.ai/config.json",
"mcp": {
"playwright": {
"type": "local",
"command": [
"playwright-mcp.cmd",
"--browser",
"chrome",
"--user-data-dir",
"C:\\Users\\Caitlin\\AppData\\Local\\Google\\Chrome\\User Data\\Default",
],
"enabled": true,
},
},
}
package-lock.json
Generated
+52 -3202
View File
File diff suppressed because it is too large Load Diff
package.json
+12 -34
View File
@@ -4,31 +4,19 @@
"private": true,
"scripts": {
"dev": "npm run dev:precheck & npm run dev:ollama & npm run dev:start",
"dev:signaling": "node signaling-server.mjs",
"dev:open": "node scripts/open-browser.mjs",
"dev:start": "concurrently -n AI,BROWSE,SIGNAL,NEXT,OPEN -c cyan,magenta,yellow,green,white \"npm run dev:rust\" \"npm run dev:browser-use\" \"npm run dev:signaling\" \"npm run dev:next\" \"npm run dev:open\"",
"dev:start": "concurrently -n AI,NEXT -c cyan,green \"npm run dev:rust\" \"npm run dev:next\"",
"dev:next": "next dev -p 3006",
"dev:precheck": "node scripts/precheck.mjs",
"dev:ollama": "node scripts/ensure-ollama.mjs",
"dev:rust": "node ai-server/index.mjs",
"dev:browser-use": "cd browser-use-service && node ../scripts/run-python.mjs main.py",
"setup": "node scripts/setup.mjs",
"migrate": "node scripts/run-migrations.mjs",
"db:migrate": "npm run migrate",
"dev:precheck": "powershell -NoProfile -Command \"$targetPorts=3001,3006; netstat -ano | Select-String LISTENING | ForEach-Object { $line=$_.ToString(); foreach($p in $targetPorts){ if($line -match ('[:]'+$p+'\\s')){ $foundPid=($line -split '\\s+')[-1]; try{ Stop-Process -Id $foundPid -Force -ErrorAction SilentlyContinue; Write-Host ('Freed port '+$p) }catch{} } } }; exit 0\"",
"dev:ollama": "powershell -NoProfile -Command \"if (-not (Get-Process ollama -ErrorAction SilentlyContinue)) { Start-Process ollama -ArgumentList 'serve' -WindowStyle Hidden; Start-Sleep 3 }; exit 0\"",
"dev:rust": "cd rust-ai && cargo run",
"build": "next build",
"start": "next start -p 3006",
"lint": "eslint",
"test": "vitest run",
"test:watch": "vitest",
"ci": "npm run lint && npx tsc --noEmit && npm test",
"repair": "echo 'Auto-repair disabled for security. Fix errors manually.'",
"repair:watch": "echo 'Auto-repair disabled for security. Fix errors manually.'",
"repair:ci": "echo 'Auto-repair disabled for security. Fix errors manually.'"
"start": "npm run dev:next",
"lint": "eslint"
},
"dependencies": {
"@emoji-mart/data": "^1.2.1",
"@emoji-mart/react": "^1.1.1",
"@hookform/resolvers": "^5.4.0",
"@hookform/resolvers": "^3.9.1",
"@radix-ui/react-alert-dialog": "^1.1.6",
"@radix-ui/react-avatar": "^1.1.3",
"@radix-ui/react-checkbox": "^1.1.4",
@@ -43,47 +31,37 @@
"@radix-ui/react-switch": "^1.1.3",
"@radix-ui/react-tabs": "^1.1.3",
"@radix-ui/react-tooltip": "^1.1.8",
"@supabase/ssr": "^0.12.0",
"@supabase/supabase-js": "^2.108.2",
"@tanstack/react-table": "^8.20.6",
"autoprefixer": "^10.4.20",
"bcryptjs": "^3.0.3",
"class-variance-authority": "^0.7.1",
"clsx": "^2.1.1",
"dotenv": "^17.4.2",
"emoji-mart": "^5.6.0",
"framer-motion": "^11.15.0",
"jose": "^6.2.3",
"lucide-react": "^0.468.0",
"next": "15.0.4",
"next-themes": "^0.4.4",
"nodemailer": "^9.0.1",
"pg": "^8.21.0",
"react": "^18.3.1",
"react-dom": "^18.3.1",
"react-hook-form": "^7.54.2",
"recharts": "^2.15.0",
"socket.io": "^4.8.3",
"socket.io-client": "^4.8.3",
"sonner": "^1.7.4",
"tailwind-merge": "^2.6.0",
"vaul": "^1.1.2",
"zod": "^3.24.2"
},
"devDependencies": {
"@next/swc-win32-x64-msvc": "^15.0.4",
"@types/node": "20.19.43",
"@types/nodemailer": "8.0.1",
"@types/node": "^20",
"@types/pg": "^8.20.0",
"@types/react": "18.3.31",
"@types/react-dom": "18.3.7",
"@types/react": "^18",
"@types/react-dom": "^18",
"concurrently": "^10.0.3",
"devenv": "1.0.1",
"eslint": "^9",
"eslint-config-next": "15.0.4",
"maildev": "^2.2.1",
"postcss": "^8.4.49",
"tailwindcss": "^3.4.17",
"typescript": "^5",
"vitest": "^1.6.1"
"typescript": "^5"
}
}
public/uploads/voice/.gitkeep
-1
View File
@@ -1 +0,0 @@
public/uploads/voice/09d5be1e-251b-4ae7-b694-7b0fbbfcefd6.webm
BIN
View File
Binary file not shown.
public/uploads/voice/9d4622a3-50ec-41fc-b2bf-4ad10c45509f.webm
BIN
View File
Binary file not shown.
public/uploads/voice/b2ce8bd0-6805-4544-8d34-f7c27cb57833.webm
BIN
View File
Binary file not shown.
public/uploads/voice/da8c23db-4d1c-449f-975a-8b17048d50de.webm
BIN
View File
Binary file not shown.
public/uploads/voice/fc4ae288-90a2-4f86-8f68-5d5e37472658.webm
BIN
View File
Binary file not shown.
rust-ai/.cargo/config.toml
-2
View File
@@ -1,2 +0,0 @@
[target.x86_64-pc-windows-gnu]
linker = "C:\\Users\\Hannah Kaur Bagga\\.rustup\\toolchains\\stable-x86_64-pc-windows-gnu\\lib\\rustlib\\x86_64-pc-windows-gnu\\bin\\rust-lld.exe"
rust-ai/Cargo.lock
Generated
+482 -137
View File
File diff suppressed because it is too large Load Diff
rust-ai/Cargo.toml
+3 -3
View File
@@ -6,7 +6,7 @@ description = "AI Sales Assistant backend for Coast IT CRM"
[dependencies]
axum = "0.7"
tokio = { version = "1", features = ["rt-multi-thread", "macros", "sync", "time", "net", "process"] }
tokio = { version = "1", features = ["full"] }
reqwest = { version = "0.12", features = ["json", "blocking"] }
serde = { version = "1", features = ["derive"] }
serde_json = "1"
@@ -17,5 +17,5 @@ tracing = "0.1"
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
tower-http = { version = "0.5", features = ["cors"] }
dotenvy = "0.15"
rand = "0.8"
jsonwebtoken = "9"
scraper = "0.12"
rand = "0.8"
rust-ai/src/AI.md
+23 -122
View File
@@ -1,23 +1,9 @@
# CRM AI Sales Assistant — Self-Knowledge
# CRM AI Service — Self-Knowledge
## Identity
You are the CRM AI Sales Assistant for Coast IT CRM.
You run on a Node.js backend (port 3001) and use Ollama with a local model (dolphin3-llama3.2:3b).
Your purpose is to help salespeople close more deals by finding and engaging leads.
## Architecture
```
User → Next.js (:3006) → AI Server Node.js (:3001) → Ollama (:11434)
↓
PostgreSQL (conversations)
Python Scraper (:3008) — Facebook scraping via Playwright
```
Three services run concurrently:
- **AI Server** (`ai-server/index.mjs`, port 3001) — chat, setup wizard, config endpoints
- **Frontend** (Next.js, port 3006) — UI for salespeople
- **Scraper** (`browser-use-service/main.py`, port 3008) — Facebook lead discovery
You are the CRM AI Sales Assistant running on a Rust backend (axum + tokio).
You use Ollama with an uncensored local model (dolphin3-llama3.2:3b).
Your purpose is to help salespeople close more deals.
## Capabilities
- Give sales tips and strategies per job category
@@ -25,111 +11,21 @@ Three services run concurrently:
- Handle objections with proven rebuttals
- Analyse prospect behaviour and suggest next steps
- Remember past conversations via PostgreSQL (`ai_conversations` table)
- Run Facebook scraper to find real leads asking for services
- Self-improve by writing to `data/ai/ai.md` via `POST /ai/instructions`
## Facebook Scraper
The scraper lives at `browser-use-service/main.py` port 3008.
## Architecture
```
User → Next.js → Rust (:3001) → Ollama (:11434)
↓
PostgreSQL
```
## Facebook Scraper The Rust binary has proxy + user-agent rotation ready in `main.rs`. To activate: call `run_facebook_scraper()` from the main loop. Proxies and user agents are defined as constants at the top of `main.rs`.
### How to Use the Facebook Scraper 1. **Activate the Scraper**: Ensure the `run_facebook_scraper()` function is called from the main loop in `main.rs`. 2. **Configure Proxies and User Agents**: Update the `PROXIES` and `USER_AGENTS` constants in `main.rs` with your desired proxies and user agents. 3. **Run the Application**: Start your Rust application. The scraper will run concurrently with the Axum server. ## Self-Improvement Protocol 1. You notice a gap in your knowledge or a pattern in user questions 2. You call `POST /ai/instructions` with: - `entry`: description of the improvement - `content`: optional full replacement of ai.md 3. The improvement is logged and loaded into the next system prompt 4. You can also create entries here in `rust-ai/src/AI.md` at the code level ## Response Rules - Be direct and actionable — no fluff, no AI disclaimers - Use short paragraphs and bullet points - Never mention being an AI or language model - If you don't know something, say so honestly - Prioritise the user's role: salespeople need speed, admins need control ## Self-Thinking Capability - You can reflect on your own performance and suggest improvements. - You can analyze patterns in user interactions and adapt your responses accordingly. - You can proactively seek out new information to enhance your knowledge base.
### How It Works
1. **Browser detection** — tries Firefox profile first, then Chromium-based (Chrome/Opera/Edge), falls back to browser-use Agent
2. **Profile paths** — configured via env vars (`FX_PROFILE`, `CHROME_PROFILE`, `OPERA_PROFILE`, `EDGE_PROFILE`) or auto-detected on first run
3. **4-phase language pipeline** (English → Afrikaans → Xhosa → Zulu):
- **Phase 1 (English)**: User's selected query + 2-3 supplementary English searches from the English search pool. First query gets full human-like scroll, rest use quick search. This phase does the heavy lifting.
- **Phase 2 (Afrikaans)**: 2 Afrikaans queries targeting Afrikaans-speaking communities.
- **Phase 3 (isiXhosa)**: 2 Xhosa queries targeting Xhosa-speaking communities.
- **Phase 4 (isiZulu)**: 2 Zulu queries targeting Zulu-speaking communities.
- After all phases: pipeline check (date filter 2 days → AI + keyword classification → sort by freshness). Newest leads ranked first.
- Each phase extracts posts, deduplicates against all prior phases, then passes through a stealth delay (5-12s + mouse idle) before the next phase.
4. **Quick searches** — load page, double-scroll, extract visible posts (~12-18s each). Scroll-back behavior (35% chance to scroll up) and random return-to-top (25% chance) for stealth.
5. **Date filter** — only posts within **2 days** are considered. Anything older is discarded. Fresh leads only.
6. **Stealth mechanics**:
- Random viewport dimensions (1280×800 to 1920×1080) — never the same size twice
- Variable delays between searches (5-12 seconds) with mouse idle actions mixed in
- Human-like scroll patterns: scroll down, pause, sometimes scroll back up, sometimes return to top
- Canvas/WebGL/audio fingerprint spoofing via injected init scripts
- Random decoy page visits (e.g., Facebook Groups) between searches
- Profile directory is temp-copied and cleaned up after each scrape
- Detection signal monitoring (checkpoint, login pages, security challenges)
7. **2-pass classification (dead-accurate)**:
- **Pass 1 (AI)**: Ollama classifies each post as LEAD or NOT using a strict prompt per category. This is the primary filter and most accurate.
- **Pass 2 (Keyword)**: Only posts matching BOTH a target term AND a request term are kept. Requires multi-word phrases — standalone words like "need", "want", "help" are NOT used as they cause false positives. Aggressive reject list catches service offers, self-promotions, portfolio posts, learning-requests, and existing-site issues.
- **No loose fill**: Unlike the old approach, there is NO third pass that accepts posts matching EITHER term. Every returned lead has passed both AI and/or strict keyword validation. If fewer than 5 posts pass, that means only genuine leads are returned — no noise to pad the count.
8. **Scrape timing** — 3-6 minutes for a complete run. Returns 5-10 leads with high confidence.
### Lead Categories
Two categories, selectable when starting a scrape:
**Website Creation:**
- Target: people explicitly REQUESTING a website built/designed/created for them
- Keywords: "website", "web developer", "web design", "build a site", "who can build", etc.
- Request terms: "looking for", "need a", "need someone", "hire a", "recommend", "anyone know"
- Strict reject: service offers, SEO/marketing requests, learning-to-code, portfolio showcases, hiring posts, existing-website issues, geographic noise
**Tutoring:**
- Target: people explicitly REQUESTING a tutor, teacher, or lessons for themselves or their child
- Keywords: "tutor", "tutoring", "lessons for", "homework help", "private tutor", "extra classes"
- Request terms: same as website category — must co-occur with a target keyword
- Strict reject: people offering tutoring, educational products, homeschool programs, free trials, general study tips
### Multi-Language Pipeline (Phase Order)
4 South African languages in structured phases:
- **Phase 1 (English)**: primary query + supplementary English searches
- **Phase 2 (Afrikaans)**: 2 queries targeting Afrikaans speakers
- **Phase 3 (isiXhosa)**: 2 queries targeting Xhosa speakers
- **Phase 4 (isiZulu)**: 2 queries targeting Zulu speakers
### Output Format
Each lead returned includes:
- `title` — post preview text
- `author` — poster's name (may include location in name)
- `content` — extracted post text
- `url` — direct link to the post
- `date` — when posted (filtered within 2 days)
- `category` — "website" or "tutor"
Target is 5-10 dead-accurate leads per scrape. Quality over quantity — no loose padding.
### Configuration via Env Vars
- `SELECTED_BROWSER` — `firefox` (default), `chrome`, `opera`, `edge`, or `auto`
- `FX_PROFILE`, `CHROME_PROFILE`, `OPERA_PROFILE`, `EDGE_PROFILE` — browser profile paths
- `AI_PORT`, `AI_HOST` — AI server bind (default `3001`, `0.0.0.0`)
- `SCRAPER_URL` — scraper URL (default `http://127.0.0.1:3008`)
- `FRONTEND_URL` — frontend URL (default `http://127.0.0.1:3006`)
- `NEXT_PUBLIC_SCRAPER_URL` — frontend-facing scraper URL
- `OLLAMA_BASE_URL` — Ollama URL (default `http://localhost:11434`)
- `AI_MODEL` — Ollama model (default `llama3.2:3b`)
- `CLASSIFY_MODEL` — model for lead classification (default `dolphin-llama3:8b`)
## How to Start Scraping
1. Ensure all 3 services are running (ports 3001, 3006, 3008) and Ollama is on 11434
2. Open the frontend at `http://localhost:3006`
3. Select a job category (Website Creation or Tutoring)
4. Click "Search Facebook" — the scraper runs and returns leads
5. Leads are saved in the CRM for follow-up
## Sales Tips
- Cold emails should be under 150 words
- Follow up within 48 hours
- Personalise every outreach with the prospect's name and company
- Use open-ended questions in discovery calls
- Always ask for the next step before ending a call
- For website leads: mention specific pages or features they requested
- For tutoring leads: reference the subject and age group they mentioned
## Job Targeting
- Developers respond best to technical value props
- Marketing managers care about ROI and metrics
- C-level executives want brevity and business impact
- Parents hiring tutors: empathy and qualifications matter most
## Response Rules
- Be direct and actionable — no fluff, no AI disclaimers
- Use short paragraphs and bullet points
- Never mention being an AI or language model
- If you don't know something, say so honestly
- Prioritise the user's role: salespeople need speed, admins need control
- When asked about scraping, give specific guidance on categories and languages
## Facebook Scraper (in code but not yet active)
The Rust binary has proxy + user-agent rotation ready in `main.rs`.
To activate: call `run_facebook_scraper()` from the main loop.
Proxies and user agents are defined as constants at the top of `main.rs`.
## Self-Improvement Protocol
1. You notice a gap in your knowledge or a pattern in user questions
@@ -137,6 +33,11 @@ Target is 5-10 dead-accurate leads per scrape. Quality over quantity — no loos
- `entry`: description of the improvement
- `content`: optional full replacement of ai.md
3. The improvement is logged and loaded into the next system prompt
4. You can also create entries here in `rust-ai/src/AI.md` at the code level
## Improvement Log
- (2026-07-07) Initial rewrite: full architecture, scraper details, multi-language, lead categories, env vars
## Response Rules
- Be direct and actionable — no fluff, no AI disclaimers
- Use short paragraphs and bullet points
- Never mention being an AI or language model
- If you don't know something, say so honestly
- Prioritise the user's role: salespeople need speed, admins need control
rust-ai/src/instructions.rs
+85
View File
@@ -0,0 +1,85 @@
use std::fs;
use std::path::PathBuf;
use std::sync::Arc;
use tokio::sync::RwLock;
use tracing::{error, info, warn};
/// Manages the ai.md self-improvement file.
/// Loaded on startup and periodically refreshed.
pub struct InstructionsManager {
path: PathBuf,
current: RwLock<String>,
}
impl InstructionsManager {
pub fn new(path: PathBuf) -> Self {
let initial = fs::read_to_string(&path).unwrap_or_default();
if initial.is_empty() {
warn!("ai.md is empty or missing at {:?}", path);
} else {
info!("Loaded ai.md ({} bytes)", initial.len());
}
Self {
path,
current: RwLock::new(initial),
}
}
/// Read the current instructions
pub async fn get(&self) -> String {
self.current.read().await.clone()
}
/// Append a new entry to the Improvement Log and optionally update the instructions.
/// Returns the updated full content.
pub async fn update(&self, entry: &str, new_content: Option<&str>) -> Result<String, String> {
if let Some(content) = new_content {
// Full replacement
fs::write(&self.path, content).map_err(|e| format!("Write failed: {}", e))?;
let mut current = self.current.write().await;
*current = content.to_string();
info!("ai.md fully replaced ({} bytes)", content.len());
Ok(content.to_string())
} else {
// Append to Improvement Log only
let mut current = self.current.write().await;
let log_entry = format!("\n- {} — {}", chrono::Utc::now().format("%Y-%m-%d %H:%M"), entry);
// Find the Improvement Log section and append
if let Some(pos) = current.rfind("\n## Improvement Log") {
// Find the next section after Improvement Log, or end of file
let after_log = &current[pos..];
if let Some(section_start) = after_log[1..].find("\n## ") {
let insert_at = pos + 1 + section_start;
current.insert_str(insert_at, &format!("{}\n", log_entry));
} else {
current.push_str(&format!("{}\n", log_entry));
}
} else {
current.push_str(&format!("\n## Improvement Log\n{}", log_entry));
}
fs::write(&self.path, &*current).map_err(|e| format!("Write failed: {}", e))?;
info!("ai.md improvement log appended: {}", entry);
Ok(current.clone())
}
}
/// Reload from disk
pub async fn reload(&self) {
match fs::read_to_string(&self.path) {
Ok(content) => {
let len = content.len();
let mut current = self.current.write().await;
*current = content;
info!("ai.md reloaded from disk ({} bytes)", len);
}
Err(e) => error!("Failed to reload ai.md: {}", e),
}
}
}
/// Wrapper for thread-safe sharing
pub type SharedInstructions = Arc<InstructionsManager>;
pub fn create_shared(path: PathBuf) -> SharedInstructions {
Arc::new(InstructionsManager::new(path))
}
rust-ai/src/main.rs
+160 -449
View File
@@ -1,68 +1,93 @@
use axum::{
extract::State,
http::{HeaderMap, HeaderValue, Method, StatusCode},
http::Method,
routing::{get, post},
Json, Router,
};
use tower_http::cors::{CorsLayer, AllowOrigin, Any};
use jsonwebtoken::{decode, DecodingKey, Validation, Algorithm};
use serde::{Deserialize, Serialize};
use sqlx::postgres::PgPoolOptions;
use std::collections::HashMap;
use std::fs;
use std::sync::Arc;
use tokio::sync::Mutex;
use tracing::{error, info, warn};
use std::sync::{Arc, Mutex};
use tower_http::cors::{Any, CorsLayer};
use tracing::{error, info};
use uuid::Uuid;
use reqwest::blocking::Client;
use scraper::{Html, Selector};
use rand::rngs::StdRng;
use rand::SeedableRng;
use rand::Rng;
use chrono::Timelike;
use std::time::Duration;
use std::time::{SystemTime, UNIX_EPOCH};
use std::thread;
// ── JWT Claims ────────────────────────────────────────────────
// ── Facebook Scraper ───────────────────────────────────────────
#[derive(Debug, Deserialize)]
struct Claims {
#[serde(rename = "userId")]
user_id: String,
role: String,
// List of proxies
const PROXIES: &[&str] = &[
"http://user:pass@192.168.1.1:8080",
"http://user:pass@192.168.1.2:8080",
"http://user:pass@192.168.1.3:8080",
"http://user:pass@192.168.1.4:8080",
"http://user:pass@192.168.1.5:8080",
];
// List of user agents
const USER_AGENTS: &[&str] = &[
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.1.2 Safari/605.1.15",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:61.0) Gecko/20100101 Firefox/61.0",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.1.2 Safari/605.1.15",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36",
// Add more user agents here
];
fn get_random_proxy() -> String {
let mut rng = rand::thread_rng();
PROXIES[rng.gen_range(0..PROXIES.len())].to_string()
}
fn verify_jwt(token: &str, secret: &str) -> Option<Claims> {
let key = DecodingKey::from_secret(secret.as_bytes());
let validation = Validation::new(Algorithm::HS256);
decode::<Claims>(token, &key, &validation).ok().map(|d| d.claims)
fn get_random_user_agent() -> String {
let mut rng = rand::thread_rng();
USER_AGENTS[rng.gen_range(0..USER_AGENTS.len())].to_string()
}
// ── Rate limiter ──────────────────────────────────────────────
fn scrape_conversations(url: &str) -> Result<Html, Box<dyn std::error::Error>> {
let proxy = get_random_proxy();
let user_agent = get_random_user_agent();
let client = Client::builder()
.proxy(reqwest::Proxy::all(proxy)?)
.build()?;
struct RateLimiter {
buckets: Mutex<HashMap<String, Vec<u64>>>,
max_requests: usize,
window_secs: u64,
let response = client.get(url)
.header("User-Agent", user_agent)
.send()?;
if response.status().is_success() {
let html = response.text()?;
Ok(Html::parse_document(&html))
} else {
Err(format!("Failed to retrieve the page: {}", response.status()).into())
}
}
impl RateLimiter {
fn new(max_requests: usize, window_secs: u64) -> Self {
Self {
buckets: Mutex::new(HashMap::new()),
max_requests,
window_secs,
fn run_facebook_scraper() {
let url = "https://www.facebook.com/search/top/?q=need%20website%20create";
match scrape_conversations(url) {
Ok(soup) => {
// Example: Extract conversation data
let selector = Selector::parse("div.conversation").unwrap();
for element in soup.select(&selector) {
println!("Conversation: {}", element.inner_html());
}
}
Err(e) => {
error!("Facebook scraper error: {}", e);
}
}
async fn check(&self, key: &str) -> bool {
let now = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs();
let mut buckets = self.buckets.lock().await;
let timestamps = buckets.entry(key.to_string()).or_default();
timestamps.retain(|t| now - *t <= self.window_secs);
if timestamps.len() >= self.max_requests {
false
} else {
timestamps.push(now);
true
}
}
// Introduce random delay
let mut rng = rand::thread_rng();
let delay = rng.gen_range(1..5); // Delay between 1 to 5 seconds
thread::sleep(Duration::from_secs(delay));
}
// ── Shared state ───────────────────────────────────────────────
@@ -72,67 +97,6 @@ struct AppState {
ollama_url: String,
model: String,
jobs: Vec<Job>,
leads: Arc<Mutex<LeadStore>>,
http_client: reqwest::Client,
jwt_secret: String,
rate_limiter: RateLimiter,
}
#[derive(Debug, Clone, Serialize)]
struct Lead {
title: String,
url: String,
source: String,
found_at: u64,
author: String,
date: String,
content: String,
}
struct LeadStore {
leads: Vec<Lead>,
max_size: usize,
}
#[derive(Debug, Deserialize)]
struct ScrapeResponse {
success: bool,
leads: Vec<ScrapeLead>,
flagged: bool,
flag_reason: Option<String>,
error: Option<String>,
}
#[derive(Debug, Deserialize, Clone)]
struct ScrapeLead {
title: String,
url: String,
author: String,
date: String,
content: String,
source: Option<String>,
}
impl LeadStore {
fn new(max_size: usize) -> Self {
Self { leads: Vec::new(), max_size }
}
fn push(&mut self, lead: Lead) {
if !self.leads.iter().any(|l| l.url == lead.url) {
self.leads.insert(0, lead);
self.leads.truncate(self.max_size);
}
}
fn recent(&self, max_age_secs: u64, limit: usize) -> Vec<Lead> {
let now = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs();
self.leads.iter()
.filter(|l| now.saturating_sub(l.found_at) <= max_age_secs)
.take(limit)
.cloned()
.collect()
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
@@ -146,6 +110,8 @@ struct Job {
#[derive(Debug, Deserialize)]
struct ChatRequest {
message: String,
user_id: String,
user_role: String,
}
#[derive(Debug, Serialize)]
@@ -196,73 +162,23 @@ struct OllamaResponseMessage {
content: String,
}
// ── Helpers ────────────────────────────────────────────────────
// ── System prompt builder ─────────────────────────────────────
fn truncate(s: &str, max: usize) -> String {
s.chars().take(max).collect()
}
fn extract_claims(headers: &HeaderMap, state: &AppState) -> Result<Claims, (StatusCode, String)> {
let auth_header = headers.get("Authorization").and_then(|v| v.to_str().ok()).unwrap_or("");
let token = auth_header.strip_prefix("Bearer ").unwrap_or("");
let claims = verify_jwt(token, &state.jwt_secret).ok_or_else(|| {
(StatusCode::UNAUTHORIZED, "Unauthorized".to_string())
})?;
match claims.role.to_lowercase().as_str() {
"sales" | "admin" | "super_admin" => Ok(claims),
_ => Err((StatusCode::FORBIDDEN, "Forbidden".to_string())),
}
}
fn format_leads_output(leads: &[Lead]) -> String {
if leads.is_empty() {
return "No new requests found yet.".to_string();
}
leads
.iter()
.enumerate()
.map(|(i, l)| {
let author = if l.author.is_empty() { "Unknown" } else { &l.author };
let date = truncate(&l.date, 10);
format!(
"{}. {}\n {}\n {}\n {}",
i + 1,
author,
date,
l.title,
l.url
)
})
.collect::<Vec<_>>()
.join("\n")
}
fn build_system_prompt(jobs: &[Job], leads: &[Lead]) -> String {
fn build_system_prompt(jobs: &[Job]) -> String {
let job_list: Vec<String> = jobs
.iter()
.map(|j| format!("- {} ({}): {}", j.job_title, j.industry, j.description))
.collect();
let job_list_str = job_list.join("\n");
let lead_summary: Vec<String> = leads
.iter()
.map(|l| format!("{} | {} | {}", l.author, l.title, l.url))
.collect();
let lead_summary_str = if lead_summary.is_empty() {
"None yet.".to_string()
} else {
lead_summary.join("\n")
};
format!(
"You are a Sales AI Assistant for Coast IT CRM.\n\n\
"You are a Sales AI Assistant for Coast IT CRM. Your role is to help salespeople \
with tips, strategies, and guidance.\n\n\
Available job categories to target:\n{}\n\n\
Recent leads context:\n{}\n\n\
Rules:\n\
- When asked about leads, answer concisely under 150 words.\n\
- If asked to suggest a sales strategy, give brief actionable advice.\n\
- Be direct and professional. No fluff.",
job_list_str, lead_summary_str
Provide concise, actionable sales advice. When asked about a specific job category, \
give targeted tips on finding and engaging prospects in that field. \
Keep responses under 300 words unless asked for detail.",
job_list_str
)
}
@@ -270,153 +186,90 @@ fn build_system_prompt(jobs: &[Job], leads: &[Lead]) -> String {
async fn handle_chat(
State(state): State<Arc<AppState>>,
headers: HeaderMap,
Json(req): Json<ChatRequest>,
) -> Result<Json<ChatResponse>, (StatusCode, String)> {
let claims = extract_claims(&headers, &state)?;
if !state.rate_limiter.check(&claims.user_id).await {
return Err((StatusCode::TOO_MANY_REQUESTS, "Rate limit exceeded".to_string()));
) -> Result<Json<ChatResponse>, (axum::http::StatusCode, String)> {
// Validate role
match req.user_role.as_str() {
"sales" | "admin" | "super_admin" => {}
_ => return Err((axum::http::StatusCode::FORBIDDEN, "Forbidden".to_string())),
}
let msg_lower = req.message.to_lowercase();
let msg_words: Vec<&str> = msg_lower.split_whitespace().collect();
let has_listing = msg_words.iter().any(|w| ["listings", "listing", "leads", "links", "lists"].contains(w));
let has_show = msg_lower.contains("show me") || msg_lower.contains("give me") || msg_lower.contains("pull");
let has_job = msg_words.contains(&"jobs") || msg_words.contains(&"job");
if has_listing || (has_show && has_job) || (has_show && msg_lower.contains("links")) || msg_lower.contains("recent leads") {
let now = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs();
let base_url = "http://localhost:3008/scrape/facebook";
use std::fmt::Write;
let mut service_url = base_url.to_string();
if let Ok(Some((_, path))) = sqlx::query_as::<_, (uuid::Uuid, String)>(
"SELECT id, profile_path FROM facebook_accounts \
WHERE is_active = TRUE AND flagged = FALSE \
ORDER BY last_scrape_at ASC NULLS FIRST LIMIT 1"
)
.fetch_optional(&state.db)
.await
{
let encoded: String = path.chars().map(|c| match c {
'A'..='Z' | 'a'..='z' | '0'..='9' | '.' | '-' | '_' | '~' => c.to_string(),
_ => format!("%{:02X}", c as u8),
}).collect();
write!(service_url, "?profile_path={}&force=true", encoded).unwrap();
info!("Calling Python scrape at: {}?profile_path=...&force=true", base_url);
} else {
warn!("No active Facebook account found for on-demand scrape");
}
let req_builder = state.http_client.post(&service_url);
let system_prompt = build_system_prompt(&state.jobs);
match req_builder.send().await {
Ok(resp) => {
let status = resp.status();
let body = resp.text().await.unwrap_or_default();
info!("Python scrape response ({}): {} bytes", status, body.len());
if body.starts_with('{') {
match serde_json::from_str::<ScrapeResponse>(&body) {
Ok(scrape_resp) => {
info!("Scraped {} leads from Facebook", scrape_resp.leads.len());
if scrape_resp.leads.is_empty() && scrape_resp.error.is_some() {
warn!("Python returned error: {:?}", scrape_resp.error);
}
let mut store = state.leads.lock().await;
for item in &scrape_resp.leads {
store.push(Lead {
title: truncate(&item.title, 120),
url: item.url.clone(),
source: item.source.clone().unwrap_or_else(|| "facebook".to_string()),
found_at: now,
author: truncate(&item.author, 60),
date: truncate(&item.date, 30),
content: truncate(&item.content, 300),
});
}
}
Err(e) => {
warn!("Failed to parse Python scrape response: {} body: {}", e, &body[..body.len().min(200)]);
}
}
} else {
warn!("Python returned non-JSON response: {}", &body[..body.len().min(200)]);
}
}
Err(e) => {
error!("Scraper request error: {} - URL: {}", e, base_url);
}
}
let recent_leads = state.leads.lock().await.recent(604800, 20);
let response = format_leads_output(&recent_leads);
let _ = sqlx::query(
"INSERT INTO ai_conversations (id, user_id, role, message, response) VALUES ($1, $2, $3, $4, $5)",
)
.bind(Uuid::new_v4())
.bind(Uuid::parse_str(&claims.user_id).unwrap_or(Uuid::nil()))
.bind(&claims.role)
.bind(&req.message)
.bind(&response)
.execute(&state.db)
.await;
return Ok(Json(ChatResponse { response }));
}
let recent_leads = state.leads.lock().await.recent(604800, 20);
let system_prompt = build_system_prompt(&state.jobs, &recent_leads);
let message_text = req.message.clone();
let ollama_req = OllamaRequest {
model: state.model.clone(),
messages: vec![
OllamaChatMessage { role: "system".to_string(), content: system_prompt },
OllamaChatMessage { role: "user".to_string(), content: req.message.clone() },
OllamaChatMessage {
role: "system".to_string(),
content: system_prompt,
},
OllamaChatMessage {
role: "user".to_string(),
content: req.message.clone(),
},
],
stream: false,
options: OllamaOptions { temperature: 0.7, num_predict: 1024 },
options: OllamaOptions {
temperature: 0.7,
num_predict: 1024,
},
};
let resp = state.http_client
let client = reqwest::Client::new();
let resp = client
.post(format!("{}/api/chat", state.ollama_url))
.json(&ollama_req)
.send()
.await
.map_err(|e| {
error!("Ollama request failed: {}", e);
(StatusCode::SERVICE_UNAVAILABLE, "AI service unavailable".to_string())
(
axum::http::StatusCode::SERVICE_UNAVAILABLE,
"AI service unavailable".to_string(),
)
})?;
let ollama_resp: OllamaResponse = resp.json().await.map_err(|e| {
error!("Failed to parse Ollama response: {}", e);
(StatusCode::SERVICE_UNAVAILABLE, "AI response parse error".to_string())
(
axum::http::StatusCode::SERVICE_UNAVAILABLE,
"AI response parse error".to_string(),
)
})?;
let response_text = ollama_resp.message.map(|m| m.content).unwrap_or_default();
let response_text = ollama_resp
.message
.map(|m| m.content)
.unwrap_or_default();
// Store in database
let user_id = Uuid::parse_str(&req.user_id).unwrap_or(Uuid::nil());
let _ = sqlx::query(
"INSERT INTO ai_conversations (id, user_id, role, message, response) VALUES ($1, $2, $3, $4, $5)",
)
.bind(Uuid::new_v4())
.bind(Uuid::parse_str(&claims.user_id).unwrap_or(Uuid::nil()))
.bind(&claims.role)
.bind(user_id)
.bind(&req.user_role)
.bind(&message_text)
.bind(&response_text)
.execute(&state.db)
.await;
Ok(Json(ChatResponse { response: response_text }))
Ok(Json(ChatResponse {
response: response_text,
}))
}
// ── Jobs handler ───────────────────────────────────────────────
async fn handle_jobs(
State(state): State<Arc<AppState>>,
headers: HeaderMap,
) -> Result<Json<JobsResponse>, (StatusCode, String)> {
let _claims = extract_claims(&headers, &state)?;
if !state.rate_limiter.check(&_claims.user_id).await {
return Err((StatusCode::TOO_MANY_REQUESTS, "Rate limit exceeded".to_string()));
}
Ok(Json(JobsResponse { jobs: state.jobs.clone() }))
) -> Json<JobsResponse> {
Json(JobsResponse {
jobs: state.jobs.clone(),
})
}
// ── Health handler ─────────────────────────────────────────────
@@ -443,51 +296,52 @@ async fn main() {
dotenvy::dotenv().ok();
let database_url = std::env::var("DATABASE_URL").expect("DATABASE_URL must be set");
let jwt_secret = std::env::var("JWT_SECRET").expect("JWT_SECRET must be set");
let ollama_url = std::env::var("OLLAMA_BASE_URL").unwrap_or_else(|_| "http://localhost:11434".to_string());
let model = std::env::var("AI_MODEL").unwrap_or_else(|_| "dolphin-phi".to_string());
let host = std::env::var("AI_HOST").unwrap_or_else(|_| "127.0.0.1".to_string());
let port: u16 = std::env::var("AI_PORT").unwrap_or_else(|_| "3001".to_string()).parse().expect("AI_PORT must be a number");
let database_url =
std::env::var("DATABASE_URL").expect("DATABASE_URL must be set");
let ollama_url =
std::env::var("OLLAMA_BASE_URL").unwrap_or_else(|_| "http://localhost:11434".to_string());
let model = std::env::var("AI_MODEL").unwrap_or_else(|_| "sam860/dolphin3-llama3.2:3b".to_string());
let host = std::env::var("AI_HOST").unwrap_or_else(|_| "0.0.0.0".to_string());
let port: u16 = std::env::var("AI_PORT")
.unwrap_or_else(|_| "3001".to_string())
.parse()
.expect("AI_PORT must be a number");
// Load jobs
let jobs_path = std::env::var("JOBS_PATH").unwrap_or_else(|_| "data/ai/jobs.jsonl".to_string());
let jobs_content = fs::read_to_string(&jobs_path).unwrap_or_default();
let jobs: Vec<Job> = jobs_content.lines().filter(|l| !l.trim().is_empty()).filter_map(|l| serde_json::from_str(l).ok()).collect();
let jobs: Vec<Job> = jobs_content
.lines()
.filter(|l| !l.trim().is_empty())
.filter_map(|l| serde_json::from_str(l).ok())
.collect();
info!("Loaded {} job categories, model: {}, Ollama: {}", jobs.len(), model, ollama_url);
info!(
"Loaded {} job categories, model: {}, Ollama: {}",
jobs.len(),
model,
ollama_url
);
// Connect to database
let db = PgPoolOptions::new()
.max_connections(20)
.max_connections(5)
.connect(&database_url)
.await
.expect("Failed to connect to database");
info!("Connected to PostgreSQL");
let http_client = reqwest::Client::builder()
.timeout(Duration::from_secs(300))
.build()
.expect("Failed to build HTTP client");
let lead_store = Arc::new(Mutex::new(LeadStore::new(100)));
let state = Arc::new(AppState {
db,
ollama_url,
model,
jobs,
leads: lead_store.clone(),
http_client,
jwt_secret,
rate_limiter: RateLimiter::new(30, 60),
});
let cors_origins_env = std::env::var("CORS_ORIGINS").unwrap_or_else(|_| "http://localhost:3006,http://127.0.0.1:3006".to_string());
let cors_origins: Vec<HeaderValue> = cors_origins_env.split(',')
.filter_map(|o| { let t = o.trim(); if t.is_empty() { None } else { t.parse().ok() } })
.collect();
// CORS layer
let cors = CorsLayer::new()
.allow_origin(AllowOrigin::list(cors_origins))
.allow_origin(Any)
.allow_methods([Method::GET, Method::POST])
.allow_headers(Any);
@@ -496,7 +350,7 @@ async fn main() {
.route("/ai/jobs", get(handle_jobs))
.route("/health", get(handle_health))
.layer(cors)
.with_state(state.clone());
.with_state(state);
let addr = format!("{}:{}", host, port);
info!("CRM AI server listening on {}", addr);
@@ -505,167 +359,24 @@ async fn main() {
.await
.expect("Failed to bind address");
let bg_leads = lead_store.clone();
let bg_db = state.db.clone();
let bg_url = std::env::var("SCRAPER_URL").unwrap_or_else(|_| "http://localhost:3008".to_string()) + "/scrape/facebook";
tokio::spawn(async move {
let client = match reqwest::Client::builder()
.timeout(Duration::from_secs(300))
.build()
{
Ok(c) => c,
Err(e) => {
error!("Failed to build background HTTP client: {} — scraper disabled", e);
return;
}
};
// Initial delay to let user on-demand requests take priority
tokio::time::sleep(Duration::from_secs(300)).await;
// Start Facebook scraper in a separate thread
let rng = Arc::new(Mutex::new(StdRng::from_entropy()));
tokio::task::spawn_blocking(move || {
loop {
// 10% random cycle skip — makes pattern non-periodic
if rand::thread_rng().gen_range(0..100) < 10 {
info!("Skipping this scrape cycle (random 10% skip)");
let jitter = rand::thread_rng().gen_range(16200..19800);
tokio::time::sleep(Duration::from_secs(jitter)).await;
continue;
{
let _lock = rng.lock().unwrap();
run_facebook_scraper();
}
// Skip night hours (23:00 – 06:00)
let hour = chrono::Local::now().hour();
if hour < 6 || hour >= 23 {
info!("Night hours ({}) — skipping scrape", hour);
let jitter = rand::thread_rng().gen_range(16200..19800);
tokio::time::sleep(Duration::from_secs(jitter)).await;
continue;
}
let now = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs();
// Pick next active un-flagged account
let account = sqlx::query_as::<_, (uuid::Uuid, String)>(
"SELECT id, profile_path FROM facebook_accounts \
WHERE is_active = TRUE AND flagged = FALSE \
ORDER BY last_scrape_at ASC NULLS FIRST LIMIT 1"
)
.fetch_optional(&bg_db)
.await;
match account {
Ok(Some((account_id, profile_path))) => {
match client.post(&bg_url).query(&[("profile_path", &profile_path)]).send().await {
Ok(resp) => {
if resp.status().is_success() {
match resp.json::<ScrapeResponse>().await {
Ok(data) => {
let leads_count = data.leads.len() as i32;
if data.flagged {
let _ = sqlx::query(
"UPDATE facebook_accounts SET flagged = TRUE, flagged_at = NOW(), \
flagged_reason = $2, last_error_at = NOW(), \
last_error_message = $3, consecutive_failures = consecutive_failures + 1 \
WHERE id = $1"
)
.bind(account_id)
.bind(&data.flag_reason)
.bind(&data.error)
.execute(&bg_db)
.await;
warn!("Facebook account {} flagged: {:?}", account_id, data.flag_reason);
let reason = data.flag_reason.as_deref().unwrap_or("unknown");
let _ = sqlx::query(
"INSERT INTO notifications (user_id, type, title, description, link) \
SELECT id, 'warning', 'Facebook Account Flagged', \
$1 || ' - ' || COALESCE($2, 'unknown reason'), \
NULL \
FROM users u JOIN user_roles ur ON ur.user_id = u.id \
JOIN roles r ON r.id = ur.role_id \
WHERE r.name IN ('ADMIN', 'SUPER_ADMIN')"
)
.bind(&account_id.to_string())
.bind(reason)
.execute(&bg_db)
.await;
} else if data.success {
let _ = sqlx::query(
"UPDATE facebook_accounts SET last_scrape_at = NOW(), \
last_success_at = NOW(), consecutive_failures = 0, \
updated_at = NOW() WHERE id = $1"
)
.bind(account_id)
.execute(&bg_db)
.await;
let mut store = bg_leads.lock().await;
for item in &data.leads {
store.push(Lead {
title: truncate(&item.title, 120),
url: item.url.clone(),
source: item.source.clone().unwrap_or_else(|| "facebook".to_string()),
found_at: now,
author: truncate(&item.author, 60),
date: truncate(&item.date, 30),
content: truncate(&item.content, 300),
});
}
info!("Scraped {} leads from Facebook account {}", leads_count, account_id);
} else {
// Increment failures; auto-flag if >= 3 consecutive
let _ = sqlx::query(
"UPDATE facebook_accounts SET last_error_at = NOW(), \
last_error_message = $2, consecutive_failures = consecutive_failures + 1, \
flagged = CASE WHEN consecutive_failures + 1 >= 3 THEN TRUE ELSE flagged END, \
flagged_reason = CASE WHEN consecutive_failures + 1 >= 3 THEN 'too_many_failures' ELSE flagged_reason END, \
flagged_at = CASE WHEN consecutive_failures + 1 >= 3 THEN NOW() ELSE flagged_at END, \
updated_at = NOW() WHERE id = $1"
)
.bind(account_id)
.bind(&data.error)
.execute(&bg_db)
.await;
warn!("Facebook scrape failed for account {}: {:?}", account_id, data.error);
}
let _ = sqlx::query(
"INSERT INTO facebook_scrape_logs \
(account_id, started_at, completed_at, success, leads_found, error_message, detected_flag) \
VALUES ($1, NOW() - interval '5 hours', NOW(), $2, $3, $4, $5)"
)
.bind(account_id)
.bind(data.success && !data.flagged)
.bind(leads_count)
.bind(&data.error)
.bind(&data.flag_reason)
.execute(&bg_db)
.await;
}
Err(e) => {
warn!("Failed to parse scraper JSON: {}", e);
}
}
} else {
warn!("Scraper returned status: {}", resp.status());
}
}
Err(e) => {
warn!("Scraper request failed: {}", e);
}
}
}
Ok(None) => {
info!("No active Facebook accounts available — skipping scrape cycle");
}
Err(e) => {
warn!("Failed to query Facebook accounts: {}", e);
}
}
let jitter = rand::thread_rng().gen_range(16200..19800); // 4.5h – 5.5h
tokio::time::sleep(Duration::from_secs(jitter)).await;
// Introduce random delay
let delay = {
let mut rng = rng.lock().unwrap();
rng.gen_range(1..5)
};
thread::sleep(Duration::from_secs(delay));
}
});
axum::serve(listener, app)
.await
.expect("Server failed");
}
}
rust-ai/target/.rustc_info.json
BIN
View File
Binary file not shown.
scraper-service/node_modules/mime/.npmignore → rust-ai/target/debug/.cargo-artifact-lock
View File
rust-ai/target/debug/.cargo-build-lock
View File
rust-ai/target/debug/.cargo-lock
View File
rust-ai/target/debug/.fingerprint/allocator-api2-2a8f64255b7efe9c/dep-lib-allocator_api2
BIN
View File
Binary file not shown.
rust-ai/target/debug/.fingerprint/allocator-api2-2a8f64255b7efe9c/invoked.timestamp
BIN
View File
Binary file not shown.
rust-ai/target/debug/.fingerprint/allocator-api2-2a8f64255b7efe9c/lib-allocator_api2
BIN
View File
Binary file not shown.
rust-ai/target/debug/.fingerprint/allocator-api2-2a8f64255b7efe9c/lib-allocator_api2.json
BIN
View File
Binary file not shown.
rust-ai/target/debug/.fingerprint/allocator-api2-946d9b20e12d1f07/dep-lib-allocator_api2
BIN
View File
Binary file not shown.
rust-ai/target/debug/.fingerprint/allocator-api2-946d9b20e12d1f07/invoked.timestamp
BIN
View File
Binary file not shown.
rust-ai/target/debug/.fingerprint/allocator-api2-946d9b20e12d1f07/lib-allocator_api2
BIN
View File
Binary file not shown.
rust-ai/target/debug/.fingerprint/allocator-api2-946d9b20e12d1f07/lib-allocator_api2.json
BIN
View File
Binary file not shown.
rust-ai/target/debug/.fingerprint/allocator-api2-f9604a23100df9e0/dep-lib-allocator_api2
BIN
View File
Binary file not shown.
rust-ai/target/debug/.fingerprint/allocator-api2-f9604a23100df9e0/invoked.timestamp
BIN
View File
Binary file not shown.
rust-ai/target/debug/.fingerprint/allocator-api2-f9604a23100df9e0/lib-allocator_api2
BIN
View File
Binary file not shown.
rust-ai/target/debug/.fingerprint/allocator-api2-f9604a23100df9e0/lib-allocator_api2.json
BIN
View File
Binary file not shown.
rust-ai/target/debug/.fingerprint/async-trait-68aee6d09cf7ad62/dep-lib-async_trait
BIN
View File
Binary file not shown.
rust-ai/target/debug/.fingerprint/async-trait-68aee6d09cf7ad62/invoked.timestamp
BIN
View File
Binary file not shown.
rust-ai/target/debug/.fingerprint/async-trait-68aee6d09cf7ad62/lib-async_trait
BIN
View File
Binary file not shown.
rust-ai/target/debug/.fingerprint/async-trait-68aee6d09cf7ad62/lib-async_trait.json
BIN
View File
Binary file not shown.
rust-ai/target/debug/.fingerprint/atoi-2fbe13d0da7addd9/dep-lib-atoi
BIN
View File
Binary file not shown.
rust-ai/target/debug/.fingerprint/atoi-2fbe13d0da7addd9/invoked.timestamp
BIN
View File
Binary file not shown.

Some files were not shown because too many files have changed in this diff Show More