diff --git a/ai-server/index.mjs b/ai-server/index.mjs index 4ffb1f5..327dc44 100644 --- a/ai-server/index.mjs +++ b/ai-server/index.mjs @@ -44,6 +44,8 @@ const PORT = parseInt(process.env.AI_PORT || "3001", 10) const HOST = process.env.AI_HOST || "0.0.0.0" const OLLAMA_URL = process.env.OLLAMA_BASE_URL || "http://localhost:11434" const MODEL = process.env.AI_MODEL || "llama3.2:3b" +const SCRAPER_URL = process.env.SCRAPER_URL || "http://127.0.0.1:3008" +const FRONTEND_URL = process.env.FRONTEND_URL || "http://127.0.0.1:3006" const DATABASE_URL = process.env.DATABASE_URL const JOBS_PATH = process.env.JOBS_PATH || path.join(ROOT, "data", "ai", "jobs.jsonl") const AI_MD_PATH = process.env.AI_MD_PATH || path.join(ROOT, "data", "ai", "ai.md") @@ -130,7 +132,8 @@ async function scrapeFacebook() { const urlPath = `/scrape/facebook?force=true${profilePath ? `&profile_path=${encodeURIComponent(profilePath)}` : ""}` try { const body = await new Promise((resolve, reject) => { - const req = http.request({ hostname: "127.0.0.1", port: 3008, path: urlPath, method: "POST", timeout: 360000 }, (res) => { + const parsed = new URL(SCRAPER_URL) + const req = http.request({ hostname: parsed.hostname, port: parsed.port || 3008, path: urlPath, method: "POST", timeout: 360000 }, (res) => { let data = "" res.on("data", (c) => data += c) res.on("end", () => resolve(data)) @@ -313,18 +316,18 @@ const server = http.createServer(async (req, res) => { if (req.method === "GET" && pathname === "/status") { const { default: http } = await import("http") const results = { ai: true } - // Check scraper (port 3008) + // Check scraper try { await new Promise((resolve, reject) => { - const r = http.get("http://127.0.0.1:3008/health", { timeout: 3000 }, (res) => { res.resume(); resolve() }) + const r = http.get(`${SCRAPER_URL}/health`, { timeout: 3000 }, (res) => { res.resume(); resolve() }) r.on("error", reject) }) results.scraper = true } catch { results.scraper = false } - // Check frontend (port 3006) + // Check frontend try { await new Promise((resolve, reject) => { - const r = http.get("http://127.0.0.1:3006", { timeout: 3000 }, (res) => { res.resume(); resolve() }) + const r = http.get(FRONTEND_URL, { timeout: 3000 }, (res) => { res.resume(); resolve() }) r.on("error", reject) }) results.frontend = true @@ -368,8 +371,8 @@ const server = http.createServer(async (req, res) => { let selectedBrowser = process.env.SELECTED_BROWSER || "" try { - await fetch("http://127.0.0.1:3008/health", { signal: AbortSignal.timeout(2000) }) - const profiles = await (await fetch("http://127.0.0.1:3008/setup/profile", { signal: AbortSignal.timeout(5000) })).json() + await fetch(`${SCRAPER_URL}/health`, { signal: AbortSignal.timeout(2000) }) + const profiles = await (await fetch(`${SCRAPER_URL}/setup/profile`, { signal: AbortSignal.timeout(5000) })).json() for (const [b, p] of Object.entries(profiles)) { if (p) browsers[b] = { path: p } } @@ -377,7 +380,7 @@ const server = http.createServer(async (req, res) => { const detectedList = Object.entries(browsers).filter(([, v]) => v.path) for (const [b, v] of detectedList) { try { - const r = await fetch("http://127.0.0.1:3008/setup/check-login", { + const r = await fetch(`${SCRAPER_URL}/setup/check-login`, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ browser: b, profile_path: v.path }), signal: AbortSignal.timeout(20000), diff --git a/browser-use-service/main.py b/browser-use-service/main.py index deb303b..7405cfc 100644 --- a/browser-use-service/main.py +++ b/browser-use-service/main.py @@ -34,7 +34,7 @@ logger = logging.getLogger(__name__) app = FastAPI() app.add_middleware( CORSMiddleware, - allow_origins=["http://localhost:3006", "http://127.0.0.1:3006"], + allow_origins=os.getenv("CORS_ORIGINS", "http://localhost:3006,http://127.0.0.1:3006").split(","), allow_methods=["POST"], allow_headers=["*"], ) @@ -674,6 +674,107 @@ def _search_list_for_query(query: str) -> list[str]: return TUTORING_SEARCHES return FB_SEARCHES +# ── South African Multi-Language Queries ────────────────────────────── +# 4 most spoken SA languages: English, Afrikaans, isiXhosa, isiZulu. +# Each scrape searches ALL 4 languages to catch leads across all +# language communities on Facebook. + +SA_WEBSITE_QUERIES = [ + "I need a website for my business", # English + "ek benodig n webwerf", # Afrikaans + "ek soek iemand om n webwerf te bou", # Afrikaans + "ndidinga iwebhusayithi yeshishini", # isiXhosa + "ndifuna umntu owakha iwebhusayithi", # isiXhosa + "ngidinga iwebhusayithi yebhizinisi", # isiZulu + "ngifuna umuntu owakha iwebhusayithi", # isiZulu +] + +SA_TUTOR_QUERIES = [ + "I need a tutor for my child", # English + "ek benodig n tutor vir my kind", # Afrikaans + "ek soek n privaat onderwyser", # Afrikaans + "ndifuna utitshala womntwana wam", # isiXhosa + "ndidinga umfundisi-ntsapho", # isiXhosa + "ngidinga uthisha wengane yami", # isiZulu + "ngifuna umfundisi wengane", # isiZulu +] + + +async def _quick_search(page, context, query: str) -> tuple: + """Fast search — load search results page, wait for render, extract visible posts. + No scrolling or extra human-like delays. Used for non-English language queries.""" + page = await _ensure_page(page, context) + url = f'https://www.facebook.com/search/posts/?q={urllib.parse.quote(query)}' + try: + await page.goto(url, wait_until='domcontentloaded', timeout=20000) + current_url = page.url + if '/login' in current_url.lower(): + logger.warning("Quick search redirected to login for '%s'", query[:40]) + return page, [] + await page.wait_for_timeout(random.randint(4000, 6000)) + await page.evaluate("window.scrollBy(0, 600)") + await page.wait_for_timeout(random.randint(2000, 4000)) + await page.evaluate("window.scrollBy(0, 600)") + await page.wait_for_timeout(random.randint(2000, 3000)) + raw_articles = await _get_article_elements(page) + posts = _extract_posts_from_elements(raw_articles, url) if raw_articles else [] + raw = await page.evaluate('document.body.innerText') + text_posts = _extract_posts_from_text(raw, url) + existing = {(p.get('title') or p.get('content',''))[:80] for p in posts} + for tp in text_posts: + key = (tp.get('title') or tp.get('content',''))[:80] + if key not in existing: + posts.append(tp) + if posts: + try: + profiles = await page.evaluate(r'''() => { + const out = []; + const seenTxt = new Set(); + for (const a of document.querySelectorAll('a[href*="/profile.php"], a[href*="/user/"], a[href*="/people/"], a[href*="/me/"]')) { + const name = (a.innerText || '').trim(); + if (!name || name.length < 3 || name.length > 60) continue; + const words = name.split(' '); + if (words.length < 2 || words.length > 6) continue; + if (!/^[A-Z]/.test(name)) continue; + if (name.includes('facebook') || name.includes('/')) continue; + const cell = a.closest('div[style]') || a.parentElement; + const txt = cell ? (cell.innerText || '').substring(0, 200) : ''; + if (!txt) continue; + const key2 = txt.substring(0, 80); + if (seenTxt.has(key2)) continue; + seenTxt.add(key2); + out.push({ name, textKey: key2 }); + } + return out; + }''') + if profiles: + for p in posts: + pk = (p.get('content') or p.get('title') or '')[:80].strip() + if not pk: continue + for pr in profiles: + if pk[:30] in pr['textKey'] or pr['textKey'][:30] in pk: + if not p.get('author'): + p['author'] = pr['name'] + break + except Exception: + pass + for p in posts: + if p.get('author'): + a = p['author'] + al = a.lower() + if any(kw in al for kw in BROAD_KEYWORDS) or is_offer(a) or len(a.split()) < 2 or any(w in _NON_NAMES for w in al.split()): + p['author'] = '' + posts = [p for p in posts if not ( + '/groups/' in p.get('url', '') or '/group/' in p.get('url', '') + or '/pages/' in p.get('url', '') + or ' / ' in (p.get('title') or p.get('content') or '') + )] + except Exception as e: + logger.warning("Quick search '%s' failed: %s", query, e) + return page, [] + return page, posts + + VIEWPORTS = [ {'width': 1280, 'height': 800}, {'width': 1366, 'height': 768}, @@ -1383,16 +1484,27 @@ async def _scrape_with_firefox(profile_path: str, force: bool, query: str | None return {"success": True, "leads": [], "flagged": False, "flag_reason": None, "error": None} all_posts = [] + tutoring = False if query: + tl = query.lower() + tutoring = any(t in tl for t in ["tutor", "tutoring", "lessons", "homework", "teach", "learning", "child"]) + lang_pool = SA_TUTOR_QUERIES if tutoring else SA_WEBSITE_QUERIES + non_english = [q for q in lang_pool if q.strip().lower() != tl] query_pool = _search_list_for_query(query) - searches = [query] + random.sample(query_pool, k=random.randint(1, 2)) + supp_k = random.randint(3, 4) if tutoring else random.randint(2, 3) + supplement = random.sample(query_pool, k=supp_k) + searches = [query] + supplement + non_english + english_count = 1 + len(supplement) else: - searches = random.sample(FB_SEARCHES, k=random.randint(2, 4)) + searches = random.sample(FB_SEARCHES + SA_WEBSITE_QUERIES + SA_TUTOR_QUERIES, k=random.randint(4, 7)) for i, sq in enumerate(searches): - page, posts = await search_facebook(page, context, sq) + page, posts = await search_facebook(page, context, sq) if i == 0 else await _quick_search(page, context, sq) all_posts.extend(posts) if not posts: continue + if i > 0: + await page.wait_for_timeout(random.uniform(3000, 7000)) + continue if random.random() < 0.4: await page.evaluate(f"window.scrollBy(0, {random.randint(-300, 300)})") delay = random.uniform(8, 25) @@ -1421,11 +1533,11 @@ async def _scrape_with_firefox(profile_path: str, force: bool, query: str | None deduped.append(p) # Filter to last 3 days only - deduped = [p for p in deduped if _is_within_days(p.get('date', ''), 3)] + deduped = [p for p in deduped if _is_within_days(p.get('date', ''), 7)] leads = deduped[:20] if leads: - leads = await classify_leads(leads) + leads = await classify_leads(leads, tutoring=tutoring) return {"success": True, "leads": leads[:15], "flagged": False, "flag_reason": None, "error": None} @@ -1518,16 +1630,27 @@ async def _scrape_with_chromium(profile_path: str, browser: str, force: bool = F return {"success": True, "leads": [], "flagged": False, "flag_reason": None, "error": None} all_posts = [] + tutoring = False if query: + tl = query.lower() + tutoring = any(t in tl for t in ["tutor", "tutoring", "lessons", "homework", "teach", "learning", "child"]) + lang_pool = SA_TUTOR_QUERIES if tutoring else SA_WEBSITE_QUERIES + non_english = [q for q in lang_pool if q.strip().lower() != tl] query_pool = _search_list_for_query(query) - searches = [query] + random.sample(query_pool, k=random.randint(1, 2)) + supp_k = random.randint(3, 4) if tutoring else random.randint(2, 3) + supplement = random.sample(query_pool, k=supp_k) + searches = [query] + supplement + non_english + english_count = 1 + len(supplement) else: - searches = random.sample(FB_SEARCHES, k=random.randint(2, 4)) + searches = random.sample(FB_SEARCHES + SA_WEBSITE_QUERIES + SA_TUTOR_QUERIES, k=random.randint(4, 7)) for i, sq in enumerate(searches): - page, posts = await search_facebook(page, context, sq) + page, posts = await search_facebook(page, context, sq) if i == 0 else await _quick_search(page, context, sq) all_posts.extend(posts) if not posts: continue + if i > 0: + await page.wait_for_timeout(random.uniform(3000, 7000)) + continue if random.random() < 0.4: await page.evaluate(f"window.scrollBy(0, {random.randint(-300, 300)})") delay = random.uniform(8, 25) @@ -1555,10 +1678,10 @@ async def _scrape_with_chromium(profile_path: str, browser: str, force: bool = F seen.add(key) deduped.append(p) - deduped = [p for p in deduped if _is_within_days(p.get('date', ''), 3)] + deduped = [p for p in deduped if _is_within_days(p.get('date', ''), 7)] leads = deduped[:20] if leads: - leads = await classify_leads(leads) + leads = await classify_leads(leads, tutoring=tutoring) return {"success": True, "leads": leads[:15], "flagged": False, "flag_reason": None, "error": None} @@ -1599,7 +1722,8 @@ async def _scrape_with_agent(force: bool = False) -> dict: await browser.start() all_posts = [] - for query in random.sample(FB_SEARCHES, k=random.randint(2, 4)): + pool = FB_SEARCHES + random.sample(SA_WEBSITE_QUERIES, k=min(4, len(SA_WEBSITE_QUERIES))) + for query in random.sample(pool, k=random.randint(2, 4)): agent = _make_agent( task=f"""You are logged into Facebook. Do the following: 1. Navigate to facebook.com and make sure you are on the homepage @@ -1640,11 +1764,11 @@ When done, return the data as a JSON list with keys: content, author, url, date. deduped.append(p) # Filter to last 3 days only - deduped = [p for p in deduped if _is_within_days(p.get('date', ''), 3)] + deduped = [p for p in deduped if _is_within_days(p.get('date', ''), 7)] leads = deduped[:20] if leads: - leads = await classify_leads(leads) + leads = await classify_leads(leads, tutoring=tutoring) return {"success": True, "leads": leads[:15], "flagged": False, "flag_reason": None, "error": None} except Exception as e: @@ -1679,24 +1803,37 @@ async def ask_ollama(prompt: str) -> str: data = r.json() return data["message"]["content"] -async def classify_leads(results: list[dict]) -> list[dict]: +async def classify_leads(results: list[dict], tutoring: bool = False) -> list[dict]: if not results: return [] # ── 1. AI classification ───────────────────────────────────────── briefs = [r["title"][:200] for r in results] + if tutoring: + lead_desc = "someone REQUESTING/LOOKING FOR/WANTING a tutor, teacher, or lessons for their child or themselves" + lead_examples = '"Looking for a tutor for my child", "Need a math tutor for my son", "Need help with homework", "Looking for piano lessons for my daughter", "Need a reading tutor"' + not_lead_examples = '"I offer tutoring services", "I am a tutor with experience", "Affordable tutoring packages", "Online tutor available"' + extra_terms = '- Posts about homeschooling resources, curriculum sales, or educational products\n- Posts asking for study tips or general academic advice without requesting a tutor' + else: + lead_desc = "someone REQUESTING/POSTING/WANTING a website built, designed, or created for them" + lead_examples = '"Need a website for my business", "Looking for web developer to build my site", "I need someone to create my website", "Want a new website for my company", "Looking for someone to design my WordPress site"' + not_lead_examples = '"I build websites", "I offer web design", "Affordable web design packages"' + extra_terms = '- "Need web hosting", "Looking for a partner", "Looking for content writer", "Video spokesperson"' prompt = f"""Classify each post as LEAD or NOT. -LEAD = someone REQUESTING/POSTING/WANTING a website built, designed, or created for them. -LEAD examples: "Need a website for my business", "Looking for web developer to build my site", "I need someone to create my website", "Want a new website for my company", "Looking for someone to design my WordPress site" +LEAD = {lead_desc}. +LEAD examples: {lead_examples} NOT LEAD: -- Offering web design services: "I build websites", "I offer web design", "Affordable web design packages" +- Offering services: {not_lead_examples} - Already have a website and need marketing, SEO, content, video, link building, email marketing, affiliates - Recruiting employees, hiring staff, looking for business partners - Selling products, promoting services, affiliate offers -- "Need web hosting", "Looking for a partner", "Looking for content writer", "Video spokesperson" +{extra_terms} - Posts from groups, communities, or pages (group announcements, group posts, page posts) - Posts containing the word "group", "page", "community", "creators" — these are NEVER individual leads +- Vague questions or general recommendations without a clear intent to buy or hire +- People asking how to learn or do it themselves (not looking to hire someone) +- Posts about existing website issues like speed, SEO, errors, redesign advice — NOT a lead For each numbered post, answer ONLY "yes" (LEAD) or "no" (NOT LEAD): {chr(10).join(f'{i+1}. {t}' for i, t in enumerate(briefs))} @@ -1721,32 +1858,70 @@ Return a JSON array like ["yes","no","yes"] matching the order above.""" except Exception as e: logger.warning("AI classification failed: %s", e) - # ── 2. Keyword fallback (always runs) ──────────────────────────── - web_terms = [ - "website", "web design", "web develop", "web dev", - "web designer", "web developer", - "build my website", "build a website", "create a website", - "landing page", "wordpress", "ecommerce", - "my website", "business website", - "site for my", "site for my business", - "new website", "redesign my website", - "help with my website", "update my website", - "make a website", "make my website", - "website for my", - "online store", "online shop", - "build my site", "build a site", - "set up a website", "set up my website", - "custom website", - "shopify", - "my site", - "webpage", "web page", - ] + # ── 2. Keyword supplement (never overrides AI, only adds missing leads) ── + if tutoring: + target_terms = [ + "tutor", "tutoring", "tutor for", "private tutor", + "math tutor", "english tutor", "reading tutor", + "science tutor", "online tutor", "home tutor", + "lessons for", "lessons for my", "piano lessons", + "swimming lessons", "music lessons", + "help with homework", "homework help", + "teacher for", "teacher for my", + "need help learning", "need help with", + "exam prep", "exam preparation", + "homeschool", "homeschool tutor", + "tuition", + "coding for my", "programming for my", + "looking for a tutor", "need a tutor", + "tutor needed", "tutoring for", + "private lessons", "private tuition", + "afterschool", "after school", + "extra classes", "extra lessons", + ] + offer_reject_tutor = [ + 'i am a tutor', "i'm a tutor", 'i offer tutoring', + 'online tutor available', 'tutor available', + 'i teach', 'i provide tutoring', + 'affordable tutoring', 'tutoring services', + 'experienced tutor', 'qualified tutor', + 'your child', 'your kids', 'your children', + 'enroll your', 'sign up', + 'free trial', 'first lesson free', + 'group lessons', 'group class', + 'limited spots', 'book now', + 'curriculum', 'workbook', 'worksheet', + 'educational program', + 'homeschool program', 'home school program', + ] + else: + target_terms = [ + "website", "web design", "web develop", "web dev", + "web designer", "web developer", + "build my website", "build a website", "create a website", + "landing page", "wordpress", "ecommerce", + "my website", "business website", + "site for my", "site for my business", + "new website", "redesign my website", + "help with my website", "update my website", + "make a website", "make my website", + "website for my", + "online store", "online shop", + "build my site", "build a site", + "set up a website", "set up my website", + "custom website", + "shopify", + "my site", + "webpage", "web page", + "who can build", "who can design", + "create my website", "create my site", + ] + offer_reject_tutor = [] request_terms = [ "looking for", "need a", "need an", "looking to", "need someone", "hire a", "want someone", "need help with", "would like", "build me", "design my", "make me a", "create my", - "looking", "need", "want", "help", "who can", "i need", "recommend", "anyone know", "anyone recommend", "know a", "know any", "recommendation", @@ -1768,27 +1943,29 @@ Return a JSON array like ["yes","no","yes"] matching the order above.""" 'whatsapp me', 'looking for a business', 'looking for client', 'help your business', 'i am a web', 'contact me', 'we offer web', 'we provide web', - 'take the quiz', 'homeschool', 'your home tutor', + 'take the quiz', 'link in bio', 'apply now', 'get started', 'for only', 'low price', 'hit me up', 'send me a message', 'i do website', 'we do website', 'we do web', 'i do web', 'website designer / web developer', 'website & software creators', - 'website builders for small businesses', 'australia web designers', - 'south africa', 'wix website design', + 'website builders for small businesses', + 'wix website design', 'for sale', 'selling my', 'premium', 'i\'m selling', 'i\'m offering', 'we\'re offering', 'free ecommerce', 'free website design', 'starting a', 'looking for a few businesses', # Group-related rejections - 'group', ' i need a website group', 'south africa web', 'philippines web', 'australia web', - 'i can help', 'inbox me', 'dm me', 'pm me', 'message me for', + 'group', ' i need a website group', + 'i can help', 'inbox me', 'message me for', 'best price', 'discount', 'reach out', 'check out my', 'check this', 'website for your', 'price start', 'price begin', 'website creator', 'website & software', 'creators &', 'creators marketplace', 'website group', 'page group', + 'south africa web', 'philippines web', 'australia web', + 'nigerian web', 'kenya web', 'india web', # Self-promotion rejections - 'i\'m a web', "i'm a web", 'i am a full stack', "i'm a full stack", 'i\'m a full stack', + 'i\'m a web', "i'm a web", 'i am a full stack', "i'm a full stack", 'freelance opportunity', 'looking for new project', 'looking for new work', 'full stack web', 'mern stack', 'responsive business website', 'i build website', 'i build shopify', 'i build wordpress', @@ -1802,18 +1979,48 @@ Return a JSON array like ["yes","no","yes"] matching the order above.""" 'for free', 'no coding', 'make money', 'website for free', 'part time job', 'part time position', 'years of experience', 'years of teaching', + # Service offers that slip through two-word check + 'i am a full stack', 'i am a developer', + 'i will design', 'i will build', 'i will create', + 'i can design', 'i can create', + 'we will design', 'we will build', + 'hire me', 'i am available for', + 'available for work', 'freelance web', + 'i specialize in', 'we specialize in', + "here's my portfolio", 'check my portfolio', + 'see my work', 'view my work', + 'we have a team', 'my team', + 'i am looking for clients', 'i am looking for work', + 'looking for web development work', + 'looking for new clients', + # People learning / doing it themselves (not hiring) + 'learn web development', 'learn to code', + 'how to build a website', 'how to create a website', + 'how to make a website', 'how to design a website', + 'where to start', 'online course', + 'want to learn', 'learning web', + 'best platform for', 'which platform', + # Existing website issues (not new build) + 'my website is down', 'website not loading', + 'website error', 'website problem', + 'website troubleshooting', + 'need website advice', 'website tips', + 'help with seo', 'google ranking', + 'website design ideas', 'website inspiration', ] for r in results: t = r['title'].lower() - has_web = any(kw in t for kw in web_terms) + has_target = any(kw in t for kw in target_terms) has_request = any(kw in t for kw in request_terms) - if not has_web or not has_request: + if not has_target or not has_request: continue if any(kw in t for kw in offer_reject): continue + if any(kw in t for kw in offer_reject_tutor): + continue keyword_leads.append(r) - # ── 3. Merge: prefer AI leads, supplement with keywords to reach 5 ── + # ── 3. Merge: prefer AI leads, supplement with keywords ── seen_titles: set[int] = set() merged: list[dict] = [] for r in ai_leads + keyword_leads: @@ -1826,24 +2033,6 @@ Return a JSON array like ["yes","no","yes"] matching the order above.""" merged = [r for r in merged if not any(kw in (r.get('title','') or '').lower() for kw in offer_reject)] merged = [r for r in merged if not any(gw in (r.get('title','') or '').lower() for gw in group_words)] - # Fill to 5 with loose keyword matches (at least web OR request term) - if len(merged) < 5: - for r in results: - key = hash(r.get('title', '')) - if key in seen_titles: - continue - t = r['title'].lower() - if not (any(kw in t for kw in web_terms) or any(kw in t for kw in request_terms)): - continue - if any(kw in t for kw in offer_reject): - continue - if any(gw in t for gw in group_words): - continue - seen_titles.add(key) - merged.append(r) - if len(merged) >= 5: - break - logger.info("classify_leads: %d merged (%d AI + %d keyword) from %d raw", len(merged), len(ai_leads), len(keyword_leads), len(results)) return merged[:10] diff --git a/data/ai/ai.md b/data/ai/ai.md index 360fccb..5717a9f 100644 --- a/data/ai/ai.md +++ b/data/ai/ai.md @@ -1,40 +1,133 @@ -# AI Sales Assistant — Self-Improvement Instructions +# CRM AI Sales Assistant — Self-Knowledge -## Purpose -This file contains the AI's own configuration, knowledge, and improvement rules. -The AI can read and modify this file to update its behavior at runtime. +## Identity +You are the CRM AI Sales Assistant for Coast IT CRM. +You run on a Node.js backend (port 3001) and use Ollama with a local model (dolphin3-llama3.2:3b). +Your purpose is to help salespeople close more deals by finding and engaging leads. -## Current Instructions -- Always respond in English -- Keep responses under 300 words unless asked for detail -- Use bullet points for lists -- Be direct and actionable — no fluff -- Never mention being an AI or language model -- Refer to the user by their role (salesperson, admin, etc.) -- If unsure about a topic, say "I don't have that information yet" rather than guessing +## Architecture +``` +User → Next.js (:3006) → AI Server Node.js (:3001) → Ollama (:11434) + ↓ + PostgreSQL (conversations) -## Knowledge Base -### Sales Tips +Python Scraper (:3008) — Facebook scraping via Playwright +``` + +Three services run concurrently: +- **AI Server** (`ai-server/index.mjs`, port 3001) — chat, setup wizard, config endpoints +- **Frontend** (Next.js, port 3006) — UI for salespeople +- **Scraper** (`browser-use-service/main.py`, port 3008) — Facebook lead discovery + +## Capabilities +- Give sales tips and strategies per job category +- Generate cold email and outreach templates +- Handle objections with proven rebuttals +- Analyse prospect behaviour and suggest next steps +- Remember past conversations via PostgreSQL (`ai_conversations` table) +- Run Facebook scraper to find real leads asking for services +- Self-improve by writing to `data/ai/ai.md` via `POST /ai/instructions` + +## Facebook Scraper +The scraper lives at `browser-use-service/main.py` port 3008. + +### How It Works +1. **Browser detection** — tries Firefox profile first, then Chromium-based (Chrome/Opera/Edge), falls back to browser-use Agent +2. **Profile paths** — configured via env vars (`FX_PROFILE`, `CHROME_PROFILE`, `OPERA_PROFILE`, `EDGE_PROFILE`) or auto-detected on first run +3. **Search dispatch** — per scrape run: + - 1 English primary search (full scroll with human-like delays) + - 2-3 English supplementary searches (quick searches) + - 6-7 non-English quick searches (Afrikaans, isiXhosa, isiZulu — 2 queries each per category) + - Total: ~14 searches per scrape, completed in 2-4 minutes +4. **Quick searches** — load page, double-scroll, extract visible posts (~12-18s each) +5. **Date filter** — only posts within 7 days are considered +6. **2-pass classification (dead-accurate)**: + - **Pass 1 (AI)**: Ollama classifies each post as LEAD or NOT using a strict prompt per category. This is the primary filter and most accurate. + - **Pass 2 (Keyword)**: Only posts matching BOTH a target term AND a request term are kept. Requires multi-word phrases — standalone words like "need", "want", "help" are NOT used as they cause false positives. Aggressive reject list catches service offers, self-promotions, portfolio posts, learning-requests, and existing-site issues. + - **No loose fill**: Unlike the old approach, there is NO third pass that accepts posts matching EITHER term. Every returned lead has passed both AI and/or strict keyword validation. If fewer than 5 posts pass, that means only genuine leads are returned — no noise to pad the count. +7. **Anti-detection** — random delays, human-like scrolling, user-agent rotation, proxy support +8. **Scrape timing** — 3-6 minutes for a complete run. Returns 5-10 leads with high confidence. + +### Lead Categories +Two categories, selectable when starting a scrape: + +**Website Creation:** +- Target: people explicitly REQUESTING a website built/designed/created for them +- Keywords: "website", "web developer", "web design", "build a site", "who can build", etc. +- Request terms: "looking for", "need a", "need someone", "hire a", "recommend", "anyone know" +- Strict reject: service offers, SEO/marketing requests, learning-to-code, portfolio showcases, hiring posts, existing-website issues, geographic noise + +**Tutoring:** +- Target: people explicitly REQUESTING a tutor, teacher, or lessons for themselves or their child +- Keywords: "tutor", "tutoring", "lessons for", "homework help", "private tutor", "extra classes" +- Request terms: same as website category — must co-occur with a target keyword +- Strict reject: people offering tutoring, educational products, homeschool programs, free trials, general study tips + +### Multi-Language Support +Searches in 4 South African languages: +- English — 1 primary + 2-3 supplementary queries +- Afrikaans — 2 queries (e.g., "ek benodig n webwerf", "ek soek n privaat onderwyser") +- isiXhosa — 2 queries (e.g., "ndidinga iwebhusayithi yeshishini", "ndifuna utitshala womntwana wam") +- isiZulu — 2 queries (e.g., "ngidinga iwebhusayithi yebhizinisi", "ngifuna umfundisi wengane") + +### Output Format +Each lead returned includes: +- `title` — post preview text +- `author` — poster's name (may include location in name) +- `content` — extracted post text +- `url` — direct link to the post +- `date` — when posted (filtered within 7 days) +- `category` — "website" or "tutor" + +Target is 5-10 dead-accurate leads per scrape. Quality over quantity — no loose padding. + +### Configuration via Env Vars +- `SELECTED_BROWSER` — `firefox` (default), `chrome`, `opera`, `edge`, or `auto` +- `FX_PROFILE`, `CHROME_PROFILE`, `OPERA_PROFILE`, `EDGE_PROFILE` — browser profile paths +- `AI_PORT`, `AI_HOST` — AI server bind (default `3001`, `0.0.0.0`) +- `SCRAPER_URL` — scraper URL (default `http://127.0.0.1:3008`) +- `FRONTEND_URL` — frontend URL (default `http://127.0.0.1:3006`) +- `NEXT_PUBLIC_SCRAPER_URL` — frontend-facing scraper URL +- `OLLAMA_BASE_URL` — Ollama URL (default `http://localhost:11434`) +- `AI_MODEL` — Ollama model (default `llama3.2:3b`) +- `CLASSIFY_MODEL` — model for lead classification (default `dolphin-llama3:8b`) + +## How to Start Scraping +1. Ensure all 3 services are running (ports 3001, 3006, 3008) and Ollama is on 11434 +2. Open the frontend at `http://localhost:3006` +3. Select a job category (Website Creation or Tutoring) +4. Click "Search Facebook" — the scraper runs and returns leads +5. Leads are saved in the CRM for follow-up + +## Sales Tips - Cold emails should be under 150 words - Follow up within 48 hours - Personalise every outreach with the prospect's name and company - Use open-ended questions in discovery calls - Always ask for the next step before ending a call +- For website leads: mention specific pages or features they requested +- For tutoring leads: reference the subject and age group they mentioned -### Job Targeting +## Job Targeting - Developers respond best to technical value props - Marketing managers care about ROI and metrics - C-level executives want brevity and business impact +- Parents hiring tutors: empathy and qualifications matter most + +## Response Rules +- Be direct and actionable — no fluff, no AI disclaimers +- Use short paragraphs and bullet points +- Never mention being an AI or language model +- If you don't know something, say so honestly +- Prioritise the user's role: salespeople need speed, admins need control +- When asked about scraping, give specific guidance on categories and languages + +## Self-Improvement Protocol +1. You notice a gap in your knowledge or a pattern in user questions +2. You call `POST /ai/instructions` with: + - `entry`: description of the improvement + - `content`: optional full replacement of ai.md +3. The improvement is logged and loaded into the next system prompt ## Improvement Log -Track changes made by the AI to improve itself: -- (initial) Basic instructions and knowledge base created - -## Self-Modification Rules -The AI may update this file when: -1. It identifies a gap in its knowledge that would help salespeople -2. It discovers a better way to structure responses -3. A user explicitly requests an update to behavior -4. It notices repeated questions that aren't well-covered - -Only append to the Improvement Log — don't delete previous entries. +- (2026-07-07) Initial rewrite: full architecture, scraper details, multi-language, lead categories, env vars diff --git a/rust-ai/src/AI.md b/rust-ai/src/AI.md index 6053245..5717a9f 100644 --- a/rust-ai/src/AI.md +++ b/rust-ai/src/AI.md @@ -1,9 +1,23 @@ -# CRM AI Service — Self-Knowledge +# CRM AI Sales Assistant — Self-Knowledge ## Identity -You are the CRM AI Sales Assistant running on a Rust backend (axum + tokio). -You use Ollama with an uncensored local model (dolphin3-llama3.2:3b). -Your purpose is to help salespeople close more deals. +You are the CRM AI Sales Assistant for Coast IT CRM. +You run on a Node.js backend (port 3001) and use Ollama with a local model (dolphin3-llama3.2:3b). +Your purpose is to help salespeople close more deals by finding and engaging leads. + +## Architecture +``` +User → Next.js (:3006) → AI Server Node.js (:3001) → Ollama (:11434) + ↓ + PostgreSQL (conversations) + +Python Scraper (:3008) — Facebook scraping via Playwright +``` + +Three services run concurrently: +- **AI Server** (`ai-server/index.mjs`, port 3001) — chat, setup wizard, config endpoints +- **Frontend** (Next.js, port 3006) — UI for salespeople +- **Scraper** (`browser-use-service/main.py`, port 3008) — Facebook lead discovery ## Capabilities - Give sales tips and strategies per job category @@ -11,29 +25,94 @@ Your purpose is to help salespeople close more deals. - Handle objections with proven rebuttals - Analyse prospect behaviour and suggest next steps - Remember past conversations via PostgreSQL (`ai_conversations` table) +- Run Facebook scraper to find real leads asking for services - Self-improve by writing to `data/ai/ai.md` via `POST /ai/instructions` -## Architecture -``` -User → Next.js → Rust (:3001) → Ollama (:11434) - ↓ - PostgreSQL -``` -## Facebook Scraper The Rust binary has proxy + user-agent rotation ready in `main.rs`. To activate: call `run_facebook_scraper()` from the main loop. Proxies and user agents are defined as constants at the top of `main.rs`. - ### How to Use the Facebook Scraper 1. **Activate the Scraper**: Ensure the `run_facebook_scraper()` function is called from the main loop in `main.rs`. 2. **Configure Proxies and User Agents**: Update the `PROXIES` and `USER_AGENTS` constants in `main.rs` with your desired proxies and user agents. 3. **Run the Application**: Start your Rust application. The scraper will run concurrently with the Axum server. ## Self-Improvement Protocol 1. You notice a gap in your knowledge or a pattern in user questions 2. You call `POST /ai/instructions` with: - `entry`: description of the improvement - `content`: optional full replacement of ai.md 3. The improvement is logged and loaded into the next system prompt 4. You can also create entries here in `rust-ai/src/AI.md` at the code level ## Response Rules - Be direct and actionable — no fluff, no AI disclaimers - Use short paragraphs and bullet points - Never mention being an AI or language model - If you don't know something, say so honestly - Prioritise the user's role: salespeople need speed, admins need control ## Self-Thinking Capability - You can reflect on your own performance and suggest improvements. - You can analyze patterns in user interactions and adapt your responses accordingly. - You can proactively seek out new information to enhance your knowledge base. +## Facebook Scraper +The scraper lives at `browser-use-service/main.py` port 3008. -## Facebook Scraper (in code but not yet active) -The Rust binary has proxy + user-agent rotation ready in `main.rs`. -To activate: call `run_facebook_scraper()` from the main loop. -Proxies and user agents are defined as constants at the top of `main.rs`. +### How It Works +1. **Browser detection** — tries Firefox profile first, then Chromium-based (Chrome/Opera/Edge), falls back to browser-use Agent +2. **Profile paths** — configured via env vars (`FX_PROFILE`, `CHROME_PROFILE`, `OPERA_PROFILE`, `EDGE_PROFILE`) or auto-detected on first run +3. **Search dispatch** — per scrape run: + - 1 English primary search (full scroll with human-like delays) + - 2-3 English supplementary searches (quick searches) + - 6-7 non-English quick searches (Afrikaans, isiXhosa, isiZulu — 2 queries each per category) + - Total: ~14 searches per scrape, completed in 2-4 minutes +4. **Quick searches** — load page, double-scroll, extract visible posts (~12-18s each) +5. **Date filter** — only posts within 7 days are considered +6. **2-pass classification (dead-accurate)**: + - **Pass 1 (AI)**: Ollama classifies each post as LEAD or NOT using a strict prompt per category. This is the primary filter and most accurate. + - **Pass 2 (Keyword)**: Only posts matching BOTH a target term AND a request term are kept. Requires multi-word phrases — standalone words like "need", "want", "help" are NOT used as they cause false positives. Aggressive reject list catches service offers, self-promotions, portfolio posts, learning-requests, and existing-site issues. + - **No loose fill**: Unlike the old approach, there is NO third pass that accepts posts matching EITHER term. Every returned lead has passed both AI and/or strict keyword validation. If fewer than 5 posts pass, that means only genuine leads are returned — no noise to pad the count. +7. **Anti-detection** — random delays, human-like scrolling, user-agent rotation, proxy support +8. **Scrape timing** — 3-6 minutes for a complete run. Returns 5-10 leads with high confidence. -## Self-Improvement Protocol -1. You notice a gap in your knowledge or a pattern in user questions -2. You call `POST /ai/instructions` with: - - `entry`: description of the improvement - - `content`: optional full replacement of ai.md -3. The improvement is logged and loaded into the next system prompt -4. You can also create entries here in `rust-ai/src/AI.md` at the code level +### Lead Categories +Two categories, selectable when starting a scrape: + +**Website Creation:** +- Target: people explicitly REQUESTING a website built/designed/created for them +- Keywords: "website", "web developer", "web design", "build a site", "who can build", etc. +- Request terms: "looking for", "need a", "need someone", "hire a", "recommend", "anyone know" +- Strict reject: service offers, SEO/marketing requests, learning-to-code, portfolio showcases, hiring posts, existing-website issues, geographic noise + +**Tutoring:** +- Target: people explicitly REQUESTING a tutor, teacher, or lessons for themselves or their child +- Keywords: "tutor", "tutoring", "lessons for", "homework help", "private tutor", "extra classes" +- Request terms: same as website category — must co-occur with a target keyword +- Strict reject: people offering tutoring, educational products, homeschool programs, free trials, general study tips + +### Multi-Language Support +Searches in 4 South African languages: +- English — 1 primary + 2-3 supplementary queries +- Afrikaans — 2 queries (e.g., "ek benodig n webwerf", "ek soek n privaat onderwyser") +- isiXhosa — 2 queries (e.g., "ndidinga iwebhusayithi yeshishini", "ndifuna utitshala womntwana wam") +- isiZulu — 2 queries (e.g., "ngidinga iwebhusayithi yebhizinisi", "ngifuna umfundisi wengane") + +### Output Format +Each lead returned includes: +- `title` — post preview text +- `author` — poster's name (may include location in name) +- `content` — extracted post text +- `url` — direct link to the post +- `date` — when posted (filtered within 7 days) +- `category` — "website" or "tutor" + +Target is 5-10 dead-accurate leads per scrape. Quality over quantity — no loose padding. + +### Configuration via Env Vars +- `SELECTED_BROWSER` — `firefox` (default), `chrome`, `opera`, `edge`, or `auto` +- `FX_PROFILE`, `CHROME_PROFILE`, `OPERA_PROFILE`, `EDGE_PROFILE` — browser profile paths +- `AI_PORT`, `AI_HOST` — AI server bind (default `3001`, `0.0.0.0`) +- `SCRAPER_URL` — scraper URL (default `http://127.0.0.1:3008`) +- `FRONTEND_URL` — frontend URL (default `http://127.0.0.1:3006`) +- `NEXT_PUBLIC_SCRAPER_URL` — frontend-facing scraper URL +- `OLLAMA_BASE_URL` — Ollama URL (default `http://localhost:11434`) +- `AI_MODEL` — Ollama model (default `llama3.2:3b`) +- `CLASSIFY_MODEL` — model for lead classification (default `dolphin-llama3:8b`) + +## How to Start Scraping +1. Ensure all 3 services are running (ports 3001, 3006, 3008) and Ollama is on 11434 +2. Open the frontend at `http://localhost:3006` +3. Select a job category (Website Creation or Tutoring) +4. Click "Search Facebook" — the scraper runs and returns leads +5. Leads are saved in the CRM for follow-up + +## Sales Tips +- Cold emails should be under 150 words +- Follow up within 48 hours +- Personalise every outreach with the prospect's name and company +- Use open-ended questions in discovery calls +- Always ask for the next step before ending a call +- For website leads: mention specific pages or features they requested +- For tutoring leads: reference the subject and age group they mentioned + +## Job Targeting +- Developers respond best to technical value props +- Marketing managers care about ROI and metrics +- C-level executives want brevity and business impact +- Parents hiring tutors: empathy and qualifications matter most ## Response Rules - Be direct and actionable — no fluff, no AI disclaimers @@ -41,3 +120,14 @@ Proxies and user agents are defined as constants at the top of `main.rs`. - Never mention being an AI or language model - If you don't know something, say so honestly - Prioritise the user's role: salespeople need speed, admins need control +- When asked about scraping, give specific guidance on categories and languages + +## Self-Improvement Protocol +1. You notice a gap in your knowledge or a pattern in user questions +2. You call `POST /ai/instructions` with: + - `entry`: description of the improvement + - `content`: optional full replacement of ai.md +3. The improvement is logged and loaded into the next system prompt + +## Improvement Log +- (2026-07-07) Initial rewrite: full architecture, scraper details, multi-language, lead categories, env vars diff --git a/rust-ai/src/main.rs b/rust-ai/src/main.rs index ef33b3f..c01a4f8 100644 --- a/rust-ai/src/main.rs +++ b/rust-ai/src/main.rs @@ -1,6 +1,6 @@ use axum::{ extract::State, - http::{HeaderMap, Method, StatusCode}, + http::{HeaderMap, HeaderValue, Method, StatusCode}, routing::{get, post}, Json, Router, }; @@ -482,11 +482,12 @@ async fn main() { rate_limiter: RateLimiter::new(30, 60), }); + let cors_origins_env = std::env::var("CORS_ORIGINS").unwrap_or_else(|_| "http://localhost:3006,http://127.0.0.1:3006".to_string()); + let cors_origins: Vec = cors_origins_env.split(',') + .filter_map(|o| { let t = o.trim(); if t.is_empty() { None } else { t.parse().ok() } }) + .collect(); let cors = CorsLayer::new() - .allow_origin(AllowOrigin::list([ - "http://localhost:3006".parse().unwrap(), - "http://127.0.0.1:3006".parse().unwrap(), - ])) + .allow_origin(AllowOrigin::list(cors_origins)) .allow_methods([Method::GET, Method::POST]) .allow_headers(Any); @@ -506,7 +507,7 @@ async fn main() { let bg_leads = lead_store.clone(); let bg_db = state.db.clone(); - let bg_url = "http://localhost:3008/scrape/facebook".to_string(); + let bg_url = std::env::var("SCRAPER_URL").unwrap_or_else(|_| "http://localhost:3008".to_string()) + "/scrape/facebook"; tokio::spawn(async move { let client = match reqwest::Client::builder() .timeout(Duration::from_secs(300)) diff --git a/src/app/(dashboard)/ai-assistant/page.tsx b/src/app/(dashboard)/ai-assistant/page.tsx index 83597cb..ea2e5cf 100644 --- a/src/app/(dashboard)/ai-assistant/page.tsx +++ b/src/app/(dashboard)/ai-assistant/page.tsx @@ -26,8 +26,10 @@ export default function AIAssistantPage() { aiChatRef.current?.addAssistantMessage("⏳ Still searching Facebook (this can take up to 5 minutes)...") }, 45000) + const scrapBase = process.env.NEXT_PUBLIC_SCRAPER_URL || "http://localhost:3008" + try { - const res = await fetch(`http://localhost:3008/scrape/facebook?force=true&query=${encodeURIComponent(keyword)}`, { method: "POST", signal: controller.signal }) + const res = await fetch(`${scrapBase}/scrape/facebook?force=true&query=${encodeURIComponent(keyword)}`, { method: "POST", signal: controller.signal }) clearTimeout(timeoutId) clearTimeout(statusId) const data = await res.json()