Added 4 Languages English, Afrikaans, Xhosa, Zulu, tested but brought leads down to 3 leads only will see how to make it more without losing effiency of the model.
This commit is contained in:
+256
-67
@@ -34,7 +34,7 @@ logger = logging.getLogger(__name__)
|
||||
app = FastAPI()
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
allow_origins=["http://localhost:3006", "http://127.0.0.1:3006"],
|
||||
allow_origins=os.getenv("CORS_ORIGINS", "http://localhost:3006,http://127.0.0.1:3006").split(","),
|
||||
allow_methods=["POST"],
|
||||
allow_headers=["*"],
|
||||
)
|
||||
@@ -674,6 +674,107 @@ def _search_list_for_query(query: str) -> list[str]:
|
||||
return TUTORING_SEARCHES
|
||||
return FB_SEARCHES
|
||||
|
||||
# ── South African Multi-Language Queries ──────────────────────────────
|
||||
# 4 most spoken SA languages: English, Afrikaans, isiXhosa, isiZulu.
|
||||
# Each scrape searches ALL 4 languages to catch leads across all
|
||||
# language communities on Facebook.
|
||||
|
||||
SA_WEBSITE_QUERIES = [
|
||||
"I need a website for my business", # English
|
||||
"ek benodig n webwerf", # Afrikaans
|
||||
"ek soek iemand om n webwerf te bou", # Afrikaans
|
||||
"ndidinga iwebhusayithi yeshishini", # isiXhosa
|
||||
"ndifuna umntu owakha iwebhusayithi", # isiXhosa
|
||||
"ngidinga iwebhusayithi yebhizinisi", # isiZulu
|
||||
"ngifuna umuntu owakha iwebhusayithi", # isiZulu
|
||||
]
|
||||
|
||||
SA_TUTOR_QUERIES = [
|
||||
"I need a tutor for my child", # English
|
||||
"ek benodig n tutor vir my kind", # Afrikaans
|
||||
"ek soek n privaat onderwyser", # Afrikaans
|
||||
"ndifuna utitshala womntwana wam", # isiXhosa
|
||||
"ndidinga umfundisi-ntsapho", # isiXhosa
|
||||
"ngidinga uthisha wengane yami", # isiZulu
|
||||
"ngifuna umfundisi wengane", # isiZulu
|
||||
]
|
||||
|
||||
|
||||
async def _quick_search(page, context, query: str) -> tuple:
|
||||
"""Fast search — load search results page, wait for render, extract visible posts.
|
||||
No scrolling or extra human-like delays. Used for non-English language queries."""
|
||||
page = await _ensure_page(page, context)
|
||||
url = f'https://www.facebook.com/search/posts/?q={urllib.parse.quote(query)}'
|
||||
try:
|
||||
await page.goto(url, wait_until='domcontentloaded', timeout=20000)
|
||||
current_url = page.url
|
||||
if '/login' in current_url.lower():
|
||||
logger.warning("Quick search redirected to login for '%s'", query[:40])
|
||||
return page, []
|
||||
await page.wait_for_timeout(random.randint(4000, 6000))
|
||||
await page.evaluate("window.scrollBy(0, 600)")
|
||||
await page.wait_for_timeout(random.randint(2000, 4000))
|
||||
await page.evaluate("window.scrollBy(0, 600)")
|
||||
await page.wait_for_timeout(random.randint(2000, 3000))
|
||||
raw_articles = await _get_article_elements(page)
|
||||
posts = _extract_posts_from_elements(raw_articles, url) if raw_articles else []
|
||||
raw = await page.evaluate('document.body.innerText')
|
||||
text_posts = _extract_posts_from_text(raw, url)
|
||||
existing = {(p.get('title') or p.get('content',''))[:80] for p in posts}
|
||||
for tp in text_posts:
|
||||
key = (tp.get('title') or tp.get('content',''))[:80]
|
||||
if key not in existing:
|
||||
posts.append(tp)
|
||||
if posts:
|
||||
try:
|
||||
profiles = await page.evaluate(r'''() => {
|
||||
const out = [];
|
||||
const seenTxt = new Set();
|
||||
for (const a of document.querySelectorAll('a[href*="/profile.php"], a[href*="/user/"], a[href*="/people/"], a[href*="/me/"]')) {
|
||||
const name = (a.innerText || '').trim();
|
||||
if (!name || name.length < 3 || name.length > 60) continue;
|
||||
const words = name.split(' ');
|
||||
if (words.length < 2 || words.length > 6) continue;
|
||||
if (!/^[A-Z]/.test(name)) continue;
|
||||
if (name.includes('facebook') || name.includes('/')) continue;
|
||||
const cell = a.closest('div[style]') || a.parentElement;
|
||||
const txt = cell ? (cell.innerText || '').substring(0, 200) : '';
|
||||
if (!txt) continue;
|
||||
const key2 = txt.substring(0, 80);
|
||||
if (seenTxt.has(key2)) continue;
|
||||
seenTxt.add(key2);
|
||||
out.push({ name, textKey: key2 });
|
||||
}
|
||||
return out;
|
||||
}''')
|
||||
if profiles:
|
||||
for p in posts:
|
||||
pk = (p.get('content') or p.get('title') or '')[:80].strip()
|
||||
if not pk: continue
|
||||
for pr in profiles:
|
||||
if pk[:30] in pr['textKey'] or pr['textKey'][:30] in pk:
|
||||
if not p.get('author'):
|
||||
p['author'] = pr['name']
|
||||
break
|
||||
except Exception:
|
||||
pass
|
||||
for p in posts:
|
||||
if p.get('author'):
|
||||
a = p['author']
|
||||
al = a.lower()
|
||||
if any(kw in al for kw in BROAD_KEYWORDS) or is_offer(a) or len(a.split()) < 2 or any(w in _NON_NAMES for w in al.split()):
|
||||
p['author'] = ''
|
||||
posts = [p for p in posts if not (
|
||||
'/groups/' in p.get('url', '') or '/group/' in p.get('url', '')
|
||||
or '/pages/' in p.get('url', '')
|
||||
or ' / ' in (p.get('title') or p.get('content') or '')
|
||||
)]
|
||||
except Exception as e:
|
||||
logger.warning("Quick search '%s' failed: %s", query, e)
|
||||
return page, []
|
||||
return page, posts
|
||||
|
||||
|
||||
VIEWPORTS = [
|
||||
{'width': 1280, 'height': 800},
|
||||
{'width': 1366, 'height': 768},
|
||||
@@ -1383,16 +1484,27 @@ async def _scrape_with_firefox(profile_path: str, force: bool, query: str | None
|
||||
return {"success": True, "leads": [], "flagged": False, "flag_reason": None, "error": None}
|
||||
|
||||
all_posts = []
|
||||
tutoring = False
|
||||
if query:
|
||||
tl = query.lower()
|
||||
tutoring = any(t in tl for t in ["tutor", "tutoring", "lessons", "homework", "teach", "learning", "child"])
|
||||
lang_pool = SA_TUTOR_QUERIES if tutoring else SA_WEBSITE_QUERIES
|
||||
non_english = [q for q in lang_pool if q.strip().lower() != tl]
|
||||
query_pool = _search_list_for_query(query)
|
||||
searches = [query] + random.sample(query_pool, k=random.randint(1, 2))
|
||||
supp_k = random.randint(3, 4) if tutoring else random.randint(2, 3)
|
||||
supplement = random.sample(query_pool, k=supp_k)
|
||||
searches = [query] + supplement + non_english
|
||||
english_count = 1 + len(supplement)
|
||||
else:
|
||||
searches = random.sample(FB_SEARCHES, k=random.randint(2, 4))
|
||||
searches = random.sample(FB_SEARCHES + SA_WEBSITE_QUERIES + SA_TUTOR_QUERIES, k=random.randint(4, 7))
|
||||
for i, sq in enumerate(searches):
|
||||
page, posts = await search_facebook(page, context, sq)
|
||||
page, posts = await search_facebook(page, context, sq) if i == 0 else await _quick_search(page, context, sq)
|
||||
all_posts.extend(posts)
|
||||
if not posts:
|
||||
continue
|
||||
if i > 0:
|
||||
await page.wait_for_timeout(random.uniform(3000, 7000))
|
||||
continue
|
||||
if random.random() < 0.4:
|
||||
await page.evaluate(f"window.scrollBy(0, {random.randint(-300, 300)})")
|
||||
delay = random.uniform(8, 25)
|
||||
@@ -1421,11 +1533,11 @@ async def _scrape_with_firefox(profile_path: str, force: bool, query: str | None
|
||||
deduped.append(p)
|
||||
|
||||
# Filter to last 3 days only
|
||||
deduped = [p for p in deduped if _is_within_days(p.get('date', ''), 3)]
|
||||
deduped = [p for p in deduped if _is_within_days(p.get('date', ''), 7)]
|
||||
|
||||
leads = deduped[:20]
|
||||
if leads:
|
||||
leads = await classify_leads(leads)
|
||||
leads = await classify_leads(leads, tutoring=tutoring)
|
||||
|
||||
return {"success": True, "leads": leads[:15], "flagged": False, "flag_reason": None, "error": None}
|
||||
|
||||
@@ -1518,16 +1630,27 @@ async def _scrape_with_chromium(profile_path: str, browser: str, force: bool = F
|
||||
return {"success": True, "leads": [], "flagged": False, "flag_reason": None, "error": None}
|
||||
|
||||
all_posts = []
|
||||
tutoring = False
|
||||
if query:
|
||||
tl = query.lower()
|
||||
tutoring = any(t in tl for t in ["tutor", "tutoring", "lessons", "homework", "teach", "learning", "child"])
|
||||
lang_pool = SA_TUTOR_QUERIES if tutoring else SA_WEBSITE_QUERIES
|
||||
non_english = [q for q in lang_pool if q.strip().lower() != tl]
|
||||
query_pool = _search_list_for_query(query)
|
||||
searches = [query] + random.sample(query_pool, k=random.randint(1, 2))
|
||||
supp_k = random.randint(3, 4) if tutoring else random.randint(2, 3)
|
||||
supplement = random.sample(query_pool, k=supp_k)
|
||||
searches = [query] + supplement + non_english
|
||||
english_count = 1 + len(supplement)
|
||||
else:
|
||||
searches = random.sample(FB_SEARCHES, k=random.randint(2, 4))
|
||||
searches = random.sample(FB_SEARCHES + SA_WEBSITE_QUERIES + SA_TUTOR_QUERIES, k=random.randint(4, 7))
|
||||
for i, sq in enumerate(searches):
|
||||
page, posts = await search_facebook(page, context, sq)
|
||||
page, posts = await search_facebook(page, context, sq) if i == 0 else await _quick_search(page, context, sq)
|
||||
all_posts.extend(posts)
|
||||
if not posts:
|
||||
continue
|
||||
if i > 0:
|
||||
await page.wait_for_timeout(random.uniform(3000, 7000))
|
||||
continue
|
||||
if random.random() < 0.4:
|
||||
await page.evaluate(f"window.scrollBy(0, {random.randint(-300, 300)})")
|
||||
delay = random.uniform(8, 25)
|
||||
@@ -1555,10 +1678,10 @@ async def _scrape_with_chromium(profile_path: str, browser: str, force: bool = F
|
||||
seen.add(key)
|
||||
deduped.append(p)
|
||||
|
||||
deduped = [p for p in deduped if _is_within_days(p.get('date', ''), 3)]
|
||||
deduped = [p for p in deduped if _is_within_days(p.get('date', ''), 7)]
|
||||
leads = deduped[:20]
|
||||
if leads:
|
||||
leads = await classify_leads(leads)
|
||||
leads = await classify_leads(leads, tutoring=tutoring)
|
||||
|
||||
return {"success": True, "leads": leads[:15], "flagged": False, "flag_reason": None, "error": None}
|
||||
|
||||
@@ -1599,7 +1722,8 @@ async def _scrape_with_agent(force: bool = False) -> dict:
|
||||
await browser.start()
|
||||
|
||||
all_posts = []
|
||||
for query in random.sample(FB_SEARCHES, k=random.randint(2, 4)):
|
||||
pool = FB_SEARCHES + random.sample(SA_WEBSITE_QUERIES, k=min(4, len(SA_WEBSITE_QUERIES)))
|
||||
for query in random.sample(pool, k=random.randint(2, 4)):
|
||||
agent = _make_agent(
|
||||
task=f"""You are logged into Facebook. Do the following:
|
||||
1. Navigate to facebook.com and make sure you are on the homepage
|
||||
@@ -1640,11 +1764,11 @@ When done, return the data as a JSON list with keys: content, author, url, date.
|
||||
deduped.append(p)
|
||||
|
||||
# Filter to last 3 days only
|
||||
deduped = [p for p in deduped if _is_within_days(p.get('date', ''), 3)]
|
||||
deduped = [p for p in deduped if _is_within_days(p.get('date', ''), 7)]
|
||||
|
||||
leads = deduped[:20]
|
||||
if leads:
|
||||
leads = await classify_leads(leads)
|
||||
leads = await classify_leads(leads, tutoring=tutoring)
|
||||
|
||||
return {"success": True, "leads": leads[:15], "flagged": False, "flag_reason": None, "error": None}
|
||||
except Exception as e:
|
||||
@@ -1679,24 +1803,37 @@ async def ask_ollama(prompt: str) -> str:
|
||||
data = r.json()
|
||||
return data["message"]["content"]
|
||||
|
||||
async def classify_leads(results: list[dict]) -> list[dict]:
|
||||
async def classify_leads(results: list[dict], tutoring: bool = False) -> list[dict]:
|
||||
if not results:
|
||||
return []
|
||||
|
||||
# ── 1. AI classification ─────────────────────────────────────────
|
||||
briefs = [r["title"][:200] for r in results]
|
||||
if tutoring:
|
||||
lead_desc = "someone REQUESTING/LOOKING FOR/WANTING a tutor, teacher, or lessons for their child or themselves"
|
||||
lead_examples = '"Looking for a tutor for my child", "Need a math tutor for my son", "Need help with homework", "Looking for piano lessons for my daughter", "Need a reading tutor"'
|
||||
not_lead_examples = '"I offer tutoring services", "I am a tutor with experience", "Affordable tutoring packages", "Online tutor available"'
|
||||
extra_terms = '- Posts about homeschooling resources, curriculum sales, or educational products\n- Posts asking for study tips or general academic advice without requesting a tutor'
|
||||
else:
|
||||
lead_desc = "someone REQUESTING/POSTING/WANTING a website built, designed, or created for them"
|
||||
lead_examples = '"Need a website for my business", "Looking for web developer to build my site", "I need someone to create my website", "Want a new website for my company", "Looking for someone to design my WordPress site"'
|
||||
not_lead_examples = '"I build websites", "I offer web design", "Affordable web design packages"'
|
||||
extra_terms = '- "Need web hosting", "Looking for a partner", "Looking for content writer", "Video spokesperson"'
|
||||
prompt = f"""Classify each post as LEAD or NOT.
|
||||
LEAD = someone REQUESTING/POSTING/WANTING a website built, designed, or created for them.
|
||||
LEAD examples: "Need a website for my business", "Looking for web developer to build my site", "I need someone to create my website", "Want a new website for my company", "Looking for someone to design my WordPress site"
|
||||
LEAD = {lead_desc}.
|
||||
LEAD examples: {lead_examples}
|
||||
|
||||
NOT LEAD:
|
||||
- Offering web design services: "I build websites", "I offer web design", "Affordable web design packages"
|
||||
- Offering services: {not_lead_examples}
|
||||
- Already have a website and need marketing, SEO, content, video, link building, email marketing, affiliates
|
||||
- Recruiting employees, hiring staff, looking for business partners
|
||||
- Selling products, promoting services, affiliate offers
|
||||
- "Need web hosting", "Looking for a partner", "Looking for content writer", "Video spokesperson"
|
||||
{extra_terms}
|
||||
- Posts from groups, communities, or pages (group announcements, group posts, page posts)
|
||||
- Posts containing the word "group", "page", "community", "creators" — these are NEVER individual leads
|
||||
- Vague questions or general recommendations without a clear intent to buy or hire
|
||||
- People asking how to learn or do it themselves (not looking to hire someone)
|
||||
- Posts about existing website issues like speed, SEO, errors, redesign advice — NOT a lead
|
||||
|
||||
For each numbered post, answer ONLY "yes" (LEAD) or "no" (NOT LEAD):
|
||||
{chr(10).join(f'{i+1}. {t}' for i, t in enumerate(briefs))}
|
||||
@@ -1721,32 +1858,70 @@ Return a JSON array like ["yes","no","yes"] matching the order above."""
|
||||
except Exception as e:
|
||||
logger.warning("AI classification failed: %s", e)
|
||||
|
||||
# ── 2. Keyword fallback (always runs) ────────────────────────────
|
||||
web_terms = [
|
||||
"website", "web design", "web develop", "web dev",
|
||||
"web designer", "web developer",
|
||||
"build my website", "build a website", "create a website",
|
||||
"landing page", "wordpress", "ecommerce",
|
||||
"my website", "business website",
|
||||
"site for my", "site for my business",
|
||||
"new website", "redesign my website",
|
||||
"help with my website", "update my website",
|
||||
"make a website", "make my website",
|
||||
"website for my",
|
||||
"online store", "online shop",
|
||||
"build my site", "build a site",
|
||||
"set up a website", "set up my website",
|
||||
"custom website",
|
||||
"shopify",
|
||||
"my site",
|
||||
"webpage", "web page",
|
||||
]
|
||||
# ── 2. Keyword supplement (never overrides AI, only adds missing leads) ──
|
||||
if tutoring:
|
||||
target_terms = [
|
||||
"tutor", "tutoring", "tutor for", "private tutor",
|
||||
"math tutor", "english tutor", "reading tutor",
|
||||
"science tutor", "online tutor", "home tutor",
|
||||
"lessons for", "lessons for my", "piano lessons",
|
||||
"swimming lessons", "music lessons",
|
||||
"help with homework", "homework help",
|
||||
"teacher for", "teacher for my",
|
||||
"need help learning", "need help with",
|
||||
"exam prep", "exam preparation",
|
||||
"homeschool", "homeschool tutor",
|
||||
"tuition",
|
||||
"coding for my", "programming for my",
|
||||
"looking for a tutor", "need a tutor",
|
||||
"tutor needed", "tutoring for",
|
||||
"private lessons", "private tuition",
|
||||
"afterschool", "after school",
|
||||
"extra classes", "extra lessons",
|
||||
]
|
||||
offer_reject_tutor = [
|
||||
'i am a tutor', "i'm a tutor", 'i offer tutoring',
|
||||
'online tutor available', 'tutor available',
|
||||
'i teach', 'i provide tutoring',
|
||||
'affordable tutoring', 'tutoring services',
|
||||
'experienced tutor', 'qualified tutor',
|
||||
'your child', 'your kids', 'your children',
|
||||
'enroll your', 'sign up',
|
||||
'free trial', 'first lesson free',
|
||||
'group lessons', 'group class',
|
||||
'limited spots', 'book now',
|
||||
'curriculum', 'workbook', 'worksheet',
|
||||
'educational program',
|
||||
'homeschool program', 'home school program',
|
||||
]
|
||||
else:
|
||||
target_terms = [
|
||||
"website", "web design", "web develop", "web dev",
|
||||
"web designer", "web developer",
|
||||
"build my website", "build a website", "create a website",
|
||||
"landing page", "wordpress", "ecommerce",
|
||||
"my website", "business website",
|
||||
"site for my", "site for my business",
|
||||
"new website", "redesign my website",
|
||||
"help with my website", "update my website",
|
||||
"make a website", "make my website",
|
||||
"website for my",
|
||||
"online store", "online shop",
|
||||
"build my site", "build a site",
|
||||
"set up a website", "set up my website",
|
||||
"custom website",
|
||||
"shopify",
|
||||
"my site",
|
||||
"webpage", "web page",
|
||||
"who can build", "who can design",
|
||||
"create my website", "create my site",
|
||||
]
|
||||
offer_reject_tutor = []
|
||||
request_terms = [
|
||||
"looking for", "need a", "need an", "looking to",
|
||||
"need someone", "hire a", "want someone",
|
||||
"need help with", "would like", "build me",
|
||||
"design my", "make me a", "create my",
|
||||
"looking", "need", "want", "help",
|
||||
"who can", "i need",
|
||||
"recommend", "anyone know", "anyone recommend",
|
||||
"know a", "know any", "recommendation",
|
||||
@@ -1768,27 +1943,29 @@ Return a JSON array like ["yes","no","yes"] matching the order above."""
|
||||
'whatsapp me', 'looking for a business', 'looking for client',
|
||||
'help your business', 'i am a web', 'contact me',
|
||||
'we offer web', 'we provide web',
|
||||
'take the quiz', 'homeschool', 'your home tutor',
|
||||
'take the quiz',
|
||||
'link in bio', 'apply now', 'get started',
|
||||
'for only', 'low price', 'hit me up',
|
||||
'send me a message', 'i do website', 'we do website',
|
||||
'we do web', 'i do web',
|
||||
'website designer / web developer', 'website & software creators',
|
||||
'website builders for small businesses', 'australia web designers',
|
||||
'south africa', 'wix website design',
|
||||
'website builders for small businesses',
|
||||
'wix website design',
|
||||
'for sale', 'selling my', 'premium',
|
||||
'i\'m selling', 'i\'m offering', 'we\'re offering',
|
||||
'free ecommerce', 'free website design',
|
||||
'starting a', 'looking for a few businesses',
|
||||
# Group-related rejections
|
||||
'group', ' i need a website group', 'south africa web', 'philippines web', 'australia web',
|
||||
'i can help', 'inbox me', 'dm me', 'pm me', 'message me for',
|
||||
'group', ' i need a website group',
|
||||
'i can help', 'inbox me', 'message me for',
|
||||
'best price', 'discount', 'reach out', 'check out my', 'check this',
|
||||
'website for your', 'price start', 'price begin', 'website creator',
|
||||
'website & software', 'creators &', 'creators marketplace',
|
||||
'website group', 'page group',
|
||||
'south africa web', 'philippines web', 'australia web',
|
||||
'nigerian web', 'kenya web', 'india web',
|
||||
# Self-promotion rejections
|
||||
'i\'m a web', "i'm a web", 'i am a full stack', "i'm a full stack", 'i\'m a full stack',
|
||||
'i\'m a web', "i'm a web", 'i am a full stack', "i'm a full stack",
|
||||
'freelance opportunity', 'looking for new project', 'looking for new work',
|
||||
'full stack web', 'mern stack', 'responsive business website',
|
||||
'i build website', 'i build shopify', 'i build wordpress',
|
||||
@@ -1802,18 +1979,48 @@ Return a JSON array like ["yes","no","yes"] matching the order above."""
|
||||
'for free', 'no coding', 'make money', 'website for free',
|
||||
'part time job', 'part time position',
|
||||
'years of experience', 'years of teaching',
|
||||
# Service offers that slip through two-word check
|
||||
'i am a full stack', 'i am a developer',
|
||||
'i will design', 'i will build', 'i will create',
|
||||
'i can design', 'i can create',
|
||||
'we will design', 'we will build',
|
||||
'hire me', 'i am available for',
|
||||
'available for work', 'freelance web',
|
||||
'i specialize in', 'we specialize in',
|
||||
"here's my portfolio", 'check my portfolio',
|
||||
'see my work', 'view my work',
|
||||
'we have a team', 'my team',
|
||||
'i am looking for clients', 'i am looking for work',
|
||||
'looking for web development work',
|
||||
'looking for new clients',
|
||||
# People learning / doing it themselves (not hiring)
|
||||
'learn web development', 'learn to code',
|
||||
'how to build a website', 'how to create a website',
|
||||
'how to make a website', 'how to design a website',
|
||||
'where to start', 'online course',
|
||||
'want to learn', 'learning web',
|
||||
'best platform for', 'which platform',
|
||||
# Existing website issues (not new build)
|
||||
'my website is down', 'website not loading',
|
||||
'website error', 'website problem',
|
||||
'website troubleshooting',
|
||||
'need website advice', 'website tips',
|
||||
'help with seo', 'google ranking',
|
||||
'website design ideas', 'website inspiration',
|
||||
]
|
||||
for r in results:
|
||||
t = r['title'].lower()
|
||||
has_web = any(kw in t for kw in web_terms)
|
||||
has_target = any(kw in t for kw in target_terms)
|
||||
has_request = any(kw in t for kw in request_terms)
|
||||
if not has_web or not has_request:
|
||||
if not has_target or not has_request:
|
||||
continue
|
||||
if any(kw in t for kw in offer_reject):
|
||||
continue
|
||||
if any(kw in t for kw in offer_reject_tutor):
|
||||
continue
|
||||
keyword_leads.append(r)
|
||||
|
||||
# ── 3. Merge: prefer AI leads, supplement with keywords to reach 5 ──
|
||||
# ── 3. Merge: prefer AI leads, supplement with keywords ──
|
||||
seen_titles: set[int] = set()
|
||||
merged: list[dict] = []
|
||||
for r in ai_leads + keyword_leads:
|
||||
@@ -1826,24 +2033,6 @@ Return a JSON array like ["yes","no","yes"] matching the order above."""
|
||||
merged = [r for r in merged if not any(kw in (r.get('title','') or '').lower() for kw in offer_reject)]
|
||||
merged = [r for r in merged if not any(gw in (r.get('title','') or '').lower() for gw in group_words)]
|
||||
|
||||
# Fill to 5 with loose keyword matches (at least web OR request term)
|
||||
if len(merged) < 5:
|
||||
for r in results:
|
||||
key = hash(r.get('title', ''))
|
||||
if key in seen_titles:
|
||||
continue
|
||||
t = r['title'].lower()
|
||||
if not (any(kw in t for kw in web_terms) or any(kw in t for kw in request_terms)):
|
||||
continue
|
||||
if any(kw in t for kw in offer_reject):
|
||||
continue
|
||||
if any(gw in t for gw in group_words):
|
||||
continue
|
||||
seen_titles.add(key)
|
||||
merged.append(r)
|
||||
if len(merged) >= 5:
|
||||
break
|
||||
|
||||
logger.info("classify_leads: %d merged (%d AI + %d keyword) from %d raw", len(merged), len(ai_leads), len(keyword_leads), len(results))
|
||||
return merged[:10]
|
||||
|
||||
|
||||
Reference in New Issue
Block a user