CRM_ENVIROMENT/CRM_ENVR
7
0
Fork 0
mirror of https://git.coastit.co.za/caitlin/CRM_ENVR.git synced 2026-07-10 11:15:43 +02:00
Code Issues Packages Projects Releases Wiki Activity

feat: add Facebook account tracking and management

Browse Source 
- Introduced new migration for `facebook_accounts` and `facebook_scrape_logs` tables.
- Updated the scraping logic to handle Facebook accounts, including success and failure tracking.
- Implemented API endpoints for managing Facebook accounts and their scrape logs.
- Enhanced user permissions to restrict access to Facebook account management to admins and super admins.
- Added a dialog component for displaying and managing Facebook accounts in the UI.
- Updated lead fetching logic to include user role checks for assignment and access.
This commit is contained in:
Ace
2026-06-23 14:18:18 +02:00
parent 1adc4806fa
commit ff56cea4b8
25 changed files with 778 additions and 216 deletions
Download Patch File Download Diff File Expand all files Collapse all files
browser-use-service/main.py
+169 -134
View File
@@ -1,9 +1,7 @@
import os, json, asyncio, re, shutil, sqlite3, traceback, urllib.parse, random, time, logging
import os, json, asyncio, re, shutil, sqlite3, urllib.parse, random, logging
from datetime import datetime, timedelta
from bs4 import BeautifulSoup
import httpx
from fastapi import FastAPI
from pydantic import BaseModel
from fastapi import FastAPI, Query
from fastapi.middleware.cors import CORSMiddleware
import uvicorn
from playwright.async_api import async_playwright
@@ -11,22 +9,20 @@ logging.basicConfig(level=logging.INFO, format='%(asctime)s [%(levelname)s] %(me
logger = logging.getLogger(__name__)
app = FastAPI()
app.add_middleware(
CORSMiddleware,
allow_origins=["http://localhost:3006", "http://127.0.0.1:3006"],
allow_methods=["POST"],
allow_headers=["*"],
)
PORT = int(os.getenv("PORT", "3008"))
OLLAMA_URL = os.getenv("OLLAMA_URL", "http://localhost:11434")
CLASSIFY_MODEL = os.getenv("CLASSIFY_MODEL", "dolphin-llama3:8b")
FX_PROFILE = os.getenv('FX_PROFILE', '')
FX_COOKIE_DB = os.path.join(FX_PROFILE, 'cookies.sqlite') if FX_PROFILE else ''
HEADERS = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
}
last_scrape_time: datetime | None = None
STRICT_KEYWORDS = [
BROAD_KEYWORDS = [
"website", "web design", "web develop", "web dev",
"build my website", "build a website", "create a website",
"need web", "looking for web", "new website",
@@ -34,9 +30,6 @@ STRICT_KEYWORDS = [
"need a website", "my website", "business website",
"need a designer", "help with my website",
"redesign", "update my website",
]
BROAD_KEYWORDS = STRICT_KEYWORDS + [
"looking for", "need a", "need an", "looking to",
"help me build", "create my", "build me",
"design my", "make me a", "would like",
@@ -44,10 +37,6 @@ BROAD_KEYWORDS = STRICT_KEYWORDS + [
"want someone", "need help with",
]
def kw_match_strict(text: str) -> bool:
t = text.lower()
return any(kw in t for kw in STRICT_KEYWORDS)
def kw_match(text: str) -> bool:
t = text.lower()
return any(kw in t for kw in BROAD_KEYWORDS)
@@ -67,14 +56,27 @@ FB_SEARCHES = [
"need a site for my business",
]
async def get_fb_cookies():
if not FX_COOKIE_DB or not os.path.exists(FX_COOKIE_DB):
logger.warning("FX_COOKIE_DB not found or FX_PROFILE not set")
VIEWPORTS = [
{'width': 1280, 'height': 800},
{'width': 1366, 'height': 768},
{'width': 1440, 'height': 900},
{'width': 1536, 'height': 864},
{'width': 1920, 'height': 1080},
]
async def get_fb_cookies(profile_path: str | None = None):
cookie_db_path = profile_path or FX_PROFILE
if not cookie_db_path:
logger.warning("No profile path provided")
return []
tmp = os.path.join(os.path.dirname(FX_COOKIE_DB), f'cookies_tmp_{os.getpid()}.sqlite')
cookie_db = os.path.join(cookie_db_path, 'cookies.sqlite')
if not os.path.exists(cookie_db):
logger.warning("Cookie DB not found at %s", cookie_db)
return []
tmp = os.path.join(os.path.dirname(cookie_db), f'cookies_tmp_{os.getpid()}.sqlite')
for attempt in range(3):
try:
shutil.copy2(FX_COOKIE_DB, tmp)
shutil.copy2(cookie_db, tmp)
break
except PermissionError:
if attempt < 2:
@@ -182,35 +184,95 @@ def _extract_posts_from_text(raw: str, url: str) -> list[dict]:
})
return posts
async def human_scroll(page, steps: int = None, total_delay: float = None):
steps = steps or random.randint(2, 5)
total_delay = total_delay or random.uniform(6, 18)
step_delay = total_delay / steps
for i in range(steps):
scroll_dist = random.randint(200, 600)
await page.evaluate(f"window.scrollBy(0, {scroll_dist})")
await page.wait_for_timeout(int(step_delay * 1000))
if random.random() < 0.3:
await page.evaluate(f"window.scrollBy(0, -{random.randint(100, 300)})")
await page.wait_for_timeout(random.randint(1000, 3000))
async def random_idle(page):
action = random.random()
if action < 0.15:
try:
elems = await page.query_selector_all('a, span, div[role="button"]')
if elems:
el = random.choice(elems[:20])
box = await el.bounding_box()
if box:
x = box['x'] + box['width'] / 2 + random.randint(-20, 20)
y = box['y'] + box['height'] / 2 + random.randint(-10, 10)
await page.mouse.move(x, y, steps=random.randint(20, 40))
await page.wait_for_timeout(random.randint(500, 2000))
except Exception:
pass
async def search_facebook(page, query: str) -> list[dict]:
url = f'https://www.facebook.com/search/posts/?q={urllib.parse.quote(query)}'
try:
await page.goto(url, wait_until='domcontentloaded', timeout=30000)
await page.wait_for_timeout(6000)
except Exception as e:
logger.warning("Facebook search navigation failed for '%s': %s", query, e)
return []
await page.wait_for_timeout(random.randint(3000, 8000))
await human_scroll(page, steps=random.randint(2, 4), total_delay=random.uniform(6, 15))
# Accidental overscroll: 10% chance to jump to bottom and back
if random.random() < 0.1:
await page.evaluate("window.scrollTo(0, document.body.scrollHeight)")
await page.wait_for_timeout(random.randint(1000, 3000))
await page.evaluate("window.scrollBy(0, -300)")
await page.wait_for_timeout(random.randint(1000, 3000))
if random.random() < 0.2:
await page.evaluate("window.scrollTo(0, 0)")
await page.wait_for_timeout(random.randint(2000, 5000))
if random.random() < 0.3:
await random_idle(page)
try:
raw = await page.evaluate('document.body.innerText')
except Exception as e:
logger.warning("Failed to evaluate page text: %s", e)
logger.warning("Facebook search failed: %s", e)
return []
return _extract_posts_from_text(raw, url)
async def scrape_facebook() -> list[dict]:
all_posts = []
fb_cookies = await get_fb_cookies()
def cleanup_chrome():
import subprocess, signal
try:
subprocess.run(["taskkill", "/F", "/IM", "chrome-headless-shell.exe"], capture_output=True, timeout=5)
except Exception:
pass
async def scrape_facebook(profile_path: str | None = None, force: bool = False) -> dict:
cleanup_chrome()
fb_cookies = await get_fb_cookies(profile_path)
if not fb_cookies:
logger.warning("No Facebook cookies available")
return []
return {"success": False, "leads": [], "flagged": False, "flag_reason": None, "error": "No cookies available"}
try:
async with async_playwright() as pw:
browser = await pw.chromium.launch(headless=True)
browser = await pw.chromium.launch(
headless=True,
args=['--disable-blink-features=AutomationControlled']
)
viewport = random.choice(VIEWPORTS)
context = await browser.new_context(
user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:150.0) Gecko/20100101 Firefox/150.0',
viewport={'width': 1280, 'height': 800},
viewport=viewport,
)
await context.add_init_script("""
Object.defineProperty(navigator, 'webdriver', { get: () => false });
Object.defineProperty(navigator, 'languages', { get: () => ['en-US', 'en'] });
Object.defineProperty(navigator, 'platform', { get: () => 'Win32' });
window.chrome = { runtime: {} };
Object.defineProperty(navigator, 'hardwareConcurrency', { get: () => 8 });
Object.defineProperty(navigator, 'deviceMemory', { get: () => 8 });
""")
for c in fb_cookies:
try:
await context.add_cookies([c])
@@ -218,36 +280,77 @@ async def scrape_facebook() -> list[dict]:
logger.warning("Failed to inject cookie %s: %s", c.get('name'), e)
page = await context.new_page()
# Navigate through google first for a legitimate Referer header
await page.goto('https://www.google.com/', wait_until='domcontentloaded', timeout=15000)
await page.wait_for_timeout(random.randint(1000, 3000))
await page.goto('https://www.facebook.com/', wait_until='domcontentloaded', timeout=30000)
await page.wait_for_timeout(5000)
await page.wait_for_timeout(random.randint(3000, 8000))
url = page.url
if '/login' in url.lower():
logger.warning("Facebook login page detected — cookies may be expired")
return []
logger.warning("Facebook login page detected — account flagged")
await browser.close()
return {"success": False, "leads": [], "flagged": True, "flag_reason": "login_page", "error": "Facebook login page detected"}
for query in FB_SEARCHES[:6]:
# Browse feed like a human before searching
await human_scroll(page, steps=random.randint(2, 4), total_delay=random.uniform(8, 20))
if random.random() < 0.25:
await page.evaluate("window.scrollTo(0, 0)")
await page.wait_for_timeout(random.randint(2000, 5000))
await human_scroll(page, steps=random.randint(1, 2))
# False start: 30% chance to idle and leave without scraping (skipped when force=true)
if not force and random.random() < 0.3:
await page.wait_for_timeout(random.randint(8000, 20000))
await browser.close()
return {"success": True, "leads": [], "flagged": False, "flag_reason": None, "error": None}
all_posts = []
searches = random.sample(FB_SEARCHES, k=random.randint(3, 6))
for i, query in enumerate(searches):
try:
posts = await search_facebook(page, query)
all_posts.extend(posts)
delay = random.uniform(3, 7)
# Between searches: random break with possible small scroll
if random.random() < 0.4:
await page.evaluate(f"window.scrollBy(0, {random.randint(-300, 300)})")
delay = random.uniform(8, 25)
await page.wait_for_timeout(int(delay * 1000))
# Tab switch: 15% chance after 2nd-3rd search
if i == random.randint(1, 2) and random.random() < 0.15:
new_page = await context.new_page()
await new_page.goto('https://www.messenger.com/', wait_until='domcontentloaded', timeout=15000)
await new_page.wait_for_timeout(random.randint(3000, 8000))
await new_page.close()
await page.wait_for_timeout(random.randint(1000, 3000))
except Exception as e:
logger.warning("Facebook search '%s' failed: %s", query, e)
# Idle before closing — human would pause
if random.random() < 0.5:
await page.wait_for_timeout(random.randint(3000, 10000))
await browser.close()
seen = set()
deduped = []
for p in all_posts:
key = p.get('content', '')[:100]
if key not in seen:
seen.add(key)
deduped.append(p)
leads = deduped[:20]
if leads:
leads = await classify_leads(leads)
return {"success": True, "leads": leads[:15], "flagged": False, "flag_reason": None, "error": None}
except Exception as e:
logger.error("Facebook scrape failed: %s", e)
return []
seen = set()
deduped = []
for p in all_posts:
key = p.get('content', '')[:100]
if key not in seen:
seen.add(key)
deduped.append(p)
return deduped[:20]
return {"success": False, "leads": [], "flagged": False, "flag_reason": None, "error": str(e)}
async def ask_ollama(prompt: str) -> str:
import httpx
async with httpx.AsyncClient(timeout=120) as c:
r = await c.post(f"{OLLAMA_URL}/api/chat", json={
"model": CLASSIFY_MODEL,
@@ -262,42 +365,6 @@ async def ask_ollama(prompt: str) -> str:
data = r.json()
return data["message"]["content"]
async def scrape_warriorforum() -> list[dict]:
results = []
try:
async with httpx.AsyncClient(headers=HEADERS, timeout=15, follow_redirects=True) as c:
r = await c.get('https://www.warriorforum.com/wanted-members-looking-hire-you/')
soup = BeautifulSoup(r.text, 'lxml')
for row in soup.find_all('td', class_='FlexTable-item--title'):
a = row.find('a', href=True)
if not a:
continue
href = a['href']
title = a.text.strip()
if not title or len(title) < 15 or not kw_match_strict(title):
continue
author_div = row.find('div', class_='FlexTable-item-author')
author = author_div.get_text(strip=True).lstrip('by') if author_div else ''
date_str = ''
date_div = row.find('div', class_=lambda c: c and 'media--available' in c)
if date_div:
txt = date_div.get_text(strip=True)
m = re.search(r'(\d{10})', txt)
if m:
date_str = datetime.utcfromtimestamp(int(m.group(1))).strftime('%Y-%m-%d')
if not href.startswith('http'):
href = 'https://www.warriorforum.com' + href
results.append({
"title": title, "url": href,
"author": author,
"content": title[:300],
"source": "warriorforum",
"date": date_str
})
except Exception:
logger.error("WarriorForum scrape failed", exc_info=True)
return results
async def classify_leads(results: list[dict]) -> list[dict]:
if not results:
return []
@@ -320,13 +387,10 @@ Return a JSON array like ["yes","no","yes"] matching the order above."""
try:
raw = await ask_ollama(prompt)
raw = raw.strip()
# Strip markdown code fences
if raw.startswith("```"):
# Remove opening fence
first_nl = raw.find('\n')
if first_nl != -1:
raw = raw[first_nl + 1:]
# Remove closing fence
if raw.endswith("```"):
raw = raw[:-3]
raw = raw.strip()
@@ -339,18 +403,25 @@ Return a JSON array like ["yes","no","yes"] matching the order above."""
filtered.append(results[i])
if filtered:
return filtered[:10]
# AI successfully classified but returned all "no" — respect that decision
logger.info("AI classified all %d items as NOT LEAD — returning empty", len(results))
return []
except Exception as e:
logger.warning("AI classification failed, falling back to keyword filter: %s", e)
# Fallback: only use keyword filter when AI failed
if not ai_succeeded:
strict_keywords = [
"website", "web design", "web develop", "web dev",
"build my website", "build a website", "create a website",
"need web", "looking for web", "new website",
"landing page", "wordpress",
"need a website", "my website", "business website",
"need a designer", "help with my website",
"redesign", "update my website",
]
filtered = []
for r in results:
t = r['title'].lower()
if not kw_match_strict(t):
if not any(kw in t for kw in strict_keywords):
continue
if any(kw in t for kw in ['i build', 'i offer', 'i create', 'i am a', 'web developer available',
'affordable web', 'web hosting', 'free website',
@@ -365,46 +436,10 @@ Return a JSON array like ["yes","no","yes"] matching the order above."""
async def health():
return {"status": "ok"}
class ScrapeRequest(BaseModel):
query: str = ""
@app.post("/scrape/requests")
async def scrape_requests(req: ScrapeRequest):
global last_scrape_time
now = datetime.now()
if last_scrape_time and (now - last_scrape_time) < timedelta(seconds=15):
return {"error": "rate_limited", "retry_after": 15}
last_scrape_time = now
results = await scrape_warriorforum()
if results:
results = await classify_leads(results)
return results[:10]
@app.post("/scrape/facebook")
async def scrape_facebook_endpoint():
results = await scrape_facebook()
if results:
results = await classify_leads(results)
return results[:15]
@app.post("/scrape/all")
async def scrape_all():
results = []
try:
wf = await scrape_warriorforum()
if wf:
wf = await classify_leads(wf)
results.extend(wf[:5])
except Exception as e:
logger.error("WarriorForum scrape in /scrape/all failed: %s", e)
try:
fb = await scrape_facebook()
if fb:
fb = await classify_leads(fb)
results.extend(fb[:8])
except Exception as e:
logger.error("Facebook scrape in /scrape/all failed: %s", e)
return results[:12]
async def scrape_facebook_endpoint(profile_path: str | None = Query(None), force: bool = Query(False)):
result = await scrape_facebook(profile_path, force)
return result
if __name__ == "__main__":
uvicorn.run(app, host="0.0.0.0", port=PORT)