Files Changed
Security Fixes
File Change
.env.local Placeholder JWT secret (rotate to random value: openssl rand -base64 64)
src/middleware.ts Removed /api/auth/me bypass; API routes return JSON 401 (not 302 redirect)
src/lib/auth.ts sameSite: "lax" → "strict" (create + destroy cookie); SVG name chars are pre-computed (no injection)
src/lib/avatar.ts Added sanitizeName() — strips non-alphanumeric/safe chars, max 50 chars
src/app/api/users/route.ts POST restricted to super_admin; validates role against whitelist ["sales","admin","super_admin","dev"]; validates active defaults to true
src/app/api/users/[id]/route.ts DELETE restricted to super_admin; blocks self-deletion
src/app/api/leads/[id]/route.ts GET: non-admin users filtered by assigned_to; PATCH: ownership check + score validated 0-100 range
src/app/api/leads/route.ts DELETE: ownership + admin check; GET: ownership filter for non-admin users
src/app/api/conversations/[id]/messages/route.ts GET/POST: verify user is a conversation_participant before access
rust-ai/src/main.rs CORS: AllowOrigin::list(["localhost:3006", "127.0.0.1:3006"]) (was Any)
Performance Fixes
File Change
src/app/api/leads/route.ts Added limit (default 50), offset query params; ownership filter in SQL
src/app/api/users/route.ts Added limit (default 50), offset query params
src/app/api/conversations/route.ts Added LIMIT 50
src/app/api/leads/[id]/notes/route.ts Added limit (default 50), offset query params
src/app/api/conversations/[id]/messages/route.ts Added limit (default 100), offset, before query params
src/app/(dashboard)/chats/page.tsx Pruned to 5 cached conversations; useMemo wraps messages/conversation/filteredConversations; otherParticipant moved outside component; added 10MB file size limit; recordingChunksRef cleared on discard
src/app/(dashboard)/leads/[id]/page.tsx Optimistic status update rolls back on fetch failure
database/migrations/003_chat.sql Added composite index idx_messages_conversation_created on (conversation_id, created_at DESC)
Code Quality Fixes
File Change
src/lib/ai.ts Removed dead getInstructions()/updateInstructions() (called nonexistent /ai/instructions)
src/lib/auth.ts bcrypt.hash(password, 12) — kept (acceptable, OWASP-recommended)
src/app/login/page.tsx "Forgot password?" button now shows alert to contact admin (was no-op)
src/components/users/user-form-dialog.tsx Password min 6 → 8 chars
26 silent catch {} blocks across 16 files (see below) Added console.warn("...") context messages
This commit is contained in:
+114
-57
@@ -1,4 +1,4 @@
|
||||
import os, json, asyncio, re, shutil, sqlite3, traceback, urllib.parse
|
||||
import os, json, asyncio, re, shutil, sqlite3, traceback, urllib.parse, random, time, logging
|
||||
from datetime import datetime, timedelta
|
||||
from bs4 import BeautifulSoup
|
||||
import httpx
|
||||
@@ -7,14 +7,17 @@ from pydantic import BaseModel
|
||||
import uvicorn
|
||||
from playwright.async_api import async_playwright
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format='%(asctime)s [%(levelname)s] %(message)s')
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
app = FastAPI()
|
||||
PORT = int(os.getenv("PORT", "3008"))
|
||||
|
||||
OLLAMA_URL = os.getenv("OLLAMA_URL", "http://localhost:11434")
|
||||
CLASSIFY_MODEL = os.getenv("CLASSIFY_MODEL", "dolphin-llama3:8b")
|
||||
|
||||
FX_PROFILE = os.getenv('FX_PROFILE', r'C:\Users\USER-PC\AppData\Roaming\Mozilla\Firefox\Profiles\h8p11vlj.default-release')
|
||||
FX_COOKIE_DB = os.path.join(FX_PROFILE, 'cookies.sqlite')
|
||||
FX_PROFILE = os.getenv('FX_PROFILE', '')
|
||||
FX_COOKIE_DB = os.path.join(FX_PROFILE, 'cookies.sqlite') if FX_PROFILE else ''
|
||||
|
||||
HEADERS = {
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
|
||||
@@ -22,7 +25,6 @@ HEADERS = {
|
||||
}
|
||||
|
||||
last_scrape_time: datetime | None = None
|
||||
_pw_browser = None
|
||||
|
||||
STRICT_KEYWORDS = [
|
||||
"website", "web design", "web develop", "web dev",
|
||||
@@ -65,40 +67,75 @@ FB_SEARCHES = [
|
||||
"need a site for my business",
|
||||
]
|
||||
|
||||
def get_fb_cookies():
|
||||
"""Copy Firefox cookie DB and extract Facebook cookies for Chromium injection."""
|
||||
tmp = os.path.join(os.path.dirname(FX_COOKIE_DB), 'cookies_tmp.sqlite')
|
||||
async def get_fb_cookies():
|
||||
if not FX_COOKIE_DB or not os.path.exists(FX_COOKIE_DB):
|
||||
logger.warning("FX_COOKIE_DB not found or FX_PROFILE not set")
|
||||
return []
|
||||
tmp = os.path.join(os.path.dirname(FX_COOKIE_DB), f'cookies_tmp_{os.getpid()}.sqlite')
|
||||
for attempt in range(3):
|
||||
try:
|
||||
shutil.copy2(FX_COOKIE_DB, tmp)
|
||||
break
|
||||
except PermissionError:
|
||||
if attempt < 2:
|
||||
await asyncio.sleep(0.5)
|
||||
else:
|
||||
logger.error("Failed to copy cookie DB after 3 attempts")
|
||||
return []
|
||||
try:
|
||||
if not os.path.exists(FX_COOKIE_DB):
|
||||
return []
|
||||
shutil.copy2(FX_COOKIE_DB, tmp)
|
||||
conn = sqlite3.connect(tmp)
|
||||
c = conn.cursor()
|
||||
c.execute("SELECT name, value, host, path FROM moz_cookies WHERE host LIKE '%facebook.com'")
|
||||
rows = c.fetchall()
|
||||
conn.close()
|
||||
os.remove(tmp)
|
||||
try:
|
||||
os.remove(tmp)
|
||||
except Exception:
|
||||
pass
|
||||
return [{
|
||||
"name": name, "value": value,
|
||||
"domain": host if host.startswith('.') else f'.{host}',
|
||||
"path": path,
|
||||
"httpOnly": False, "secure": False, "sameSite": "Lax",
|
||||
"httpOnly": True, "secure": True, "sameSite": "Lax",
|
||||
} for name, value, host, path in rows]
|
||||
except Exception as e:
|
||||
logger.error("Cookie DB read error: %s", e)
|
||||
try:
|
||||
os.remove(tmp)
|
||||
except Exception:
|
||||
pass
|
||||
return []
|
||||
|
||||
def _is_real_text(line: str) -> bool:
|
||||
if not line:
|
||||
return False
|
||||
# Count words (sequences of letters)
|
||||
words = re.findall(r'[A-Za-z]{2,}', line)
|
||||
return len(words) >= 2
|
||||
WEEKDAY_ORDER = ['monday', 'tuesday', 'wednesday', 'thursday', 'friday', 'saturday', 'sunday']
|
||||
|
||||
def _parse_fb_date(block: list[str]) -> str:
|
||||
for l in block:
|
||||
l = l.strip()
|
||||
m = re.match(r'(\d+)\s*(h|hr|hrs|hour|hours)\s*ago', l)
|
||||
if m:
|
||||
hours = int(m.group(1))
|
||||
d = datetime.now() - timedelta(hours=hours)
|
||||
return d.strftime('%Y-%m-%d')
|
||||
if l.lower() == 'yesterday':
|
||||
d = datetime.now() - timedelta(days=1)
|
||||
return d.strftime('%Y-%m-%d')
|
||||
low = l.lower()
|
||||
if low in WEEKDAY_ORDER:
|
||||
day_idx = WEEKDAY_ORDER.index(low)
|
||||
today_idx = datetime.now().weekday()
|
||||
days_ago = (today_idx - day_idx) % 7
|
||||
d = datetime.now() - timedelta(days=days_ago)
|
||||
return d.strftime('%Y-%m-%d')
|
||||
for fmt in ['%Y-%m-%d', '%d %B %Y', '%B %d %Y', '%d %b %Y', '%b %d %Y', '%d %b', '%b %d']:
|
||||
try:
|
||||
dt = datetime.strptime(l, fmt)
|
||||
return dt.strftime('%Y-%m-%d')
|
||||
except ValueError:
|
||||
pass
|
||||
return datetime.now().strftime('%Y-%m-%d')
|
||||
|
||||
def _extract_posts_from_text(raw: str, url: str) -> list[dict]:
|
||||
"""Parse Facebook search results page text into structured posts."""
|
||||
lines = [l.strip() for l in raw.split('\n')]
|
||||
|
||||
# Split into blocks separated by "Facebook" boilerplate lines (2+ consecutive)
|
||||
blocks = []
|
||||
cur = []
|
||||
fb_run = 0
|
||||
@@ -111,7 +148,6 @@ def _extract_posts_from_text(raw: str, url: str) -> list[dict]:
|
||||
else:
|
||||
fb_run = 0
|
||||
if l and len(l) >= 15:
|
||||
# Check for real words (not scattered reaction chars)
|
||||
words = re.findall(r'[A-Za-z]{2,}', l)
|
||||
if len(words) >= 2:
|
||||
cur.append(l)
|
||||
@@ -123,11 +159,9 @@ def _extract_posts_from_text(raw: str, url: str) -> list[dict]:
|
||||
for block in blocks:
|
||||
if not block:
|
||||
continue
|
||||
# Find keyword-matched content in this block
|
||||
kw_indices = [i for i, l in enumerate(block) if kw_match(l)]
|
||||
if not kw_indices:
|
||||
continue
|
||||
# Use first keyword match as anchor, grab context
|
||||
i = kw_indices[0]
|
||||
start = max(0, i - 2)
|
||||
end = min(len(block), i + 5)
|
||||
@@ -144,29 +178,31 @@ def _extract_posts_from_text(raw: str, url: str) -> list[dict]:
|
||||
"author": block[start] if start < i else '',
|
||||
"url": url,
|
||||
"source": "facebook",
|
||||
"date": datetime.now().strftime('%Y-%m-%d'),
|
||||
"date": _parse_fb_date(block),
|
||||
})
|
||||
|
||||
return posts
|
||||
|
||||
async def search_facebook(page, query: str) -> list[dict]:
|
||||
"""Search Facebook and extract post leads from raw page text."""
|
||||
url = f'https://www.facebook.com/search/posts/?q={urllib.parse.quote(query)}'
|
||||
try:
|
||||
await page.goto(url, wait_until='domcontentloaded', timeout=30000)
|
||||
# Wait for the page content to fully render (React/XHR loads)
|
||||
await page.wait_for_timeout(6000)
|
||||
except:
|
||||
except Exception as e:
|
||||
logger.warning("Facebook search navigation failed for '%s': %s", query, e)
|
||||
return []
|
||||
|
||||
raw = await page.evaluate('document.body.innerText')
|
||||
try:
|
||||
raw = await page.evaluate('document.body.innerText')
|
||||
except Exception as e:
|
||||
logger.warning("Failed to evaluate page text: %s", e)
|
||||
return []
|
||||
return _extract_posts_from_text(raw, url)
|
||||
|
||||
async def scrape_facebook() -> list[dict]:
|
||||
"""Launch headless Chromium, inject Firefox Facebook cookies, search for leads."""
|
||||
all_posts = []
|
||||
fb_cookies = get_fb_cookies()
|
||||
fb_cookies = await get_fb_cookies()
|
||||
if not fb_cookies:
|
||||
logger.warning("No Facebook cookies available")
|
||||
return []
|
||||
try:
|
||||
async with async_playwright() as pw:
|
||||
@@ -178,32 +214,34 @@ async def scrape_facebook() -> list[dict]:
|
||||
for c in fb_cookies:
|
||||
try:
|
||||
await context.add_cookies([c])
|
||||
except:
|
||||
pass
|
||||
except Exception as e:
|
||||
logger.warning("Failed to inject cookie %s: %s", c.get('name'), e)
|
||||
|
||||
page = await context.new_page()
|
||||
|
||||
await page.goto('https://www.facebook.com/', wait_until='domcontentloaded', timeout=30000)
|
||||
await page.wait_for_timeout(5000)
|
||||
|
||||
url = page.url
|
||||
if '/login' in url.lower():
|
||||
logger.warning("Facebook login page detected — cookies may be expired")
|
||||
return []
|
||||
|
||||
for query in FB_SEARCHES[:6]:
|
||||
try:
|
||||
posts = await search_facebook(page, query)
|
||||
all_posts.extend(posts)
|
||||
await page.wait_for_timeout(2000)
|
||||
except:
|
||||
continue
|
||||
except Exception:
|
||||
delay = random.uniform(3, 7)
|
||||
await page.wait_for_timeout(int(delay * 1000))
|
||||
except Exception as e:
|
||||
logger.warning("Facebook search '%s' failed: %s", query, e)
|
||||
except Exception as e:
|
||||
logger.error("Facebook scrape failed: %s", e)
|
||||
return []
|
||||
|
||||
seen = set()
|
||||
deduped = []
|
||||
for p in all_posts:
|
||||
key = p['content'][:100]
|
||||
key = p.get('content', '')[:100]
|
||||
if key not in seen:
|
||||
seen.add(key)
|
||||
deduped.append(p)
|
||||
@@ -221,7 +259,8 @@ async def ask_ollama(prompt: str) -> str:
|
||||
"options": {"temperature": 0.05, "num_predict": 1024},
|
||||
})
|
||||
r.raise_for_status()
|
||||
return r.json()["message"]["content"]
|
||||
data = r.json()
|
||||
return data["message"]["content"]
|
||||
|
||||
async def scrape_warriorforum() -> list[dict]:
|
||||
results = []
|
||||
@@ -245,8 +284,9 @@ async def scrape_warriorforum() -> list[dict]:
|
||||
txt = date_div.get_text(strip=True)
|
||||
m = re.search(r'(\d{10})', txt)
|
||||
if m:
|
||||
from datetime import datetime as dtdt
|
||||
date_str = dtdt.utcfromtimestamp(int(m.group(1))).strftime('%Y-%m-%d')
|
||||
date_str = datetime.utcfromtimestamp(int(m.group(1))).strftime('%Y-%m-%d')
|
||||
if not href.startswith('http'):
|
||||
href = 'https://www.warriorforum.com' + href
|
||||
results.append({
|
||||
"title": title, "url": href,
|
||||
"author": author,
|
||||
@@ -255,13 +295,12 @@ async def scrape_warriorforum() -> list[dict]:
|
||||
"date": date_str
|
||||
})
|
||||
except Exception:
|
||||
traceback.print_exc()
|
||||
logger.error("WarriorForum scrape failed", exc_info=True)
|
||||
return results
|
||||
|
||||
async def classify_leads(results: list[dict]) -> list[dict]:
|
||||
if not results:
|
||||
return []
|
||||
filtered = []
|
||||
briefs = [r["title"][:200] for r in results]
|
||||
prompt = f"""Classify each post as LEAD or NOT.
|
||||
LEAD = someone REQUESTING/POSTING/WANTING a website built, designed, or created for them.
|
||||
@@ -277,32 +316,50 @@ NOT LEAD:
|
||||
For each numbered post, answer ONLY "yes" (LEAD) or "no" (NOT LEAD):
|
||||
{chr(10).join(f'{i+1}. {t}' for i, t in enumerate(briefs))}
|
||||
Return a JSON array like ["yes","no","yes"] matching the order above."""
|
||||
ai_succeeded = False
|
||||
try:
|
||||
raw = await ask_ollama(prompt)
|
||||
raw = raw.strip()
|
||||
if raw.startswith("```json"): raw = raw[7:]
|
||||
if raw.startswith("```"): raw = raw.split("\n", 1)[-1].rsplit("```", 1)[0]
|
||||
# Strip markdown code fences
|
||||
if raw.startswith("```"):
|
||||
# Remove opening fence
|
||||
first_nl = raw.find('\n')
|
||||
if first_nl != -1:
|
||||
raw = raw[first_nl + 1:]
|
||||
# Remove closing fence
|
||||
if raw.endswith("```"):
|
||||
raw = raw[:-3]
|
||||
raw = raw.strip()
|
||||
answers = json.loads(raw)
|
||||
if isinstance(answers, list) and len(answers) == len(results):
|
||||
ai_succeeded = True
|
||||
filtered = []
|
||||
for i, ans in enumerate(answers):
|
||||
if isinstance(ans, str) and ans.lower() == 'yes':
|
||||
filtered.append(results[i])
|
||||
except:
|
||||
pass
|
||||
if not filtered:
|
||||
if filtered:
|
||||
return filtered[:10]
|
||||
# AI successfully classified but returned all "no" — respect that decision
|
||||
logger.info("AI classified all %d items as NOT LEAD — returning empty", len(results))
|
||||
return []
|
||||
except Exception as e:
|
||||
logger.warning("AI classification failed, falling back to keyword filter: %s", e)
|
||||
|
||||
# Fallback: only use keyword filter when AI failed
|
||||
if not ai_succeeded:
|
||||
filtered = []
|
||||
for r in results:
|
||||
t = r['title'].lower()
|
||||
# MUST match a web-dev keyword to be a lead
|
||||
if not kw_match_strict(t):
|
||||
continue
|
||||
# But NOT these (offerings, not requests)
|
||||
if any(kw in t for kw in ['i build', 'i offer', 'i create', 'i am a', 'web developer available',
|
||||
'affordable web', 'web hosting', 'free website',
|
||||
'limited time', 'special offer', 'sign up now',
|
||||
'offer']):
|
||||
continue
|
||||
filtered.append(r)
|
||||
return filtered[:10]
|
||||
return filtered[:10]
|
||||
return []
|
||||
|
||||
@app.get("/health")
|
||||
async def health():
|
||||
@@ -338,15 +395,15 @@ async def scrape_all():
|
||||
if wf:
|
||||
wf = await classify_leads(wf)
|
||||
results.extend(wf[:5])
|
||||
except:
|
||||
pass
|
||||
except Exception as e:
|
||||
logger.error("WarriorForum scrape in /scrape/all failed: %s", e)
|
||||
try:
|
||||
fb = await scrape_facebook()
|
||||
if fb:
|
||||
fb = await classify_leads(fb)
|
||||
results.extend(fb[:8])
|
||||
except:
|
||||
pass
|
||||
except Exception as e:
|
||||
logger.error("Facebook scrape in /scrape/all failed: %s", e)
|
||||
return results[:12]
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
Reference in New Issue
Block a user