Got facebook to work test tomorrow with new accounts please
This commit is contained in:
+176
-96
@@ -11,84 +11,9 @@ use std::sync::{Arc, Mutex};
|
||||
use tower_http::cors::{Any, CorsLayer};
|
||||
use tracing::{error, info};
|
||||
use uuid::Uuid;
|
||||
use reqwest::blocking::Client;
|
||||
use scraper::{Html, Selector};
|
||||
use rand::rngs::StdRng;
|
||||
use rand::SeedableRng;
|
||||
use rand::Rng;
|
||||
use std::time::Duration;
|
||||
use std::thread;
|
||||
|
||||
// ── Facebook Scraper ───────────────────────────────────────────
|
||||
|
||||
// List of proxies
|
||||
const PROXIES: &[&str] = &[
|
||||
"http://user:pass@192.168.1.1:8080",
|
||||
"http://user:pass@192.168.1.2:8080",
|
||||
"http://user:pass@192.168.1.3:8080",
|
||||
"http://user:pass@192.168.1.4:8080",
|
||||
"http://user:pass@192.168.1.5:8080",
|
||||
];
|
||||
|
||||
// List of user agents
|
||||
const USER_AGENTS: &[&str] = &[
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3",
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.1.2 Safari/605.1.15",
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:61.0) Gecko/20100101 Firefox/61.0",
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.1.2 Safari/605.1.15",
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36",
|
||||
// Add more user agents here
|
||||
];
|
||||
|
||||
fn get_random_proxy() -> String {
|
||||
let mut rng = rand::thread_rng();
|
||||
PROXIES[rng.gen_range(0..PROXIES.len())].to_string()
|
||||
}
|
||||
|
||||
fn get_random_user_agent() -> String {
|
||||
let mut rng = rand::thread_rng();
|
||||
USER_AGENTS[rng.gen_range(0..USER_AGENTS.len())].to_string()
|
||||
}
|
||||
|
||||
fn scrape_conversations(url: &str) -> Result<Html, Box<dyn std::error::Error>> {
|
||||
let proxy = get_random_proxy();
|
||||
let user_agent = get_random_user_agent();
|
||||
let client = Client::builder()
|
||||
.proxy(reqwest::Proxy::all(proxy)?)
|
||||
.build()?;
|
||||
|
||||
let response = client.get(url)
|
||||
.header("User-Agent", user_agent)
|
||||
.send()?;
|
||||
|
||||
if response.status().is_success() {
|
||||
let html = response.text()?;
|
||||
Ok(Html::parse_document(&html))
|
||||
} else {
|
||||
Err(format!("Failed to retrieve the page: {}", response.status()).into())
|
||||
}
|
||||
}
|
||||
|
||||
fn run_facebook_scraper() {
|
||||
let url = "https://www.facebook.com/search/top/?q=need%20website%20create";
|
||||
match scrape_conversations(url) {
|
||||
Ok(soup) => {
|
||||
// Example: Extract conversation data
|
||||
let selector = Selector::parse("div.conversation").unwrap();
|
||||
for element in soup.select(&selector) {
|
||||
println!("Conversation: {}", element.inner_html());
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
error!("Facebook scraper error: {}", e);
|
||||
}
|
||||
}
|
||||
|
||||
// Introduce random delay
|
||||
let mut rng = rand::thread_rng();
|
||||
let delay = rng.gen_range(1..5); // Delay between 1 to 5 seconds
|
||||
thread::sleep(Duration::from_secs(delay));
|
||||
}
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
// ── Shared state ───────────────────────────────────────────────
|
||||
|
||||
@@ -97,6 +22,46 @@ struct AppState {
|
||||
ollama_url: String,
|
||||
model: String,
|
||||
jobs: Vec<Job>,
|
||||
leads: Arc<Mutex<LeadStore>>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
struct Lead {
|
||||
title: String,
|
||||
url: String,
|
||||
source: String,
|
||||
found_at: u64,
|
||||
author: String,
|
||||
date: String,
|
||||
content: String,
|
||||
}
|
||||
|
||||
struct LeadStore {
|
||||
leads: Vec<Lead>,
|
||||
max_size: usize,
|
||||
}
|
||||
|
||||
impl LeadStore {
|
||||
fn new(max_size: usize) -> Self {
|
||||
Self { leads: Vec::new(), max_size }
|
||||
}
|
||||
|
||||
fn push(&mut self, lead: Lead) {
|
||||
// Deduplicate by URL
|
||||
if !self.leads.iter().any(|l| l.url == lead.url) {
|
||||
self.leads.insert(0, lead);
|
||||
self.leads.truncate(self.max_size);
|
||||
}
|
||||
}
|
||||
|
||||
fn recent(&self, max_age_secs: u64, limit: usize) -> Vec<Lead> {
|
||||
let now = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs();
|
||||
self.leads.iter()
|
||||
.filter(|l| now - l.found_at <= max_age_secs)
|
||||
.take(limit)
|
||||
.cloned()
|
||||
.collect()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
@@ -164,21 +129,57 @@ struct OllamaResponseMessage {
|
||||
|
||||
// ── System prompt builder ─────────────────────────────────────
|
||||
|
||||
fn build_system_prompt(jobs: &[Job]) -> String {
|
||||
fn format_leads_output(leads: &[Lead]) -> String {
|
||||
if leads.is_empty() {
|
||||
return "No new requests found yet.".to_string();
|
||||
}
|
||||
leads
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(i, l)| {
|
||||
let author = if l.author.is_empty() { "Unknown" } else { &l.author };
|
||||
let date = if l.date.is_empty() { "Unknown" } else { &l.date[..l.date.len().min(10)] };
|
||||
format!(
|
||||
"{}. {}\n {}\n {}\n {}",
|
||||
i + 1,
|
||||
author,
|
||||
date,
|
||||
l.title,
|
||||
l.url
|
||||
)
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n")
|
||||
}
|
||||
|
||||
fn build_system_prompt(jobs: &[Job], leads: &[Lead]) -> String {
|
||||
let job_list: Vec<String> = jobs
|
||||
.iter()
|
||||
.map(|j| format!("- {} ({}): {}", j.job_title, j.industry, j.description))
|
||||
.collect();
|
||||
let job_list_str = job_list.join("\n");
|
||||
|
||||
let lead_summary: Vec<String> = leads
|
||||
.iter()
|
||||
.map(|l| {
|
||||
format!("{} | {} | {}", l.author, l.title, l.url)
|
||||
})
|
||||
.collect();
|
||||
let lead_summary_str = if lead_summary.is_empty() {
|
||||
"None yet.".to_string()
|
||||
} else {
|
||||
lead_summary.join("\n")
|
||||
};
|
||||
|
||||
format!(
|
||||
"You are a Sales AI Assistant for Coast IT CRM. Your role is to help salespeople \
|
||||
with tips, strategies, and guidance.\n\n\
|
||||
"You are a Sales AI Assistant for Coast IT CRM.\n\n\
|
||||
Available job categories to target:\n{}\n\n\
|
||||
Provide concise, actionable sales advice. When asked about a specific job category, \
|
||||
give targeted tips on finding and engaging prospects in that field. \
|
||||
Keep responses under 300 words unless asked for detail.",
|
||||
job_list_str
|
||||
Recent leads context:\n{}\n\n\
|
||||
Rules:\n\
|
||||
- When asked about leads, answer concisely under 150 words.\n\
|
||||
- If asked to suggest a sales strategy, give brief actionable advice.\n\
|
||||
- Be direct and professional. No fluff.",
|
||||
job_list_str, lead_summary_str
|
||||
)
|
||||
}
|
||||
|
||||
@@ -194,7 +195,64 @@ async fn handle_chat(
|
||||
_ => return Err((axum::http::StatusCode::FORBIDDEN, "Forbidden".to_string())),
|
||||
}
|
||||
|
||||
let system_prompt = build_system_prompt(&state.jobs);
|
||||
// Intercept lead listing requests — scrape on demand then return
|
||||
let msg_lower = req.message.to_lowercase();
|
||||
let msg_words: Vec<&str> = msg_lower.split_whitespace().collect();
|
||||
let has_listing = msg_words.iter().any(|w| ["listings", "listing", "leads", "links", "lists"].contains(w));
|
||||
let has_show = msg_lower.contains("show me") || msg_lower.contains("give me") || msg_lower.contains("pull");
|
||||
let has_job = msg_words.contains(&"jobs") || msg_words.contains(&"job");
|
||||
if has_listing || (has_show && has_job) || (has_show && msg_lower.contains("links")) || msg_lower.contains("recent leads") {
|
||||
// Scrape fresh leads now — call both services
|
||||
let now = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs();
|
||||
let client = reqwest::Client::new();
|
||||
let services = [
|
||||
"http://localhost:3008/scrape/all".to_string(),
|
||||
];
|
||||
for service_url in &services {
|
||||
if let Ok(resp) = client
|
||||
.post(service_url)
|
||||
.timeout(Duration::from_secs(120))
|
||||
.send()
|
||||
.await
|
||||
{
|
||||
if let Ok(leads_data) = resp.json::<Vec<serde_json::Value>>().await {
|
||||
info!("Scraped {} leads from {}", leads_data.len(), service_url);
|
||||
let mut store = state.leads.lock().unwrap();
|
||||
for item in &leads_data {
|
||||
store.push(Lead {
|
||||
title: item["title"].as_str().unwrap_or("").chars().take(120).collect(),
|
||||
url: item["url"].as_str().unwrap_or("").to_string(),
|
||||
source: item["source"].as_str().unwrap_or("unknown").to_string(),
|
||||
found_at: now,
|
||||
author: item["author"].as_str().unwrap_or("").chars().take(60).collect(),
|
||||
date: item["date"].as_str().unwrap_or("").chars().take(30).collect(),
|
||||
content: item["content"].as_str().unwrap_or("").chars().take(300).collect(),
|
||||
});
|
||||
}
|
||||
}
|
||||
} else {
|
||||
error!("Scraper service unreachable: {}", service_url);
|
||||
}
|
||||
}
|
||||
|
||||
let recent_leads = state.leads.lock().unwrap().recent(604800, 20);
|
||||
let response = format_leads_output(&recent_leads);
|
||||
let _ = sqlx::query(
|
||||
"INSERT INTO ai_conversations (id, user_id, role, message, response) VALUES ($1, $2, $3, $4, $5)",
|
||||
)
|
||||
.bind(Uuid::new_v4())
|
||||
.bind(Uuid::parse_str(&req.user_id).unwrap_or(Uuid::nil()))
|
||||
.bind(&req.user_role)
|
||||
.bind(&req.message)
|
||||
.bind(&response)
|
||||
.execute(&state.db)
|
||||
.await;
|
||||
return Ok(Json(ChatResponse { response }));
|
||||
}
|
||||
|
||||
let recent_leads = state.leads.lock().unwrap().recent(604800, 20);
|
||||
|
||||
let system_prompt = build_system_prompt(&state.jobs, &recent_leads);
|
||||
|
||||
let message_text = req.message.clone();
|
||||
|
||||
@@ -300,7 +358,7 @@ async fn main() {
|
||||
std::env::var("DATABASE_URL").expect("DATABASE_URL must be set");
|
||||
let ollama_url =
|
||||
std::env::var("OLLAMA_BASE_URL").unwrap_or_else(|_| "http://localhost:11434".to_string());
|
||||
let model = std::env::var("AI_MODEL").unwrap_or_else(|_| "sam860/dolphin3-llama3.2:3b".to_string());
|
||||
let model = std::env::var("AI_MODEL").unwrap_or_else(|_| "dolphin-phi".to_string());
|
||||
let host = std::env::var("AI_HOST").unwrap_or_else(|_| "0.0.0.0".to_string());
|
||||
let port: u16 = std::env::var("AI_PORT")
|
||||
.unwrap_or_else(|_| "3001".to_string())
|
||||
@@ -332,11 +390,14 @@ async fn main() {
|
||||
|
||||
info!("Connected to PostgreSQL");
|
||||
|
||||
let lead_store = Arc::new(Mutex::new(LeadStore::new(100)));
|
||||
|
||||
let state = Arc::new(AppState {
|
||||
db,
|
||||
ollama_url,
|
||||
model,
|
||||
jobs,
|
||||
leads: lead_store.clone(),
|
||||
});
|
||||
|
||||
// CORS layer
|
||||
@@ -359,20 +420,39 @@ async fn main() {
|
||||
.await
|
||||
.expect("Failed to bind address");
|
||||
|
||||
// Start Facebook scraper in a separate thread
|
||||
let rng = Arc::new(Mutex::new(StdRng::from_entropy()));
|
||||
// Start background scrapers
|
||||
let bg_leads = lead_store.clone();
|
||||
tokio::task::spawn_blocking(move || {
|
||||
let bg_client = reqwest::blocking::Client::builder()
|
||||
.timeout(Duration::from_secs(120))
|
||||
.build().ok();
|
||||
loop {
|
||||
{
|
||||
let _lock = rng.lock().unwrap();
|
||||
run_facebook_scraper();
|
||||
if let Some(ref c) = bg_client {
|
||||
let urls = vec![
|
||||
"http://localhost:3008/scrape/all".to_string(),
|
||||
];
|
||||
let now = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs();
|
||||
for url in &urls {
|
||||
if let Ok(resp) = c.post(url).send() {
|
||||
if let Ok(data) = resp.json::<Vec<serde_json::Value>>() {
|
||||
let mut store = bg_leads.lock().unwrap();
|
||||
for item in &data {
|
||||
store.push(Lead {
|
||||
title: item["title"].as_str().unwrap_or("").chars().take(120).collect(),
|
||||
url: item["url"].as_str().unwrap_or("").to_string(),
|
||||
source: item["source"].as_str().unwrap_or("unknown").to_string(),
|
||||
found_at: now,
|
||||
author: item["author"].as_str().unwrap_or("").chars().take(60).collect(),
|
||||
date: item["date"].as_str().unwrap_or("").chars().take(30).collect(),
|
||||
content: item["content"].as_str().unwrap_or("").chars().take(300).collect(),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// Introduce random delay
|
||||
let delay = {
|
||||
let mut rng = rng.lock().unwrap();
|
||||
rng.gen_range(1..5)
|
||||
};
|
||||
thread::sleep(Duration::from_secs(delay));
|
||||
let delay = rand::thread_rng().gen_range(120..300);
|
||||
std::thread::sleep(Duration::from_secs(delay));
|
||||
}
|
||||
});
|
||||
|
||||
|
||||
Reference in New Issue
Block a user