382 lines
11 KiB
Rust
382 lines
11 KiB
Rust
use axum::{
|
|
extract::State,
|
|
http::Method,
|
|
routing::{get, post},
|
|
Json, Router,
|
|
};
|
|
use serde::{Deserialize, Serialize};
|
|
use sqlx::postgres::PgPoolOptions;
|
|
use std::fs;
|
|
use std::sync::{Arc, Mutex};
|
|
use tower_http::cors::{Any, CorsLayer};
|
|
use tracing::{error, info};
|
|
use uuid::Uuid;
|
|
use reqwest::blocking::Client;
|
|
use scraper::{Html, Selector};
|
|
use rand::rngs::StdRng;
|
|
use rand::SeedableRng;
|
|
use rand::Rng;
|
|
use std::time::Duration;
|
|
use std::thread;
|
|
|
|
// ── Facebook Scraper ───────────────────────────────────────────
|
|
|
|
// List of proxies
|
|
const PROXIES: &[&str] = &[
|
|
"http://user:pass@192.168.1.1:8080",
|
|
"http://user:pass@192.168.1.2:8080",
|
|
"http://user:pass@192.168.1.3:8080",
|
|
"http://user:pass@192.168.1.4:8080",
|
|
"http://user:pass@192.168.1.5:8080",
|
|
];
|
|
|
|
// List of user agents
|
|
const USER_AGENTS: &[&str] = &[
|
|
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3",
|
|
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.1.2 Safari/605.1.15",
|
|
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:61.0) Gecko/20100101 Firefox/61.0",
|
|
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.1.2 Safari/605.1.15",
|
|
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36",
|
|
// Add more user agents here
|
|
];
|
|
|
|
fn get_random_proxy() -> String {
|
|
let mut rng = rand::thread_rng();
|
|
PROXIES[rng.gen_range(0..PROXIES.len())].to_string()
|
|
}
|
|
|
|
fn get_random_user_agent() -> String {
|
|
let mut rng = rand::thread_rng();
|
|
USER_AGENTS[rng.gen_range(0..USER_AGENTS.len())].to_string()
|
|
}
|
|
|
|
fn scrape_conversations(url: &str) -> Result<Html, Box<dyn std::error::Error>> {
|
|
let proxy = get_random_proxy();
|
|
let user_agent = get_random_user_agent();
|
|
let client = Client::builder()
|
|
.proxy(reqwest::Proxy::all(proxy)?)
|
|
.build()?;
|
|
|
|
let response = client.get(url)
|
|
.header("User-Agent", user_agent)
|
|
.send()?;
|
|
|
|
if response.status().is_success() {
|
|
let html = response.text()?;
|
|
Ok(Html::parse_document(&html))
|
|
} else {
|
|
Err(format!("Failed to retrieve the page: {}", response.status()).into())
|
|
}
|
|
}
|
|
|
|
fn run_facebook_scraper() {
|
|
let url = "https://www.facebook.com/messages";
|
|
match scrape_conversations(url) {
|
|
Ok(soup) => {
|
|
// Example: Extract conversation data
|
|
let selector = Selector::parse("div.conversation").unwrap();
|
|
for element in soup.select(&selector) {
|
|
println!("Conversation: {}", element.inner_html());
|
|
}
|
|
}
|
|
Err(e) => {
|
|
error!("Facebook scraper error: {}", e);
|
|
}
|
|
}
|
|
|
|
// Introduce random delay
|
|
let mut rng = rand::thread_rng();
|
|
let delay = rng.gen_range(1..5); // Delay between 1 to 5 seconds
|
|
thread::sleep(Duration::from_secs(delay));
|
|
}
|
|
|
|
// ── Shared state ───────────────────────────────────────────────
|
|
|
|
struct AppState {
|
|
db: sqlx::PgPool,
|
|
ollama_url: String,
|
|
model: String,
|
|
jobs: Vec<Job>,
|
|
}
|
|
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
struct Job {
|
|
job_title: String,
|
|
keywords: Vec<String>,
|
|
industry: String,
|
|
description: String,
|
|
}
|
|
|
|
#[derive(Debug, Deserialize)]
|
|
struct ChatRequest {
|
|
message: String,
|
|
user_id: String,
|
|
user_role: String,
|
|
}
|
|
|
|
#[derive(Debug, Serialize)]
|
|
struct ChatResponse {
|
|
response: String,
|
|
}
|
|
|
|
#[derive(Debug, Serialize)]
|
|
struct JobsResponse {
|
|
jobs: Vec<Job>,
|
|
}
|
|
|
|
#[derive(Debug, Serialize)]
|
|
struct HealthResponse {
|
|
status: String,
|
|
model: String,
|
|
}
|
|
|
|
// ── Ollama API types ───────────────────────────────────────────
|
|
|
|
#[derive(Debug, Serialize)]
|
|
struct OllamaChatMessage {
|
|
role: String,
|
|
content: String,
|
|
}
|
|
|
|
#[derive(Debug, Serialize)]
|
|
struct OllamaRequest {
|
|
model: String,
|
|
messages: Vec<OllamaChatMessage>,
|
|
stream: bool,
|
|
options: OllamaOptions,
|
|
}
|
|
|
|
#[derive(Debug, Serialize)]
|
|
struct OllamaOptions {
|
|
temperature: f32,
|
|
num_predict: u32,
|
|
}
|
|
|
|
#[derive(Debug, Deserialize)]
|
|
struct OllamaResponse {
|
|
message: Option<OllamaResponseMessage>,
|
|
}
|
|
|
|
#[derive(Debug, Deserialize)]
|
|
struct OllamaResponseMessage {
|
|
content: String,
|
|
}
|
|
|
|
// ── System prompt builder ─────────────────────────────────────
|
|
|
|
fn build_system_prompt(jobs: &[Job]) -> String {
|
|
let job_list: Vec<String> = jobs
|
|
.iter()
|
|
.map(|j| format!("- {} ({}): {}", j.job_title, j.industry, j.description))
|
|
.collect();
|
|
let job_list_str = job_list.join("\n");
|
|
|
|
format!(
|
|
"You are a Sales AI Assistant for Coast IT CRM. Your role is to help salespeople \
|
|
with tips, strategies, and guidance.\n\n\
|
|
Available job categories to target:\n{}\n\n\
|
|
Provide concise, actionable sales advice. When asked about a specific job category, \
|
|
give targeted tips on finding and engaging prospects in that field. \
|
|
Keep responses under 300 words unless asked for detail.",
|
|
job_list_str
|
|
)
|
|
}
|
|
|
|
// ── Chat handler ───────────────────────────────────────────────
|
|
|
|
async fn handle_chat(
|
|
State(state): State<Arc<AppState>>,
|
|
Json(req): Json<ChatRequest>,
|
|
) -> Result<Json<ChatResponse>, (axum::http::StatusCode, String)> {
|
|
// Validate role
|
|
match req.user_role.as_str() {
|
|
"sales" | "admin" | "super_admin" => {}
|
|
_ => return Err((axum::http::StatusCode::FORBIDDEN, "Forbidden".to_string())),
|
|
}
|
|
|
|
let system_prompt = build_system_prompt(&state.jobs);
|
|
|
|
let message_text = req.message.clone();
|
|
|
|
let ollama_req = OllamaRequest {
|
|
model: state.model.clone(),
|
|
messages: vec![
|
|
OllamaChatMessage {
|
|
role: "system".to_string(),
|
|
content: system_prompt,
|
|
},
|
|
OllamaChatMessage {
|
|
role: "user".to_string(),
|
|
content: req.message.clone(),
|
|
},
|
|
],
|
|
stream: false,
|
|
options: OllamaOptions {
|
|
temperature: 0.7,
|
|
num_predict: 1024,
|
|
},
|
|
};
|
|
|
|
let client = reqwest::Client::new();
|
|
let resp = client
|
|
.post(format!("{}/api/chat", state.ollama_url))
|
|
.json(&ollama_req)
|
|
.send()
|
|
.await
|
|
.map_err(|e| {
|
|
error!("Ollama request failed: {}", e);
|
|
(
|
|
axum::http::StatusCode::SERVICE_UNAVAILABLE,
|
|
"AI service unavailable".to_string(),
|
|
)
|
|
})?;
|
|
|
|
let ollama_resp: OllamaResponse = resp.json().await.map_err(|e| {
|
|
error!("Failed to parse Ollama response: {}", e);
|
|
(
|
|
axum::http::StatusCode::SERVICE_UNAVAILABLE,
|
|
"AI response parse error".to_string(),
|
|
)
|
|
})?;
|
|
|
|
let response_text = ollama_resp
|
|
.message
|
|
.map(|m| m.content)
|
|
.unwrap_or_default();
|
|
|
|
// Store in database
|
|
let user_id = Uuid::parse_str(&req.user_id).unwrap_or(Uuid::nil());
|
|
let _ = sqlx::query(
|
|
"INSERT INTO ai_conversations (id, user_id, role, message, response) VALUES ($1, $2, $3, $4, $5)",
|
|
)
|
|
.bind(Uuid::new_v4())
|
|
.bind(user_id)
|
|
.bind(&req.user_role)
|
|
.bind(&message_text)
|
|
.bind(&response_text)
|
|
.execute(&state.db)
|
|
.await;
|
|
|
|
Ok(Json(ChatResponse {
|
|
response: response_text,
|
|
}))
|
|
}
|
|
|
|
// ── Jobs handler ───────────────────────────────────────────────
|
|
|
|
async fn handle_jobs(
|
|
State(state): State<Arc<AppState>>,
|
|
) -> Json<JobsResponse> {
|
|
Json(JobsResponse {
|
|
jobs: state.jobs.clone(),
|
|
})
|
|
}
|
|
|
|
// ── Health handler ─────────────────────────────────────────────
|
|
|
|
async fn handle_health(
|
|
State(state): State<Arc<AppState>>,
|
|
) -> Json<HealthResponse> {
|
|
Json(HealthResponse {
|
|
status: "ok".to_string(),
|
|
model: state.model.clone(),
|
|
})
|
|
}
|
|
|
|
// ── Main ───────────────────────────────────────────────────────
|
|
|
|
#[tokio::main]
|
|
async fn main() {
|
|
tracing_subscriber::fmt()
|
|
.with_env_filter(
|
|
tracing_subscriber::EnvFilter::try_from_default_env()
|
|
.unwrap_or_else(|_| "crm_ai=info,tower_http=info".into()),
|
|
)
|
|
.init();
|
|
|
|
dotenvy::dotenv().ok();
|
|
|
|
let database_url =
|
|
std::env::var("DATABASE_URL").expect("DATABASE_URL must be set");
|
|
let ollama_url =
|
|
std::env::var("OLLAMA_BASE_URL").unwrap_or_else(|_| "http://localhost:11434".to_string());
|
|
let model = std::env::var("AI_MODEL").unwrap_or_else(|_| "sam860/dolphin3-llama3.2:3b".to_string());
|
|
let host = std::env::var("AI_HOST").unwrap_or_else(|_| "0.0.0.0".to_string());
|
|
let port: u16 = std::env::var("AI_PORT")
|
|
.unwrap_or_else(|_| "3001".to_string())
|
|
.parse()
|
|
.expect("AI_PORT must be a number");
|
|
|
|
// Load jobs
|
|
let jobs_path = std::env::var("JOBS_PATH").unwrap_or_else(|_| "data/ai/jobs.jsonl".to_string());
|
|
let jobs_content = fs::read_to_string(&jobs_path).unwrap_or_default();
|
|
let jobs: Vec<Job> = jobs_content
|
|
.lines()
|
|
.filter(|l| !l.trim().is_empty())
|
|
.filter_map(|l| serde_json::from_str(l).ok())
|
|
.collect();
|
|
|
|
info!(
|
|
"Loaded {} job categories, model: {}, Ollama: {}",
|
|
jobs.len(),
|
|
model,
|
|
ollama_url
|
|
);
|
|
|
|
// Connect to database
|
|
let db = PgPoolOptions::new()
|
|
.max_connections(5)
|
|
.connect(&database_url)
|
|
.await
|
|
.expect("Failed to connect to database");
|
|
|
|
info!("Connected to PostgreSQL");
|
|
|
|
let state = Arc::new(AppState {
|
|
db,
|
|
ollama_url,
|
|
model,
|
|
jobs,
|
|
});
|
|
|
|
// CORS layer
|
|
let cors = CorsLayer::new()
|
|
.allow_origin(Any)
|
|
.allow_methods([Method::GET, Method::POST])
|
|
.allow_headers(Any);
|
|
|
|
let app = Router::new()
|
|
.route("/ai/chat", post(handle_chat))
|
|
.route("/ai/jobs", get(handle_jobs))
|
|
.route("/health", get(handle_health))
|
|
.layer(cors)
|
|
.with_state(state);
|
|
|
|
let addr = format!("{}:{}", host, port);
|
|
info!("CRM AI server listening on {}", addr);
|
|
|
|
let listener = tokio::net::TcpListener::bind(&addr)
|
|
.await
|
|
.expect("Failed to bind address");
|
|
|
|
// Start Facebook scraper in a separate thread
|
|
let rng = Arc::new(Mutex::new(StdRng::from_entropy()));
|
|
tokio::task::spawn_blocking(move || {
|
|
loop {
|
|
{
|
|
let _lock = rng.lock().unwrap();
|
|
run_facebook_scraper();
|
|
}
|
|
// Introduce random delay
|
|
let delay = {
|
|
let mut rng = rng.lock().unwrap();
|
|
rng.gen_range(1..5)
|
|
};
|
|
thread::sleep(Duration::from_secs(delay));
|
|
}
|
|
});
|
|
|
|
axum::serve(listener, app)
|
|
.await
|
|
.expect("Server failed");
|
|
} |