Merge branch 'main' of https://git.coastit.co.za/caitlin/CRM_ENVR
# Conflicts: # package.json # rust-ai/target/.rustc_info.json # rust-ai/target/debug/.fingerprint/crm-ai-f6bc192b1f678367/bin-crm-ai # rust-ai/target/debug/.fingerprint/crm-ai-f6bc192b1f678367/bin-crm-ai.json # rust-ai/target/debug/build/crossbeam-utils-8066693d49c7288a/build-script-build.exe # rust-ai/target/debug/build/crossbeam-utils-8066693d49c7288a/build_script_build-8066693d49c7288a.exe # rust-ai/target/debug/build/crossbeam-utils-8066693d49c7288a/build_script_build-8066693d49c7288a.pdb # rust-ai/target/debug/build/crossbeam-utils-8066693d49c7288a/build_script_build.pdb # rust-ai/target/debug/build/generic-array-91b16dd255e0db51/build-script-build.exe # rust-ai/target/debug/build/generic-array-91b16dd255e0db51/build_script_build-91b16dd255e0db51.exe # rust-ai/target/debug/build/generic-array-91b16dd255e0db51/build_script_build-91b16dd255e0db51.pdb # rust-ai/target/debug/build/generic-array-91b16dd255e0db51/build_script_build.pdb # rust-ai/target/debug/build/getrandom-2c7c133266bf5cf5/build-script-build.exe # rust-ai/target/debug/build/getrandom-2c7c133266bf5cf5/build_script_build-2c7c133266bf5cf5.exe # rust-ai/target/debug/build/getrandom-2c7c133266bf5cf5/build_script_build-2c7c133266bf5cf5.pdb # rust-ai/target/debug/build/getrandom-2c7c133266bf5cf5/build_script_build.pdb # rust-ai/target/debug/build/httparse-4dc4630409fa7385/build-script-build.exe # rust-ai/target/debug/build/httparse-4dc4630409fa7385/build_script_build-4dc4630409fa7385.exe # rust-ai/target/debug/build/httparse-4dc4630409fa7385/build_script_build-4dc4630409fa7385.pdb # rust-ai/target/debug/build/httparse-4dc4630409fa7385/build_script_build.pdb # rust-ai/target/debug/build/icu_normalizer_data-96a52fd262d0f4c6/build-script-build.exe # rust-ai/target/debug/build/icu_normalizer_data-96a52fd262d0f4c6/build_script_build-96a52fd262d0f4c6.exe # rust-ai/target/debug/build/icu_normalizer_data-96a52fd262d0f4c6/build_script_build-96a52fd262d0f4c6.pdb # rust-ai/target/debug/build/icu_normalizer_data-96a52fd262d0f4c6/build_script_build.pdb # rust-ai/target/debug/build/icu_properties_data-cd63a06a3e531c28/build-script-build.exe # rust-ai/target/debug/build/icu_properties_data-cd63a06a3e531c28/build_script_build-cd63a06a3e531c28.exe # rust-ai/target/debug/build/icu_properties_data-cd63a06a3e531c28/build_script_build-cd63a06a3e531c28.pdb # rust-ai/target/debug/build/icu_properties_data-cd63a06a3e531c28/build_script_build.pdb # rust-ai/target/debug/build/libc-763fb040b2d663ab/build-script-build.exe # rust-ai/target/debug/build/libc-763fb040b2d663ab/build_script_build-763fb040b2d663ab.exe # rust-ai/target/debug/build/libc-763fb040b2d663ab/build_script_build-763fb040b2d663ab.pdb # rust-ai/target/debug/build/libc-763fb040b2d663ab/build_script_build.pdb # rust-ai/target/debug/build/native-tls-33399c41462f9840/build-script-build.exe # rust-ai/target/debug/build/native-tls-33399c41462f9840/build_script_build-33399c41462f9840.exe # rust-ai/target/debug/build/native-tls-33399c41462f9840/build_script_build-33399c41462f9840.pdb # rust-ai/target/debug/build/native-tls-33399c41462f9840/build_script_build.pdb # rust-ai/target/debug/build/num-traits-7471ce6e6595e59b/build-script-build.exe # rust-ai/target/debug/build/num-traits-7471ce6e6595e59b/build_script_build-7471ce6e6595e59b.exe # rust-ai/target/debug/build/num-traits-7471ce6e6595e59b/build_script_build-7471ce6e6595e59b.pdb # rust-ai/target/debug/build/num-traits-7471ce6e6595e59b/build_script_build.pdb # rust-ai/target/debug/build/parking_lot_core-12fc7eb0299b59cc/build-script-build.exe # rust-ai/target/debug/build/parking_lot_core-12fc7eb0299b59cc/build_script_build-12fc7eb0299b59cc.exe # rust-ai/target/debug/build/parking_lot_core-12fc7eb0299b59cc/build_script_build-12fc7eb0299b59cc.pdb # rust-ai/target/debug/build/parking_lot_core-12fc7eb0299b59cc/build_script_build.pdb # rust-ai/target/debug/build/proc-macro2-f29c3a04cd092e27/build-script-build.exe # rust-ai/target/debug/build/proc-macro2-f29c3a04cd092e27/build_script_build-f29c3a04cd092e27.exe # rust-ai/target/debug/build/proc-macro2-f29c3a04cd092e27/build_script_build-f29c3a04cd092e27.pdb # rust-ai/target/debug/build/proc-macro2-f29c3a04cd092e27/build_script_build.pdb # rust-ai/target/debug/build/quote-a01905ec43427c6e/build-script-build.exe # rust-ai/target/debug/build/quote-a01905ec43427c6e/build_script_build-a01905ec43427c6e.exe # rust-ai/target/debug/build/quote-a01905ec43427c6e/build_script_build-a01905ec43427c6e.pdb # rust-ai/target/debug/build/quote-a01905ec43427c6e/build_script_build.pdb # rust-ai/target/debug/build/rustversion-f8e8dfefa013a996/build-script-build.exe # rust-ai/target/debug/build/rustversion-f8e8dfefa013a996/build_script_build-f8e8dfefa013a996.exe # rust-ai/target/debug/build/rustversion-f8e8dfefa013a996/build_script_build-f8e8dfefa013a996.pdb # rust-ai/target/debug/build/rustversion-f8e8dfefa013a996/build_script_build.pdb # rust-ai/target/debug/build/serde-b4e8599ad30d0584/build-script-build.exe # rust-ai/target/debug/build/serde-b4e8599ad30d0584/build_script_build-b4e8599ad30d0584.exe # rust-ai/target/debug/build/serde-b4e8599ad30d0584/build_script_build-b4e8599ad30d0584.pdb # rust-ai/target/debug/build/serde-b4e8599ad30d0584/build_script_build.pdb # rust-ai/target/debug/build/serde_core-fdf5f60f2a35bbe7/build-script-build.exe # rust-ai/target/debug/build/serde_core-fdf5f60f2a35bbe7/build_script_build-fdf5f60f2a35bbe7.exe # rust-ai/target/debug/build/serde_core-fdf5f60f2a35bbe7/build_script_build-fdf5f60f2a35bbe7.pdb # rust-ai/target/debug/build/serde_core-fdf5f60f2a35bbe7/build_script_build.pdb # rust-ai/target/debug/build/serde_json-257d8c53e770cd03/build-script-build.exe # rust-ai/target/debug/build/serde_json-257d8c53e770cd03/build_script_build-257d8c53e770cd03.exe # rust-ai/target/debug/build/serde_json-257d8c53e770cd03/build_script_build-257d8c53e770cd03.pdb # rust-ai/target/debug/build/serde_json-257d8c53e770cd03/build_script_build.pdb # rust-ai/target/debug/build/thiserror-f1e01e19f7ffccfe/build-script-build.exe # rust-ai/target/debug/build/thiserror-f1e01e19f7ffccfe/build_script_build-f1e01e19f7ffccfe.exe # rust-ai/target/debug/build/thiserror-f1e01e19f7ffccfe/build_script_build-f1e01e19f7ffccfe.pdb # rust-ai/target/debug/build/thiserror-f1e01e19f7ffccfe/build_script_build.pdb # rust-ai/target/debug/build/zerocopy-c9c41e065706d9f3/build-script-build.exe # rust-ai/target/debug/build/zerocopy-c9c41e065706d9f3/build_script_build-c9c41e065706d9f3.exe # rust-ai/target/debug/build/zerocopy-c9c41e065706d9f3/build_script_build-c9c41e065706d9f3.pdb # rust-ai/target/debug/build/zerocopy-c9c41e065706d9f3/build_script_build.pdb # rust-ai/target/debug/build/zmij-bbaef9d295c9b9a4/build-script-build.exe # rust-ai/target/debug/build/zmij-bbaef9d295c9b9a4/build_script_build-bbaef9d295c9b9a4.exe # rust-ai/target/debug/build/zmij-bbaef9d295c9b9a4/build_script_build-bbaef9d295c9b9a4.pdb # rust-ai/target/debug/build/zmij-bbaef9d295c9b9a4/build_script_build.pdb # rust-ai/target/debug/crm-ai.exe # rust-ai/target/debug/crm_ai.pdb # rust-ai/target/debug/deps/async_trait-68aee6d09cf7ad62.dll # rust-ai/target/debug/deps/async_trait-68aee6d09cf7ad62.dll.exp # rust-ai/target/debug/deps/async_trait-68aee6d09cf7ad62.pdb # rust-ai/target/debug/deps/crm_ai.exe # rust-ai/target/debug/deps/crm_ai.pdb # rust-ai/target/debug/deps/displaydoc-df5efcb720a2661d.dll # rust-ai/target/debug/deps/displaydoc-df5efcb720a2661d.dll.exp # rust-ai/target/debug/deps/displaydoc-df5efcb720a2661d.pdb # rust-ai/target/debug/deps/rustversion-3bff64c9379d1bfc.dll # rust-ai/target/debug/deps/rustversion-3bff64c9379d1bfc.dll.exp # rust-ai/target/debug/deps/rustversion-3bff64c9379d1bfc.pdb # rust-ai/target/debug/deps/serde_derive-5075d2cdf4f787ec.dll # rust-ai/target/debug/deps/serde_derive-5075d2cdf4f787ec.dll.exp # rust-ai/target/debug/deps/serde_derive-5075d2cdf4f787ec.pdb # rust-ai/target/debug/deps/thiserror_impl-d7f41de49f2cc66d.dll # rust-ai/target/debug/deps/thiserror_impl-d7f41de49f2cc66d.dll.exp # rust-ai/target/debug/deps/thiserror_impl-d7f41de49f2cc66d.pdb # rust-ai/target/debug/deps/tokio_macros-afc286fa0daf0a5b.dll # rust-ai/target/debug/deps/tokio_macros-afc286fa0daf0a5b.dll.exp # rust-ai/target/debug/deps/tokio_macros-afc286fa0daf0a5b.pdb # rust-ai/target/debug/deps/tracing_attributes-c958f756e18a5ca5.dll # rust-ai/target/debug/deps/tracing_attributes-c958f756e18a5ca5.dll.exp # rust-ai/target/debug/deps/tracing_attributes-c958f756e18a5ca5.pdb # rust-ai/target/debug/deps/yoke_derive-1ee9709a84873876.dll # rust-ai/target/debug/deps/yoke_derive-1ee9709a84873876.dll.exp # rust-ai/target/debug/deps/yoke_derive-1ee9709a84873876.pdb # rust-ai/target/debug/deps/zerofrom_derive-84258ff2c2774032.dll # rust-ai/target/debug/deps/zerofrom_derive-84258ff2c2774032.dll.exp # rust-ai/target/debug/deps/zerofrom_derive-84258ff2c2774032.pdb # rust-ai/target/debug/deps/zerovec_derive-ca6ec60ddda5cee1.dll # rust-ai/target/debug/deps/zerovec_derive-ca6ec60ddda5cee1.dll.exp # rust-ai/target/debug/deps/zerovec_derive-ca6ec60ddda5cee1.pdb
This commit is contained in:
Generated
+137
-482
File diff suppressed because it is too large
Load Diff
+3
-3
@@ -6,7 +6,7 @@ description = "AI Sales Assistant backend for Coast IT CRM"
|
||||
|
||||
[dependencies]
|
||||
axum = "0.7"
|
||||
tokio = { version = "1", features = ["full"] }
|
||||
tokio = { version = "1", features = ["rt-multi-thread", "macros", "sync", "time", "net", "process"] }
|
||||
reqwest = { version = "0.12", features = ["json", "blocking"] }
|
||||
serde = { version = "1", features = ["derive"] }
|
||||
serde_json = "1"
|
||||
@@ -17,5 +17,5 @@ tracing = "0.1"
|
||||
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
|
||||
tower-http = { version = "0.5", features = ["cors"] }
|
||||
dotenvy = "0.15"
|
||||
scraper = "0.12"
|
||||
rand = "0.8"
|
||||
rand = "0.8"
|
||||
jsonwebtoken = "9"
|
||||
@@ -1,85 +0,0 @@
|
||||
use std::fs;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
use tokio::sync::RwLock;
|
||||
use tracing::{error, info, warn};
|
||||
|
||||
/// Manages the ai.md self-improvement file.
|
||||
/// Loaded on startup and periodically refreshed.
|
||||
pub struct InstructionsManager {
|
||||
path: PathBuf,
|
||||
current: RwLock<String>,
|
||||
}
|
||||
|
||||
impl InstructionsManager {
|
||||
pub fn new(path: PathBuf) -> Self {
|
||||
let initial = fs::read_to_string(&path).unwrap_or_default();
|
||||
if initial.is_empty() {
|
||||
warn!("ai.md is empty or missing at {:?}", path);
|
||||
} else {
|
||||
info!("Loaded ai.md ({} bytes)", initial.len());
|
||||
}
|
||||
Self {
|
||||
path,
|
||||
current: RwLock::new(initial),
|
||||
}
|
||||
}
|
||||
|
||||
/// Read the current instructions
|
||||
pub async fn get(&self) -> String {
|
||||
self.current.read().await.clone()
|
||||
}
|
||||
|
||||
/// Append a new entry to the Improvement Log and optionally update the instructions.
|
||||
/// Returns the updated full content.
|
||||
pub async fn update(&self, entry: &str, new_content: Option<&str>) -> Result<String, String> {
|
||||
if let Some(content) = new_content {
|
||||
// Full replacement
|
||||
fs::write(&self.path, content).map_err(|e| format!("Write failed: {}", e))?;
|
||||
let mut current = self.current.write().await;
|
||||
*current = content.to_string();
|
||||
info!("ai.md fully replaced ({} bytes)", content.len());
|
||||
Ok(content.to_string())
|
||||
} else {
|
||||
// Append to Improvement Log only
|
||||
let mut current = self.current.write().await;
|
||||
let log_entry = format!("\n- {} — {}", chrono::Utc::now().format("%Y-%m-%d %H:%M"), entry);
|
||||
// Find the Improvement Log section and append
|
||||
if let Some(pos) = current.rfind("\n## Improvement Log") {
|
||||
// Find the next section after Improvement Log, or end of file
|
||||
let after_log = ¤t[pos..];
|
||||
if let Some(section_start) = after_log[1..].find("\n## ") {
|
||||
let insert_at = pos + 1 + section_start;
|
||||
current.insert_str(insert_at, &format!("{}\n", log_entry));
|
||||
} else {
|
||||
current.push_str(&format!("{}\n", log_entry));
|
||||
}
|
||||
} else {
|
||||
current.push_str(&format!("\n## Improvement Log\n{}", log_entry));
|
||||
}
|
||||
fs::write(&self.path, &*current).map_err(|e| format!("Write failed: {}", e))?;
|
||||
info!("ai.md improvement log appended: {}", entry);
|
||||
Ok(current.clone())
|
||||
}
|
||||
}
|
||||
|
||||
/// Reload from disk
|
||||
pub async fn reload(&self) {
|
||||
match fs::read_to_string(&self.path) {
|
||||
Ok(content) => {
|
||||
let len = content.len();
|
||||
let mut current = self.current.write().await;
|
||||
*current = content;
|
||||
info!("ai.md reloaded from disk ({} bytes)", len);
|
||||
}
|
||||
Err(e) => error!("Failed to reload ai.md: {}", e),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Wrapper for thread-safe sharing
|
||||
pub type SharedInstructions = Arc<InstructionsManager>;
|
||||
|
||||
pub fn create_shared(path: PathBuf) -> SharedInstructions {
|
||||
Arc::new(InstructionsManager::new(path))
|
||||
}
|
||||
+284
-160
@@ -1,93 +1,67 @@
|
||||
use axum::{
|
||||
extract::State,
|
||||
http::Method,
|
||||
http::{HeaderMap, Method, StatusCode},
|
||||
routing::{get, post},
|
||||
Json, Router,
|
||||
};
|
||||
use tower_http::cors::{CorsLayer, AllowOrigin, Any};
|
||||
use jsonwebtoken::{decode, DecodingKey, Validation, Algorithm};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use sqlx::postgres::PgPoolOptions;
|
||||
use std::collections::HashMap;
|
||||
use std::fs;
|
||||
use std::sync::{Arc, Mutex};
|
||||
use tower_http::cors::{Any, CorsLayer};
|
||||
use tracing::{error, info};
|
||||
use std::sync::Arc;
|
||||
use tokio::sync::Mutex;
|
||||
use tracing::{error, info, warn};
|
||||
use uuid::Uuid;
|
||||
use reqwest::blocking::Client;
|
||||
use scraper::{Html, Selector};
|
||||
use rand::rngs::StdRng;
|
||||
use rand::SeedableRng;
|
||||
use rand::Rng;
|
||||
use std::time::Duration;
|
||||
use std::thread;
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
// ── Facebook Scraper ───────────────────────────────────────────
|
||||
// ── JWT Claims ────────────────────────────────────────────────
|
||||
|
||||
// List of proxies
|
||||
const PROXIES: &[&str] = &[
|
||||
"http://user:pass@192.168.1.1:8080",
|
||||
"http://user:pass@192.168.1.2:8080",
|
||||
"http://user:pass@192.168.1.3:8080",
|
||||
"http://user:pass@192.168.1.4:8080",
|
||||
"http://user:pass@192.168.1.5:8080",
|
||||
];
|
||||
|
||||
// List of user agents
|
||||
const USER_AGENTS: &[&str] = &[
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3",
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.1.2 Safari/605.1.15",
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:61.0) Gecko/20100101 Firefox/61.0",
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.1.2 Safari/605.1.15",
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36",
|
||||
// Add more user agents here
|
||||
];
|
||||
|
||||
fn get_random_proxy() -> String {
|
||||
let mut rng = rand::thread_rng();
|
||||
PROXIES[rng.gen_range(0..PROXIES.len())].to_string()
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct Claims {
|
||||
#[serde(rename = "userId")]
|
||||
user_id: String,
|
||||
role: String,
|
||||
}
|
||||
|
||||
fn get_random_user_agent() -> String {
|
||||
let mut rng = rand::thread_rng();
|
||||
USER_AGENTS[rng.gen_range(0..USER_AGENTS.len())].to_string()
|
||||
fn verify_jwt(token: &str, secret: &str) -> Option<Claims> {
|
||||
let key = DecodingKey::from_secret(secret.as_bytes());
|
||||
let validation = Validation::new(Algorithm::HS256);
|
||||
decode::<Claims>(token, &key, &validation).ok().map(|d| d.claims)
|
||||
}
|
||||
|
||||
fn scrape_conversations(url: &str) -> Result<Html, Box<dyn std::error::Error>> {
|
||||
let proxy = get_random_proxy();
|
||||
let user_agent = get_random_user_agent();
|
||||
let client = Client::builder()
|
||||
.proxy(reqwest::Proxy::all(proxy)?)
|
||||
.build()?;
|
||||
// ── Rate limiter ──────────────────────────────────────────────
|
||||
|
||||
let response = client.get(url)
|
||||
.header("User-Agent", user_agent)
|
||||
.send()?;
|
||||
|
||||
if response.status().is_success() {
|
||||
let html = response.text()?;
|
||||
Ok(Html::parse_document(&html))
|
||||
} else {
|
||||
Err(format!("Failed to retrieve the page: {}", response.status()).into())
|
||||
}
|
||||
struct RateLimiter {
|
||||
buckets: Mutex<HashMap<String, Vec<u64>>>,
|
||||
max_requests: usize,
|
||||
window_secs: u64,
|
||||
}
|
||||
|
||||
fn run_facebook_scraper() {
|
||||
let url = "https://www.facebook.com/search/top/?q=need%20website%20create";
|
||||
match scrape_conversations(url) {
|
||||
Ok(soup) => {
|
||||
// Example: Extract conversation data
|
||||
let selector = Selector::parse("div.conversation").unwrap();
|
||||
for element in soup.select(&selector) {
|
||||
println!("Conversation: {}", element.inner_html());
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
error!("Facebook scraper error: {}", e);
|
||||
impl RateLimiter {
|
||||
fn new(max_requests: usize, window_secs: u64) -> Self {
|
||||
Self {
|
||||
buckets: Mutex::new(HashMap::new()),
|
||||
max_requests,
|
||||
window_secs,
|
||||
}
|
||||
}
|
||||
|
||||
// Introduce random delay
|
||||
let mut rng = rand::thread_rng();
|
||||
let delay = rng.gen_range(1..5); // Delay between 1 to 5 seconds
|
||||
thread::sleep(Duration::from_secs(delay));
|
||||
async fn check(&self, key: &str) -> bool {
|
||||
let now = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs();
|
||||
let mut buckets = self.buckets.lock().await;
|
||||
let timestamps = buckets.entry(key.to_string()).or_default();
|
||||
timestamps.retain(|t| now - *t <= self.window_secs);
|
||||
if timestamps.len() >= self.max_requests {
|
||||
false
|
||||
} else {
|
||||
timestamps.push(now);
|
||||
true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ── Shared state ───────────────────────────────────────────────
|
||||
@@ -97,6 +71,48 @@ struct AppState {
|
||||
ollama_url: String,
|
||||
model: String,
|
||||
jobs: Vec<Job>,
|
||||
leads: Arc<Mutex<LeadStore>>,
|
||||
http_client: reqwest::Client,
|
||||
jwt_secret: String,
|
||||
rate_limiter: RateLimiter,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
struct Lead {
|
||||
title: String,
|
||||
url: String,
|
||||
source: String,
|
||||
found_at: u64,
|
||||
author: String,
|
||||
date: String,
|
||||
content: String,
|
||||
}
|
||||
|
||||
struct LeadStore {
|
||||
leads: Vec<Lead>,
|
||||
max_size: usize,
|
||||
}
|
||||
|
||||
impl LeadStore {
|
||||
fn new(max_size: usize) -> Self {
|
||||
Self { leads: Vec::new(), max_size }
|
||||
}
|
||||
|
||||
fn push(&mut self, lead: Lead) {
|
||||
if !self.leads.iter().any(|l| l.url == lead.url) {
|
||||
self.leads.insert(0, lead);
|
||||
self.leads.truncate(self.max_size);
|
||||
}
|
||||
}
|
||||
|
||||
fn recent(&self, max_age_secs: u64, limit: usize) -> Vec<Lead> {
|
||||
let now = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs();
|
||||
self.leads.iter()
|
||||
.filter(|l| now.saturating_sub(l.found_at) <= max_age_secs)
|
||||
.take(limit)
|
||||
.cloned()
|
||||
.collect()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
@@ -110,8 +126,6 @@ struct Job {
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct ChatRequest {
|
||||
message: String,
|
||||
user_id: String,
|
||||
user_role: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
@@ -162,23 +176,73 @@ struct OllamaResponseMessage {
|
||||
content: String,
|
||||
}
|
||||
|
||||
// ── System prompt builder ─────────────────────────────────────
|
||||
// ── Helpers ────────────────────────────────────────────────────
|
||||
|
||||
fn build_system_prompt(jobs: &[Job]) -> String {
|
||||
fn truncate(s: &str, max: usize) -> String {
|
||||
s.chars().take(max).collect()
|
||||
}
|
||||
|
||||
fn extract_claims(headers: &HeaderMap, state: &AppState) -> Result<Claims, (StatusCode, String)> {
|
||||
let auth_header = headers.get("Authorization").and_then(|v| v.to_str().ok()).unwrap_or("");
|
||||
let token = auth_header.strip_prefix("Bearer ").unwrap_or("");
|
||||
let claims = verify_jwt(token, &state.jwt_secret).ok_or_else(|| {
|
||||
(StatusCode::UNAUTHORIZED, "Unauthorized".to_string())
|
||||
})?;
|
||||
match claims.role.as_str() {
|
||||
"sales" | "admin" | "super_admin" => Ok(claims),
|
||||
_ => Err((StatusCode::FORBIDDEN, "Forbidden".to_string())),
|
||||
}
|
||||
}
|
||||
|
||||
fn format_leads_output(leads: &[Lead]) -> String {
|
||||
if leads.is_empty() {
|
||||
return "No new requests found yet.".to_string();
|
||||
}
|
||||
leads
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(i, l)| {
|
||||
let author = if l.author.is_empty() { "Unknown" } else { &l.author };
|
||||
let date = truncate(&l.date, 10);
|
||||
format!(
|
||||
"{}. {}\n {}\n {}\n {}",
|
||||
i + 1,
|
||||
author,
|
||||
date,
|
||||
l.title,
|
||||
l.url
|
||||
)
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n")
|
||||
}
|
||||
|
||||
fn build_system_prompt(jobs: &[Job], leads: &[Lead]) -> String {
|
||||
let job_list: Vec<String> = jobs
|
||||
.iter()
|
||||
.map(|j| format!("- {} ({}): {}", j.job_title, j.industry, j.description))
|
||||
.collect();
|
||||
let job_list_str = job_list.join("\n");
|
||||
|
||||
let lead_summary: Vec<String> = leads
|
||||
.iter()
|
||||
.map(|l| format!("{} | {} | {}", l.author, l.title, l.url))
|
||||
.collect();
|
||||
let lead_summary_str = if lead_summary.is_empty() {
|
||||
"None yet.".to_string()
|
||||
} else {
|
||||
lead_summary.join("\n")
|
||||
};
|
||||
|
||||
format!(
|
||||
"You are a Sales AI Assistant for Coast IT CRM. Your role is to help salespeople \
|
||||
with tips, strategies, and guidance.\n\n\
|
||||
"You are a Sales AI Assistant for Coast IT CRM.\n\n\
|
||||
Available job categories to target:\n{}\n\n\
|
||||
Provide concise, actionable sales advice. When asked about a specific job category, \
|
||||
give targeted tips on finding and engaging prospects in that field. \
|
||||
Keep responses under 300 words unless asked for detail.",
|
||||
job_list_str
|
||||
Recent leads context:\n{}\n\n\
|
||||
Rules:\n\
|
||||
- When asked about leads, answer concisely under 150 words.\n\
|
||||
- If asked to suggest a sales strategy, give brief actionable advice.\n\
|
||||
- Be direct and professional. No fluff.",
|
||||
job_list_str, lead_summary_str
|
||||
)
|
||||
}
|
||||
|
||||
@@ -186,90 +250,119 @@ fn build_system_prompt(jobs: &[Job]) -> String {
|
||||
|
||||
async fn handle_chat(
|
||||
State(state): State<Arc<AppState>>,
|
||||
headers: HeaderMap,
|
||||
Json(req): Json<ChatRequest>,
|
||||
) -> Result<Json<ChatResponse>, (axum::http::StatusCode, String)> {
|
||||
// Validate role
|
||||
match req.user_role.as_str() {
|
||||
"sales" | "admin" | "super_admin" => {}
|
||||
_ => return Err((axum::http::StatusCode::FORBIDDEN, "Forbidden".to_string())),
|
||||
) -> Result<Json<ChatResponse>, (StatusCode, String)> {
|
||||
let claims = extract_claims(&headers, &state)?;
|
||||
|
||||
if !state.rate_limiter.check(&claims.user_id).await {
|
||||
return Err((StatusCode::TOO_MANY_REQUESTS, "Rate limit exceeded".to_string()));
|
||||
}
|
||||
|
||||
let system_prompt = build_system_prompt(&state.jobs);
|
||||
let msg_lower = req.message.to_lowercase();
|
||||
let msg_words: Vec<&str> = msg_lower.split_whitespace().collect();
|
||||
let has_listing = msg_words.iter().any(|w| ["listings", "listing", "leads", "links", "lists"].contains(w));
|
||||
let has_show = msg_lower.contains("show me") || msg_lower.contains("give me") || msg_lower.contains("pull");
|
||||
let has_job = msg_words.contains(&"jobs") || msg_words.contains(&"job");
|
||||
if has_listing || (has_show && has_job) || (has_show && msg_lower.contains("links")) || msg_lower.contains("recent leads") {
|
||||
let now = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs();
|
||||
let service_url = "http://localhost:3008/scrape/all".to_string();
|
||||
if let Ok(resp) = state.http_client
|
||||
.post(&service_url)
|
||||
.timeout(Duration::from_secs(120))
|
||||
.send()
|
||||
.await
|
||||
{
|
||||
if let Ok(leads_data) = resp.json::<Vec<serde_json::Value>>().await {
|
||||
info!("Scraped {} leads from {}", leads_data.len(), service_url);
|
||||
let mut store = state.leads.lock().await;
|
||||
for item in &leads_data {
|
||||
store.push(Lead {
|
||||
title: truncate(item["title"].as_str().unwrap_or(""), 120),
|
||||
url: item["url"].as_str().unwrap_or("").to_string(),
|
||||
source: item["source"].as_str().unwrap_or("unknown").to_string(),
|
||||
found_at: now,
|
||||
author: truncate(item["author"].as_str().unwrap_or(""), 60),
|
||||
date: truncate(item["date"].as_str().unwrap_or(""), 30),
|
||||
content: truncate(item["content"].as_str().unwrap_or(""), 300),
|
||||
});
|
||||
}
|
||||
}
|
||||
} else {
|
||||
error!("Scraper service unreachable: {}", service_url);
|
||||
}
|
||||
|
||||
let recent_leads = state.leads.lock().await.recent(604800, 20);
|
||||
let response = format_leads_output(&recent_leads);
|
||||
let _ = sqlx::query(
|
||||
"INSERT INTO ai_conversations (id, user_id, role, message, response) VALUES ($1, $2, $3, $4, $5)",
|
||||
)
|
||||
.bind(Uuid::new_v4())
|
||||
.bind(Uuid::parse_str(&claims.user_id).unwrap_or(Uuid::nil()))
|
||||
.bind(&claims.role)
|
||||
.bind(&req.message)
|
||||
.bind(&response)
|
||||
.execute(&state.db)
|
||||
.await;
|
||||
return Ok(Json(ChatResponse { response }));
|
||||
}
|
||||
|
||||
let recent_leads = state.leads.lock().await.recent(604800, 20);
|
||||
let system_prompt = build_system_prompt(&state.jobs, &recent_leads);
|
||||
let message_text = req.message.clone();
|
||||
|
||||
let ollama_req = OllamaRequest {
|
||||
model: state.model.clone(),
|
||||
messages: vec![
|
||||
OllamaChatMessage {
|
||||
role: "system".to_string(),
|
||||
content: system_prompt,
|
||||
},
|
||||
OllamaChatMessage {
|
||||
role: "user".to_string(),
|
||||
content: req.message.clone(),
|
||||
},
|
||||
OllamaChatMessage { role: "system".to_string(), content: system_prompt },
|
||||
OllamaChatMessage { role: "user".to_string(), content: req.message.clone() },
|
||||
],
|
||||
stream: false,
|
||||
options: OllamaOptions {
|
||||
temperature: 0.7,
|
||||
num_predict: 1024,
|
||||
},
|
||||
options: OllamaOptions { temperature: 0.7, num_predict: 1024 },
|
||||
};
|
||||
|
||||
let client = reqwest::Client::new();
|
||||
let resp = client
|
||||
let resp = state.http_client
|
||||
.post(format!("{}/api/chat", state.ollama_url))
|
||||
.json(&ollama_req)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| {
|
||||
error!("Ollama request failed: {}", e);
|
||||
(
|
||||
axum::http::StatusCode::SERVICE_UNAVAILABLE,
|
||||
"AI service unavailable".to_string(),
|
||||
)
|
||||
(StatusCode::SERVICE_UNAVAILABLE, "AI service unavailable".to_string())
|
||||
})?;
|
||||
|
||||
let ollama_resp: OllamaResponse = resp.json().await.map_err(|e| {
|
||||
error!("Failed to parse Ollama response: {}", e);
|
||||
(
|
||||
axum::http::StatusCode::SERVICE_UNAVAILABLE,
|
||||
"AI response parse error".to_string(),
|
||||
)
|
||||
(StatusCode::SERVICE_UNAVAILABLE, "AI response parse error".to_string())
|
||||
})?;
|
||||
|
||||
let response_text = ollama_resp
|
||||
.message
|
||||
.map(|m| m.content)
|
||||
.unwrap_or_default();
|
||||
let response_text = ollama_resp.message.map(|m| m.content).unwrap_or_default();
|
||||
|
||||
// Store in database
|
||||
let user_id = Uuid::parse_str(&req.user_id).unwrap_or(Uuid::nil());
|
||||
let _ = sqlx::query(
|
||||
"INSERT INTO ai_conversations (id, user_id, role, message, response) VALUES ($1, $2, $3, $4, $5)",
|
||||
)
|
||||
.bind(Uuid::new_v4())
|
||||
.bind(user_id)
|
||||
.bind(&req.user_role)
|
||||
.bind(Uuid::parse_str(&claims.user_id).unwrap_or(Uuid::nil()))
|
||||
.bind(&claims.role)
|
||||
.bind(&message_text)
|
||||
.bind(&response_text)
|
||||
.execute(&state.db)
|
||||
.await;
|
||||
|
||||
Ok(Json(ChatResponse {
|
||||
response: response_text,
|
||||
}))
|
||||
Ok(Json(ChatResponse { response: response_text }))
|
||||
}
|
||||
|
||||
// ── Jobs handler ───────────────────────────────────────────────
|
||||
|
||||
async fn handle_jobs(
|
||||
State(state): State<Arc<AppState>>,
|
||||
) -> Json<JobsResponse> {
|
||||
Json(JobsResponse {
|
||||
jobs: state.jobs.clone(),
|
||||
})
|
||||
headers: HeaderMap,
|
||||
) -> Result<Json<JobsResponse>, (StatusCode, String)> {
|
||||
let _claims = extract_claims(&headers, &state)?;
|
||||
if !state.rate_limiter.check(&_claims.user_id).await {
|
||||
return Err((StatusCode::TOO_MANY_REQUESTS, "Rate limit exceeded".to_string()));
|
||||
}
|
||||
Ok(Json(JobsResponse { jobs: state.jobs.clone() }))
|
||||
}
|
||||
|
||||
// ── Health handler ─────────────────────────────────────────────
|
||||
@@ -296,52 +389,50 @@ async fn main() {
|
||||
|
||||
dotenvy::dotenv().ok();
|
||||
|
||||
let database_url =
|
||||
std::env::var("DATABASE_URL").expect("DATABASE_URL must be set");
|
||||
let ollama_url =
|
||||
std::env::var("OLLAMA_BASE_URL").unwrap_or_else(|_| "http://localhost:11434".to_string());
|
||||
let model = std::env::var("AI_MODEL").unwrap_or_else(|_| "sam860/dolphin3-llama3.2:3b".to_string());
|
||||
let host = std::env::var("AI_HOST").unwrap_or_else(|_| "0.0.0.0".to_string());
|
||||
let port: u16 = std::env::var("AI_PORT")
|
||||
.unwrap_or_else(|_| "3001".to_string())
|
||||
.parse()
|
||||
.expect("AI_PORT must be a number");
|
||||
let database_url = std::env::var("DATABASE_URL").expect("DATABASE_URL must be set");
|
||||
let jwt_secret = std::env::var("JWT_SECRET").expect("JWT_SECRET must be set");
|
||||
let ollama_url = std::env::var("OLLAMA_BASE_URL").unwrap_or_else(|_| "http://localhost:11434".to_string());
|
||||
let model = std::env::var("AI_MODEL").unwrap_or_else(|_| "dolphin-phi".to_string());
|
||||
let host = std::env::var("AI_HOST").unwrap_or_else(|_| "127.0.0.1".to_string());
|
||||
let port: u16 = std::env::var("AI_PORT").unwrap_or_else(|_| "3001".to_string()).parse().expect("AI_PORT must be a number");
|
||||
|
||||
// Load jobs
|
||||
let jobs_path = std::env::var("JOBS_PATH").unwrap_or_else(|_| "data/ai/jobs.jsonl".to_string());
|
||||
let jobs_content = fs::read_to_string(&jobs_path).unwrap_or_default();
|
||||
let jobs: Vec<Job> = jobs_content
|
||||
.lines()
|
||||
.filter(|l| !l.trim().is_empty())
|
||||
.filter_map(|l| serde_json::from_str(l).ok())
|
||||
.collect();
|
||||
let jobs: Vec<Job> = jobs_content.lines().filter(|l| !l.trim().is_empty()).filter_map(|l| serde_json::from_str(l).ok()).collect();
|
||||
|
||||
info!(
|
||||
"Loaded {} job categories, model: {}, Ollama: {}",
|
||||
jobs.len(),
|
||||
model,
|
||||
ollama_url
|
||||
);
|
||||
info!("Loaded {} job categories, model: {}, Ollama: {}", jobs.len(), model, ollama_url);
|
||||
|
||||
// Connect to database
|
||||
let db = PgPoolOptions::new()
|
||||
.max_connections(5)
|
||||
.max_connections(20)
|
||||
.connect(&database_url)
|
||||
.await
|
||||
.expect("Failed to connect to database");
|
||||
|
||||
info!("Connected to PostgreSQL");
|
||||
|
||||
let http_client = reqwest::Client::builder()
|
||||
.timeout(Duration::from_secs(120))
|
||||
.build()
|
||||
.expect("Failed to build HTTP client");
|
||||
|
||||
let lead_store = Arc::new(Mutex::new(LeadStore::new(100)));
|
||||
|
||||
let state = Arc::new(AppState {
|
||||
db,
|
||||
ollama_url,
|
||||
model,
|
||||
jobs,
|
||||
leads: lead_store.clone(),
|
||||
http_client,
|
||||
jwt_secret,
|
||||
rate_limiter: RateLimiter::new(30, 60),
|
||||
});
|
||||
|
||||
// CORS layer
|
||||
let cors = CorsLayer::new()
|
||||
.allow_origin(Any)
|
||||
.allow_origin(AllowOrigin::list([
|
||||
"http://localhost:3006".parse().unwrap(),
|
||||
"http://127.0.0.1:3006".parse().unwrap(),
|
||||
]))
|
||||
.allow_methods([Method::GET, Method::POST])
|
||||
.allow_headers(Any);
|
||||
|
||||
@@ -359,24 +450,57 @@ async fn main() {
|
||||
.await
|
||||
.expect("Failed to bind address");
|
||||
|
||||
// Start Facebook scraper in a separate thread
|
||||
let rng = Arc::new(Mutex::new(StdRng::from_entropy()));
|
||||
tokio::task::spawn_blocking(move || {
|
||||
loop {
|
||||
{
|
||||
let _lock = rng.lock().unwrap();
|
||||
run_facebook_scraper();
|
||||
let bg_leads = lead_store.clone();
|
||||
let bg_url = "http://localhost:3008/scrape/all".to_string();
|
||||
tokio::spawn(async move {
|
||||
let client = match reqwest::Client::builder()
|
||||
.timeout(Duration::from_secs(120))
|
||||
.build()
|
||||
{
|
||||
Ok(c) => c,
|
||||
Err(e) => {
|
||||
error!("Failed to build background HTTP client: {} — scraper disabled", e);
|
||||
return;
|
||||
}
|
||||
// Introduce random delay
|
||||
let delay = {
|
||||
let mut rng = rng.lock().unwrap();
|
||||
rng.gen_range(1..5)
|
||||
};
|
||||
thread::sleep(Duration::from_secs(delay));
|
||||
};
|
||||
loop {
|
||||
let now = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs();
|
||||
match client.post(&bg_url).send().await {
|
||||
Ok(resp) => {
|
||||
if resp.status().is_success() {
|
||||
match resp.json::<Vec<serde_json::Value>>().await {
|
||||
Ok(data) => {
|
||||
let mut store = bg_leads.lock().await;
|
||||
for item in &data {
|
||||
store.push(Lead {
|
||||
title: truncate(item["title"].as_str().unwrap_or(""), 120),
|
||||
url: item["url"].as_str().unwrap_or("").to_string(),
|
||||
source: item["source"].as_str().unwrap_or("unknown").to_string(),
|
||||
found_at: now,
|
||||
author: truncate(item["author"].as_str().unwrap_or(""), 60),
|
||||
date: truncate(item["date"].as_str().unwrap_or(""), 30),
|
||||
content: truncate(item["content"].as_str().unwrap_or(""), 300),
|
||||
});
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
warn!("Failed to parse scraper JSON: {}", e);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
warn!("Scraper returned status: {}", resp.status());
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
warn!("Scraper request failed: {}", e);
|
||||
}
|
||||
}
|
||||
let delay = rand::thread_rng().gen_range(120..300);
|
||||
tokio::time::sleep(Duration::from_secs(delay)).await;
|
||||
}
|
||||
});
|
||||
|
||||
axum::serve(listener, app)
|
||||
.await
|
||||
.expect("Server failed");
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user