CRM_ENVIROMENT/CRM_ENVR
7
0
Fork 0
mirror of https://git.coastit.co.za/caitlin/CRM_ENVR.git synced 2026-07-10 11:15:43 +02:00
Code Issues Packages Projects Releases Wiki Activity

feat: add Facebook account tracking and management

Browse Source 
- Introduced new migration for `facebook_accounts` and `facebook_scrape_logs` tables.
- Updated the scraping logic to handle Facebook accounts, including success and failure tracking.
- Implemented API endpoints for managing Facebook accounts and their scrape logs.
- Enhanced user permissions to restrict access to Facebook account management to admins and super admins.
- Added a dialog component for displaying and managing Facebook accounts in the UI.
- Updated lead fetching logic to include user role checks for assignment and access.
This commit is contained in:
Ace
2026-06-23 14:18:18 +02:00
parent 1adc4806fa
commit ff56cea4b8
25 changed files with 778 additions and 216 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rust-ai/src/main.rs
+190 -50
View File
@@ -15,6 +15,7 @@ use tokio::sync::Mutex;
use tracing::{error, info, warn};
use uuid::Uuid;
use rand::Rng;
use chrono::Timelike;
use std::time::Duration;
use std::time::{SystemTime, UNIX_EPOCH};
@@ -93,6 +94,25 @@ struct LeadStore {
max_size: usize,
}
#[derive(Debug, Deserialize)]
struct ScrapeResponse {
success: bool,
leads: Vec<ScrapeLead>,
flagged: bool,
flag_reason: Option<String>,
error: Option<String>,
}
#[derive(Debug, Deserialize, Clone)]
struct ScrapeLead {
title: String,
url: String,
author: String,
date: String,
content: String,
source: Option<String>,
}
impl LeadStore {
fn new(max_size: usize) -> Self {
Self { leads: Vec::new(), max_size }
@@ -188,7 +208,7 @@ fn extract_claims(headers: &HeaderMap, state: &AppState) -> Result<Claims, (Stat
let claims = verify_jwt(token, &state.jwt_secret).ok_or_else(|| {
(StatusCode::UNAUTHORIZED, "Unauthorized".to_string())
})?;
match claims.role.as_str() {
match claims.role.to_lowercase().as_str() {
"sales" | "admin" | "super_admin" => Ok(claims),
_ => Err((StatusCode::FORBIDDEN, "Forbidden".to_string())),
}
@@ -266,30 +286,43 @@ async fn handle_chat(
let has_job = msg_words.contains(&"jobs") || msg_words.contains(&"job");
if has_listing || (has_show && has_job) || (has_show && msg_lower.contains("links")) || msg_lower.contains("recent leads") {
let now = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs();
let service_url = "http://localhost:3008/scrape/all".to_string();
if let Ok(resp) = state.http_client
.post(&service_url)
.timeout(Duration::from_secs(120))
.send()
.await
let base_url = "http://localhost:3008/scrape/facebook";
let mut req_builder = state.http_client.post(base_url).timeout(Duration::from_secs(300));
if let Ok(Some((_, path))) = sqlx::query_as::<_, (uuid::Uuid, String)>(
"SELECT id, profile_path FROM facebook_accounts \
WHERE is_active = TRUE AND flagged = FALSE \
ORDER BY last_scrape_at ASC NULLS FIRST LIMIT 1"
)
.fetch_optional(&state.db)
.await
{
if let Ok(leads_data) = resp.json::<Vec<serde_json::Value>>().await {
info!("Scraped {} leads from {}", leads_data.len(), service_url);
let mut store = state.leads.lock().await;
for item in &leads_data {
store.push(Lead {
title: truncate(item["title"].as_str().unwrap_or(""), 120),
url: item["url"].as_str().unwrap_or("").to_string(),
source: item["source"].as_str().unwrap_or("unknown").to_string(),
found_at: now,
author: truncate(item["author"].as_str().unwrap_or(""), 60),
date: truncate(item["date"].as_str().unwrap_or(""), 30),
content: truncate(item["content"].as_str().unwrap_or(""), 300),
});
req_builder = req_builder.query(&[("profile_path", &path), ("force", &"true".to_string())]);
} else {
warn!("No active Facebook account found for on-demand scrape");
}
match req_builder.send().await {
Ok(resp) => {
if let Ok(scrape_resp) = resp.json::<ScrapeResponse>().await {
info!("Scraped {} leads from Facebook", scrape_resp.leads.len());
let mut store = state.leads.lock().await;
for item in &scrape_resp.leads {
store.push(Lead {
title: truncate(&item.title, 120),
url: item.url.clone(),
source: item.source.clone().unwrap_or_else(|| "facebook".to_string()),
found_at: now,
author: truncate(&item.author, 60),
date: truncate(&item.date, 30),
content: truncate(&item.content, 300),
});
}
}
}
} else {
error!("Scraper service unreachable: {}", service_url);
Err(e) => {
error!("Scraper request error: {} - URL: {}", e, base_url);
}
}
let recent_leads = state.leads.lock().await.recent(604800, 20);
@@ -411,7 +444,7 @@ async fn main() {
info!("Connected to PostgreSQL");
let http_client = reqwest::Client::builder()
.timeout(Duration::from_secs(120))
.timeout(Duration::from_secs(300))
.build()
.expect("Failed to build HTTP client");
@@ -441,7 +474,7 @@ async fn main() {
.route("/ai/jobs", get(handle_jobs))
.route("/health", get(handle_health))
.layer(cors)
.with_state(state);
.with_state(state.clone());
let addr = format!("{}:{}", host, port);
info!("CRM AI server listening on {}", addr);
@@ -451,10 +484,11 @@ async fn main() {
.expect("Failed to bind address");
let bg_leads = lead_store.clone();
let bg_url = "http://localhost:3008/scrape/all".to_string();
let bg_db = state.db.clone();
let bg_url = "http://localhost:3008/scrape/facebook".to_string();
tokio::spawn(async move {
let client = match reqwest::Client::builder()
.timeout(Duration::from_secs(120))
.timeout(Duration::from_secs(300))
.build()
{
Ok(c) => c,
@@ -464,39 +498,145 @@ async fn main() {
}
};
loop {
// 10% random cycle skip — makes pattern non-periodic
if rand::thread_rng().gen_range(0..100) < 10 {
info!("Skipping this scrape cycle (random 10% skip)");
let jitter = rand::thread_rng().gen_range(16200..19800);
tokio::time::sleep(Duration::from_secs(jitter)).await;
continue;
}
// Skip night hours (23:00 – 06:00)
let hour = chrono::Local::now().hour();
if hour < 6 || hour >= 23 {
info!("Night hours ({}) — skipping scrape", hour);
let jitter = rand::thread_rng().gen_range(16200..19800);
tokio::time::sleep(Duration::from_secs(jitter)).await;
continue;
}
let now = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs();
match client.post(&bg_url).send().await {
Ok(resp) => {
if resp.status().is_success() {
match resp.json::<Vec<serde_json::Value>>().await {
Ok(data) => {
let mut store = bg_leads.lock().await;
for item in &data {
store.push(Lead {
title: truncate(item["title"].as_str().unwrap_or(""), 120),
url: item["url"].as_str().unwrap_or("").to_string(),
source: item["source"].as_str().unwrap_or("unknown").to_string(),
found_at: now,
author: truncate(item["author"].as_str().unwrap_or(""), 60),
date: truncate(item["date"].as_str().unwrap_or(""), 30),
content: truncate(item["content"].as_str().unwrap_or(""), 300),
});
// Pick next active un-flagged account
let account = sqlx::query_as::<_, (uuid::Uuid, String)>(
"SELECT id, profile_path FROM facebook_accounts \
WHERE is_active = TRUE AND flagged = FALSE \
ORDER BY last_scrape_at ASC NULLS FIRST LIMIT 1"
)
.fetch_optional(&bg_db)
.await;
match account {
Ok(Some((account_id, profile_path))) => {
match client.post(&bg_url).query(&[("profile_path", &profile_path)]).send().await {
Ok(resp) => {
if resp.status().is_success() {
match resp.json::<ScrapeResponse>().await {
Ok(data) => {
let leads_count = data.leads.len() as i32;
if data.flagged {
let _ = sqlx::query(
"UPDATE facebook_accounts SET flagged = TRUE, flagged_at = NOW(), \
flagged_reason = $2, last_error_at = NOW(), \
last_error_message = $3, consecutive_failures = consecutive_failures + 1 \
WHERE id = $1"
)
.bind(account_id)
.bind(&data.flag_reason)
.bind(&data.error)
.execute(&bg_db)
.await;
warn!("Facebook account {} flagged: {:?}", account_id, data.flag_reason);
let reason = data.flag_reason.as_deref().unwrap_or("unknown");
let _ = sqlx::query(
"INSERT INTO notifications (user_id, type, title, description, link) \
SELECT id, 'warning', 'Facebook Account Flagged', \
$1 || ' - ' || COALESCE($2, 'unknown reason'), \
NULL \
FROM users u JOIN user_roles ur ON ur.user_id = u.id \
JOIN roles r ON r.id = ur.role_id \
WHERE r.name IN ('ADMIN', 'SUPER_ADMIN')"
)
.bind(&account_id.to_string())
.bind(reason)
.execute(&bg_db)
.await;
} else if data.success {
let _ = sqlx::query(
"UPDATE facebook_accounts SET last_scrape_at = NOW(), \
last_success_at = NOW(), consecutive_failures = 0, \
updated_at = NOW() WHERE id = $1"
)
.bind(account_id)
.execute(&bg_db)
.await;
let mut store = bg_leads.lock().await;
for item in &data.leads {
store.push(Lead {
title: truncate(&item.title, 120),
url: item.url.clone(),
source: item.source.clone().unwrap_or_else(|| "facebook".to_string()),
found_at: now,
author: truncate(&item.author, 60),
date: truncate(&item.date, 30),
content: truncate(&item.content, 300),
});
}
info!("Scraped {} leads from Facebook account {}", leads_count, account_id);
} else {
// Increment failures; auto-flag if >= 3 consecutive
let _ = sqlx::query(
"UPDATE facebook_accounts SET last_error_at = NOW(), \
last_error_message = $2, consecutive_failures = consecutive_failures + 1, \
flagged = CASE WHEN consecutive_failures + 1 >= 3 THEN TRUE ELSE flagged END, \
flagged_reason = CASE WHEN consecutive_failures + 1 >= 3 THEN 'too_many_failures' ELSE flagged_reason END, \
flagged_at = CASE WHEN consecutive_failures + 1 >= 3 THEN NOW() ELSE flagged_at END, \
updated_at = NOW() WHERE id = $1"
)
.bind(account_id)
.bind(&data.error)
.execute(&bg_db)
.await;
warn!("Facebook scrape failed for account {}: {:?}", account_id, data.error);
}
let _ = sqlx::query(
"INSERT INTO facebook_scrape_logs \
(account_id, started_at, completed_at, success, leads_found, error_message, detected_flag) \
VALUES ($1, NOW() - interval '5 hours', NOW(), $2, $3, $4, $5)"
)
.bind(account_id)
.bind(data.success && !data.flagged)
.bind(leads_count)
.bind(&data.error)
.bind(&data.flag_reason)
.execute(&bg_db)
.await;
}
Err(e) => {
warn!("Failed to parse scraper JSON: {}", e);
}
}
}
Err(e) => {
warn!("Failed to parse scraper JSON: {}", e);
} else {
warn!("Scraper returned status: {}", resp.status());
}
}
} else {
warn!("Scraper returned status: {}", resp.status());
Err(e) => {
warn!("Scraper request failed: {}", e);
}
}
}
Ok(None) => {
info!("No active Facebook accounts available — skipping scrape cycle");
}
Err(e) => {
warn!("Scraper request failed: {}", e);
warn!("Failed to query Facebook accounts: {}", e);
}
}
let delay = rand::thread_rng().gen_range(120..300);
tokio::time::sleep(Duration::from_secs(delay)).await;
let jitter = rand::thread_rng().gen_range(16200..19800); // 4.5h – 5.5h
tokio::time::sleep(Duration::from_secs(jitter)).await;
}
});