JCBSComputer
109 lines
3.6 KiB
JavaScript
109 lines
3.6 KiB
JavaScript
const express = require('express');
|
|
const { chromium } = require('playwright');
|
|
|
|
const app = express();
|
|
app.use(express.json());
|
|
app.use((req, res, next) => {
|
|
res.header('Access-Control-Allow-Origin', '*');
|
|
res.header('Access-Control-Allow-Methods', 'GET, POST, OPTIONS');
|
|
res.header('Access-Control-Allow-Headers', 'Content-Type');
|
|
if (req.method === 'OPTIONS') return res.sendStatus(204);
|
|
next();
|
|
});
|
|
const PORT = process.env.PORT || 3007;
|
|
|
|
app.get('/health', (req, res) => {
|
|
res.json({ status: 'ok' });
|
|
});
|
|
|
|
app.post('/scrape/indeed', async (req, res) => {
|
|
const results = [];
|
|
let browser;
|
|
try {
|
|
browser = await chromium.launch({ headless: true });
|
|
const context = await browser.newContext({
|
|
userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
|
});
|
|
|
|
const queries = [
|
|
'need a website built',
|
|
'looking for someone to build my website',
|
|
'need web designer',
|
|
'looking for web developer',
|
|
'need help with my website',
|
|
'need ecommerce website',
|
|
'need wordpress website',
|
|
'looking for website designer',
|
|
];
|
|
|
|
for (const q of queries) {
|
|
const url = `https://www.indeed.com/jobs?q=${encodeURIComponent(q)}&sort=date`;
|
|
try {
|
|
const page = await context.newPage();
|
|
await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 20000 });
|
|
await page.waitForTimeout(2000);
|
|
|
|
await page.waitForTimeout(3000);
|
|
const items = await page.evaluate(() => {
|
|
const cards = document.querySelectorAll('.job_seen_beacon, [class*="card"], [class*="result"], li');
|
|
return Array.from(cards).slice(0, 10).map(card => {
|
|
const titleEl = card.querySelector('.jcs-JobTitle, a[class*="title"], a[class*="job"], h2 a');
|
|
const companyEl = card.querySelector('[data-testid="inlineHeader-companyName"], [class*="company"], [class*="comp"]');
|
|
const snippetEl = card.querySelector('.job-snippet, [class*="snippet"], [class*="summary"]');
|
|
return {
|
|
title: titleEl?.textContent?.trim() || '',
|
|
url: titleEl?.href || '',
|
|
company: companyEl?.textContent?.trim() || '',
|
|
snippet: snippetEl?.textContent?.trim()?.slice(0, 200) || '',
|
|
};
|
|
}).filter(j => j.title);
|
|
});
|
|
|
|
for (const item of items) {
|
|
if (item.title && item.url && !results.some(r => r.url === item.url)) {
|
|
results.push({
|
|
title: item.title,
|
|
url: item.url,
|
|
author: item.company,
|
|
date: '',
|
|
content: item.snippet,
|
|
source: 'indeed',
|
|
});
|
|
}
|
|
}
|
|
await page.close();
|
|
} catch (e) {
|
|
console.error(`Indeed failed ${q}:`, e.message);
|
|
}
|
|
}
|
|
|
|
await browser.close();
|
|
res.json(results.slice(0, 50));
|
|
} catch (e) {
|
|
if (browser) await browser.close().catch(() => {});
|
|
console.error('Indeed error:', e.message);
|
|
res.status(500).json({ error: e.message });
|
|
}
|
|
});
|
|
|
|
app.post('/scrape/all', async (req, res) => {
|
|
try {
|
|
const [indeed] = await Promise.allSettled([
|
|
fetch(`http://localhost:${PORT}/scrape/indeed`, { method: 'POST' }).then(r => r.json()).catch(() => []),
|
|
]);
|
|
|
|
const all = [
|
|
...(indeed.status === 'fulfilled' ? indeed.value : []),
|
|
];
|
|
console.log(`Scrape all: ${all.length} leads`);
|
|
res.json(all);
|
|
} catch (e) {
|
|
console.error('Scrape all error:', e.message);
|
|
res.status(500).json({ error: e.message });
|
|
}
|
|
});
|
|
|
|
app.listen(PORT, () => {
|
|
console.log(`Scraper service running on port ${PORT}`);
|
|
});
|