diff --git a/scraper-service/index.js b/scraper-service/index.js index 46565c9..83ac473 100644 --- a/scraper-service/index.js +++ b/scraper-service/index.js @@ -3,6 +3,13 @@ const { chromium } = require('playwright'); const app = express(); app.use(express.json()); +app.use((req, res, next) => { + res.header('Access-Control-Allow-Origin', '*'); + res.header('Access-Control-Allow-Methods', 'GET, POST, OPTIONS'); + res.header('Access-Control-Allow-Headers', 'Content-Type'); + if (req.method === 'OPTIONS') return res.sendStatus(204); + next(); +}); const PORT = process.env.PORT || 3007; app.get('/health', (req, res) => { @@ -36,19 +43,20 @@ app.post('/scrape/indeed', async (req, res) => { await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 20000 }); await page.waitForTimeout(2000); + await page.waitForTimeout(3000); const items = await page.evaluate(() => { - const cards = document.querySelectorAll('.job_seen_beacon'); - return Array.from(cards).slice(0, 8).map(card => { - const titleEl = card.querySelector('.jcs-JobTitle'); - const companyEl = card.querySelector('[data-testid="inlineHeader-companyName"]'); - const snippetEl = card.querySelector('.job-snippet'); + const cards = document.querySelectorAll('.job_seen_beacon, [class*="card"], [class*="result"], li'); + return Array.from(cards).slice(0, 10).map(card => { + const titleEl = card.querySelector('.jcs-JobTitle, a[class*="title"], a[class*="job"], h2 a'); + const companyEl = card.querySelector('[data-testid="inlineHeader-companyName"], [class*="company"], [class*="comp"]'); + const snippetEl = card.querySelector('.job-snippet, [class*="snippet"], [class*="summary"]'); return { title: titleEl?.textContent?.trim() || '', url: titleEl?.href || '', company: companyEl?.textContent?.trim() || '', snippet: snippetEl?.textContent?.trim()?.slice(0, 200) || '', }; - }); + }).filter(j => j.title); }); for (const item of items) { diff --git a/scraper-service/node_modules/.package-lock.json b/scraper-service/node_modules/.package-lock.json index 7873a80..80f4ce9 100644 --- a/scraper-service/node_modules/.package-lock.json +++ b/scraper-service/node_modules/.package-lock.json @@ -558,12 +558,11 @@ "license": "MIT" }, "node_modules/playwright": { - "version": "1.61.0", - "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.61.0.tgz", - "integrity": "sha512-Z+7BeeqQPRRzklHsVFP4KTGIyMxKUmfeRA4WisM6G3/XW6nwGeX6fX9qYaDa+CiUqpOkb2f6X3nar05R3kSuJQ==", - "license": "Apache-2.0", + "version": "1.61.1", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.61.1.tgz", + "integrity": "sha512-DWnY5o3YbLWK4GovuAVwpqL+1VwGNdUGrRr++8j8PtQQzvAVZUIMjKQ90fY689sEJZJBbZVw1rXaOKSTitkzPQ==", "dependencies": { - "playwright-core": "1.61.0" + "playwright-core": "1.61.1" }, "bin": { "playwright": "cli.js" @@ -576,10 +575,9 @@ } }, "node_modules/playwright-core": { - "version": "1.61.0", - "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.61.0.tgz", - "integrity": "sha512-caX7TrY3Ml6egyDX0WUcTHDxodl/b51y5wJOdCEA36QviK/s2g081hvmGs8eaE3DWb6NYZQ6BjO/QkNRPenoPA==", - "license": "Apache-2.0", + "version": "1.61.1", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.61.1.tgz", + "integrity": "sha512-h7Qlt6m4REp25qvIdvbDtVmD4LqVXfpRxhORv9L0jzETM05p4fuPJ3dKyuSXQxDSbXnmS79HAgi9589lGSpLkg==", "bin": { "playwright-core": "cli.js" }, diff --git a/scraper-service/node_modules/playwright-core/lib/coreBundle.js b/scraper-service/node_modules/playwright-core/lib/coreBundle.js index 1d36a37..7d8468f 100644 --- a/scraper-service/node_modules/playwright-core/lib/coreBundle.js +++ b/scraper-service/node_modules/playwright-core/lib/coreBundle.js @@ -23831,6 +23831,7 @@ var init_harTracer = __esm({ this._pageEntries = []; this._eventListeners = []; this._started = false; + this._omitWebSocketFrames = true; this._context = context2; this._page = page; this._delegate = delegate; @@ -23847,6 +23848,9 @@ var init_harTracer = __esm({ this._pageEntrySymbol = Symbol("pageHarEntry"); this._baseURL = context2 instanceof APIRequestContext ? context2._defaultOptions().baseURL : context2._options.baseURL; } + setOmitWebSocketFrames(omitWebSocketFrames) { + this._omitWebSocketFrames = omitWebSocketFrames; + } start(options) { if (this._started) return; @@ -24157,7 +24161,7 @@ var init_harTracer = __esm({ harEntry._resourceType = "websocket"; let sha1 = void 0; const recordMessage = (type3, opcode, data, wallTimeMs) => { - if (this._options.omitWebSocketFrames) + if (this._omitWebSocketFrames) return; const message = { type: type3, time: this._options.omitTiming ? -1 : wallTimeMs, opcode, data }; if (this._options.content === "embed") { @@ -24441,9 +24445,9 @@ var init_harRecorder = __esm({ includeTraceInfo: false, recordRequestOverrides: true, waitForContentOnStop: true, - urlFilter: urlFilterRe ?? options.urlGlob, - omitWebSocketFrames: !!process.env.PLAYWRIGHT_HAR_NO_WEBSOCKET_FRAMES + urlFilter: urlFilterRe ?? options.urlGlob }); + this._tracer.setOmitWebSocketFrames(!!process.env.PLAYWRIGHT_HAR_NO_WEBSOCKET_FRAMES); this._tracer.start({ omitScripts: false }); } onEntryStarted(entry) { @@ -24652,8 +24656,7 @@ var init_tracing = __esm({ content: "attach", includeTraceInfo: true, recordRequestOverrides: false, - waitForContentOnStop: false, - omitWebSocketFrames: !!process.env.PLAYWRIGHT_TRACING_NO_WEBSOCKET_FRAMES + waitForContentOnStop: false }); const testIdAttributeName2 = "selectors" in context2 ? context2.selectors().testIdAttributeName() : void 0; this._contextCreatedEvent = { @@ -24747,6 +24750,7 @@ var init_tracing = __esm({ ); if (this._state.options.screenshots) this._startScreencast(); + this._harTracer.setOmitWebSocketFrames(!!process.env.PLAYWRIGHT_TRACING_NO_WEBSOCKET_FRAMES); if (this._state.options.snapshots) await this._snapshotter?.start(progress2); return { traceName: this._state.traceName }; @@ -24896,6 +24900,7 @@ var init_tracing = __esm({ eventsHelper.removeEventListeners(this._eventListeners); if (this._state.options.screenshots) this._stopScreencast(); + this._harTracer.setOmitWebSocketFrames(true); if (this._state.options.snapshots) this._snapshotter?.stop(); this.flushHarEntries(); @@ -35989,7 +35994,7 @@ var init_crNetworkManager = __esm({ } _onWebSocketWillSendHandshakeRequest(event) { const wallTimeMs = event.wallTime * 1e3; - this._timestampBaselineForWebSocket.set(event.requestId, wallTimeMs - event.timestamp); + this._timestampBaselineForWebSocket.set(event.requestId, wallTimeMs - event.timestamp * 1e3); this._page.frameManager.onWebSocketRequest(event.requestId, headersObjectToArray(event.request.headers, "\n"), wallTimeMs); } _onWebSocketClosed(event) { @@ -35997,7 +36002,7 @@ var init_crNetworkManager = __esm({ this._page.frameManager.webSocketClosed(event.requestId); } _timestampToWallTimeMsForWebSocket(requestId, timestamp) { - return this._timestampBaselineForWebSocket.get(requestId) + timestamp; + return this._timestampBaselineForWebSocket.get(requestId) + timestamp * 1e3; } _maybeUpdateRequestSession(sessionInfo, request2) { if (request2.session !== sessionInfo.session && !sessionInfo.isMain && (request2._documentId === request2._requestId || sessionInfo.workerFrame)) @@ -44350,15 +44355,11 @@ var init_ffPage = __esm({ this._page.frameManager.frameAbortedNavigation(params2.frameId, params2.errorText, params2.navigationId); } _onNavigationCommitted(params2) { - if (!params2.navigationId) { - this._page.frameManager.frameCommittedSameDocumentNavigation(params2.frameId, params2.url); - return; - } for (const [workerId, worker] of this._workers) { if (worker.frameId === params2.frameId) this._onWorkerDestroyed({ workerId }); } - this._page.frameManager.frameCommittedNewDocumentNavigation(params2.frameId, params2.url, params2.name || "", params2.navigationId, false); + this._page.frameManager.frameCommittedNewDocumentNavigation(params2.frameId, params2.url, params2.name || "", params2.navigationId || "", false); } _onSameDocumentNavigation(params2) { this._page.frameManager.frameCommittedSameDocumentNavigation(params2.frameId, params2.url); @@ -46941,7 +46942,7 @@ var init_wkPage = __esm({ } _onWebSocketWillSendHandshakeRequest(event) { const wallTimeMs = event.walltime * 1e3; - this._timestampBaselineForWebSocket.set(event.requestId, wallTimeMs - event.timestamp); + this._timestampBaselineForWebSocket.set(event.requestId, wallTimeMs - event.timestamp * 1e3); this._page.frameManager.onWebSocketRequest(event.requestId, headersObjectToArray(event.request.headers), wallTimeMs); } _onWebSocketClosed(event) { @@ -46949,7 +46950,7 @@ var init_wkPage = __esm({ this._page.frameManager.webSocketClosed(event.requestId); } _timestampToWallTimeMsForWebSocket(requestId, timestamp) { - return this._timestampBaselineForWebSocket.get(requestId) + timestamp; + return this._timestampBaselineForWebSocket.get(requestId) + timestamp * 1e3; } async _grantPermissions(origin, permissions) { const webPermissionToProtocol = /* @__PURE__ */ new Map([ @@ -49287,7 +49288,7 @@ var init_wvPage = __esm({ } _onWebSocketWillSendHandshakeRequest(event) { const wallTimeMs = event.walltime * 1e3; - this._timestampBaselineForWebSocket.set(event.requestId, wallTimeMs - event.timestamp); + this._timestampBaselineForWebSocket.set(event.requestId, wallTimeMs - event.timestamp * 1e3); this._page.frameManager.onWebSocketRequest(event.requestId, headersObjectToArray(event.request.headers), wallTimeMs); } _onWebSocketClosed(event) { @@ -49295,7 +49296,7 @@ var init_wvPage = __esm({ this._page.frameManager.webSocketClosed(event.requestId); } _timestampToWallTimeMsForWebSocket(requestId, timestamp) { - return this._timestampBaselineForWebSocket.get(requestId) + timestamp; + return this._timestampBaselineForWebSocket.get(requestId) + timestamp * 1e3; } shouldToggleStyleSheetToSyncAnimations() { return true; diff --git a/scraper-service/node_modules/playwright-core/lib/vite/traceViewer/index.DEBl1tfz.js b/scraper-service/node_modules/playwright-core/lib/vite/traceViewer/index.DEBl1tfz.js deleted file mode 100644 index afb6132..0000000 --- a/scraper-service/node_modules/playwright-core/lib/vite/traceViewer/index.DEBl1tfz.js +++ /dev/null @@ -1,2 +0,0 @@ -import"./assets/urlMatch-BYQrIQwR.js";import{T as x,r,a as C,W as F,j as e,D as z,b as I,c as P,d as A,e as O,f as V}from"./assets/defaultSettingsView-BNmKHKpQ.js";const $=()=>{const[o,c]=r.useState(!1),[n,l]=r.useState(),[h,u]=r.useState(),[p,b]=r.useState(N),[f,j]=r.useState({done:0,total:0}),[k,v]=r.useState(!1),[y,m]=r.useState(null),[T,R]=r.useState(null),[U,E]=r.useState(!1),g=r.useCallback(t=>{const s=new URL(window.location.href);if(!t.length)return;const i=t.item(0),a=URL.createObjectURL(i);s.searchParams.append("trace",a);const d=s.toString();window.history.pushState({},"",d),l(a),u(i.name),v(!1),m(null)},[]);r.useEffect(()=>{const t=async s=>{var a;if(!((a=s.clipboardData)!=null&&a.files.length))return;const i=["application/zip","application/x-zip-compressed"];for(const d of s.clipboardData.files)if(!i.includes(d.type))return;s.preventDefault(),g(s.clipboardData.files)};return document.addEventListener("paste",t),()=>document.removeEventListener("paste",t)}),r.useEffect(()=>{const t=s=>{const{method:i,params:a}=s.data;if(i!=="load"||!((a==null?void 0:a.trace)instanceof Blob))return;const d=new File([a.trace],"trace.zip",{type:"application/zip"}),w=new DataTransfer;w.items.add(d),g(w.files)};return window.addEventListener("message",t),()=>window.removeEventListener("message",t)});const W=r.useCallback(t=>{t.preventDefault(),g(t.dataTransfer.files)},[g]),M=r.useCallback(t=>{t.preventDefault(),t.target.files&&g(t.target.files)},[g]);r.useEffect(()=>{const t=new URL(window.location.href).searchParams,s=t.get("trace");if(c(t.has("isServer")),s!=null&&s.startsWith("file:")){R(s||null);return}if(t.has("isServer")){const i=new URLSearchParams(window.location.search).get("ws"),a=new URL(`../${i}`,window.location.toString());a.protocol=window.location.protocol==="https:"?"wss:":"ws:";const d=new C(new F(a));d.onLoadTraceRequested(async w=>{l(w.traceUrl),v(!1),m(null)}),d.initialize({}).catch(()=>{})}else s&&!s.startsWith("blob:")&&l(s)},[]);const S=r.useCallback(async t=>{const s=new URLSearchParams;s.set("trace",t);const i=await fetch(`contexts?${s.toString()}`);if(!i.ok){const{error:w}=await i.json();return m(w),w}const a=await i.json(),d=new x(t,a);j({done:0,total:0}),m(null),b(d)},[]);r.useEffect(()=>{(async()=>{if(!n){b(N);return}const t=s=>{s.data.method==="progress"&&j(s.data.params)};try{navigator.serviceWorker.addEventListener("message",t),j({done:0,total:1});let s=await S(n);s!=null&&s.includes("please grant permission for Local Network Access")&&(await fetch(n,{method:"HEAD",headers:{"x-pw-serviceworker":"skip"}}),s=await S(n)),s&&(o||l(void 0))}finally{navigator.serviceWorker.removeEventListener("message",t)}})()},[o,n,h,S]);const D=f.done!==f.total&&f.total!==0&&!y;r.useEffect(()=>{if(D){const t=setTimeout(()=>{E(!0)},200);return()=>clearTimeout(t)}else E(!1)},[D]);const L=!!(!o&&!k&&!T&&(!n||y));return e.jsxs("div",{className:"vbox workbench-loader",onDragOver:t=>{t.preventDefault(),t.dataTransfer.types.includes("Files")&&v(!0)},children:[e.jsxs("div",{className:"hbox workbench-loader-header",...L?{inert:!0}:{},children:[e.jsx("div",{className:"logo",children:e.jsx("img",{src:"playwright-logo.svg",alt:"Playwright logo"})}),e.jsx("div",{className:"product",children:"Playwright"}),p.title&&e.jsx("div",{className:"title",children:p.title}),e.jsx("div",{className:"spacer"}),e.jsx(z,{icon:"settings-gear",title:"Settings",dialogDataTestId:"settings-toolbar-dialog",children:e.jsx(I,{location:"trace-viewer"})})]}),e.jsx(P,{model:p,inert:L}),T&&e.jsxs("div",{className:"drop-target",children:[e.jsx("div",{children:"Trace Viewer uses Service Workers to show traces. To view trace:"}),e.jsxs("div",{style:{paddingTop:20},children:[e.jsxs("div",{children:["1. Click ",e.jsx("a",{href:T,children:"here"})," to put your trace into the download shelf"]}),e.jsxs("div",{children:["2. Go to ",e.jsx("a",{href:"https://trace.playwright.dev",children:"trace.playwright.dev"})]}),e.jsx("div",{children:"3. Drop the trace from the download shelf into the page"})]})]}),e.jsx(A,{open:U,isModal:!0,className:"progress-dialog",children:e.jsxs("div",{className:"progress-content",children:[e.jsx("div",{className:"title",role:"heading","aria-level":1,children:"Loading Playwright Trace..."}),e.jsx("div",{className:"progress-wrapper",children:e.jsx("div",{className:"inner-progress",style:{width:f.total?100*f.done/f.total+"%":0}})})]})}),L&&e.jsxs("div",{className:"drop-target",children:[e.jsx("div",{className:"processing-error",role:"alert",children:y}),e.jsx("div",{className:"title",role:"heading","aria-level":1,children:"Drop Playwright Trace to load"}),e.jsx("div",{children:"or"}),e.jsx("button",{onClick:()=>{const t=document.createElement("input");t.type="file",t.click(),t.addEventListener("change",s=>M(s))},type:"button",children:"Select file"}),e.jsx("div",{className:"info",children:"Playwright Trace Viewer is a Progressive Web App, it does not send your trace anywhere, it opens it locally."}),e.jsxs("div",{className:"version",children:["Playwright v","1.61.0"]})]}),o&&!n&&e.jsx("div",{className:"drop-target",children:e.jsx("div",{className:"title",children:"Select test to see the trace"})}),k&&e.jsx("div",{className:"drop-target",onDragLeave:()=>{v(!1)},onDrop:t=>W(t),children:e.jsx("div",{className:"title",children:"Release to analyse the Playwright Trace"})})]})},N=new x("",[]),q=({traceJson:o})=>{const[c,n]=r.useState(void 0),[l,h]=r.useState(0),u=r.useRef(null);return r.useEffect(()=>(u.current&&clearTimeout(u.current),u.current=setTimeout(async()=>{try{const p=await B(o);n(p)}catch{const p=new x("",[]);n(p)}finally{h(l+1)}},500),()=>{u.current&&clearTimeout(u.current)}),[o,l]),e.jsx(P,{isLive:!0,model:c})};async function B(o){const c=new URLSearchParams;c.set("trace",o);const l=await(await fetch(`contexts?${c.toString()}`)).json();return new x(o,l)}(async()=>{const o=new URLSearchParams(window.location.search);if(O(),window.location.protocol!=="file:"){if(o.get("isUnderTest")==="true"&&await new Promise(h=>setTimeout(h,1e3)),!navigator.serviceWorker)throw new Error(`Service workers are not supported. -Make sure to serve the Trace Viewer (${window.location}) via HTTPS or localhost.`);navigator.serviceWorker.register("sw.bundle.js"),navigator.serviceWorker.controller||await new Promise(h=>{navigator.serviceWorker.oncontrollerchange=()=>h()}),setInterval(function(){fetch("ping")},1e4)}const c=o.get("trace"),l=(c==null?void 0:c.endsWith(".json"))?e.jsx(q,{traceJson:c}):e.jsx($,{});V.createRoot(document.querySelector("#root")).render(l)})(); diff --git a/scraper-service/node_modules/playwright-core/lib/vite/traceViewer/index.html b/scraper-service/node_modules/playwright-core/lib/vite/traceViewer/index.html index f9c8948..4e61864 100644 --- a/scraper-service/node_modules/playwright-core/lib/vite/traceViewer/index.html +++ b/scraper-service/node_modules/playwright-core/lib/vite/traceViewer/index.html @@ -7,7 +7,7 @@ Playwright Trace Viewer - + diff --git a/scraper-service/node_modules/playwright-core/package.json b/scraper-service/node_modules/playwright-core/package.json index cf72e5e..44be66b 100644 --- a/scraper-service/node_modules/playwright-core/package.json +++ b/scraper-service/node_modules/playwright-core/package.json @@ -1,6 +1,6 @@ { "name": "playwright-core", - "version": "1.61.0", + "version": "1.61.1", "description": "A high-level API to automate web browsers", "repository": { "type": "git", diff --git a/scraper-service/node_modules/playwright/lib/common/index.js b/scraper-service/node_modules/playwright/lib/common/index.js index ace2c1a..93b0e23 100644 --- a/scraper-service/node_modules/playwright/lib/common/index.js +++ b/scraper-service/node_modules/playwright/lib/common/index.js @@ -869,7 +869,7 @@ function resolve(specifier, context, nextResolve) { const filename = import_url.default.fileURLToPath(context.parentURL); const resolved = resolveHook(filename, specifier); if (resolved !== void 0) { - specifier = context.conditions?.includes("import") ? import_url.default.pathToFileURL(resolved).toString() : resolved; + specifier = new Set(context.conditions).has("import") ? import_url.default.pathToFileURL(resolved).toString() : resolved; } } const result2 = nextResolve(specifier, context); @@ -894,7 +894,8 @@ function load(moduleUrl, context, nextLoad) { const filename = import_url.default.fileURLToPath(moduleUrl); if (!shouldTransform(filename)) return nextLoad(moduleUrl, context); - const format = kSupportedFormats.get(context.format) || (fileIsModule(filename) ? "module" : "commonjs"); + const isRequire = !new Set(context.conditions).has("import"); + const format = isRequire ? "commonjs" : kSupportedFormats.get(context.format) || (fileIsModule(filename) ? "module" : "commonjs"); const code = import_fs5.default.readFileSync(filename, "utf-8"); const transformed = transformHook(code, filename, format === "module" ? moduleUrl : void 0); return { diff --git a/scraper-service/node_modules/playwright/lib/common/index.js.txt b/scraper-service/node_modules/playwright/lib/common/index.js.txt index c048adf..694c607 100644 --- a/scraper-service/node_modules/playwright/lib/common/index.js.txt +++ b/scraper-service/node_modules/playwright/lib/common/index.js.txt @@ -16,7 +16,7 @@ 1.2 KB packages/playwright/src/common/validators.ts 1.3 KB packages/playwright/src/isomorphic/teleReceiver.ts 9.6 KB packages/playwright/src/transform/compilationCache.ts - 1.6 KB packages/playwright/src/transform/esmLoaderSync.ts + 1.7 KB packages/playwright/src/transform/esmLoaderSync.ts 1.1 KB packages/playwright/src/transform/pirates.ts 1.1 KB packages/playwright/src/transform/portTransport.ts 12.3 KB packages/playwright/src/transform/transform.ts diff --git a/scraper-service/node_modules/playwright/lib/matchers/expect.js b/scraper-service/node_modules/playwright/lib/matchers/expect.js index b6a3ebd..f42cfcd 100644 --- a/scraper-service/node_modules/playwright/lib/matchers/expect.js +++ b/scraper-service/node_modules/playwright/lib/matchers/expect.js @@ -12832,8 +12832,10 @@ function createExpect(info) { if (typeof m !== "function") throw new TypeError(`expect.extend: \`${name}\` is not a valid matcher. Must be a function, is "${typeof m}"`); } - Object.assign(info.userMatchers, matchers2); for (const [name, matcher] of Object.entries(matchers2)) { + if (name in allBuiltinMatchers) + continue; + info.userMatchers[name] = matcher; const { positive, inverse } = buildCustomAsymmetricMatcher(name, matcher); expectFn[name] = positive; notAsymmetric[name] = inverse; diff --git a/scraper-service/node_modules/playwright/lib/matchers/expect.js.txt b/scraper-service/node_modules/playwright/lib/matchers/expect.js.txt index f1a34c8..86b44dc 100644 --- a/scraper-service/node_modules/playwright/lib/matchers/expect.js.txt +++ b/scraper-service/node_modules/playwright/lib/matchers/expect.js.txt @@ -1,5 +1,5 @@ # packages/playwright/lib/matchers/expect.js -# total: 513.8 KB +# total: 513.9 KB ## Inlined (64) 7.9 KB node_modules/@babel/code-frame/lib/index.js diff --git a/scraper-service/node_modules/playwright/lib/transform/esmLoader.js b/scraper-service/node_modules/playwright/lib/transform/esmLoader.js index 2ae509c..f718074 100644 --- a/scraper-service/node_modules/playwright/lib/transform/esmLoader.js +++ b/scraper-service/node_modules/playwright/lib/transform/esmLoader.js @@ -5629,7 +5629,7 @@ function resolve(specifier, context, nextResolve) { const filename = import_url.default.fileURLToPath(context.parentURL); const resolved = resolveHook(filename, specifier); if (resolved !== void 0) { - specifier = context.conditions?.includes("import") ? import_url.default.pathToFileURL(resolved).toString() : resolved; + specifier = new Set(context.conditions).has("import") ? import_url.default.pathToFileURL(resolved).toString() : resolved; } } const result2 = nextResolve(specifier, context); @@ -5654,7 +5654,8 @@ function load(moduleUrl, context, nextLoad) { const filename = import_url.default.fileURLToPath(moduleUrl); if (!shouldTransform(filename)) return nextLoad(moduleUrl, context); - const format = kSupportedFormats.get(context.format) || (fileIsModule(filename) ? "module" : "commonjs"); + const isRequire = !new Set(context.conditions).has("import"); + const format = isRequire ? "commonjs" : kSupportedFormats.get(context.format) || (fileIsModule(filename) ? "module" : "commonjs"); const code = import_fs4.default.readFileSync(filename, "utf-8"); const transformed = transformHook(code, filename, format === "module" ? moduleUrl : void 0); return { diff --git a/scraper-service/node_modules/playwright/lib/transform/esmLoader.js.txt b/scraper-service/node_modules/playwright/lib/transform/esmLoader.js.txt index 515741a..be777eb 100644 --- a/scraper-service/node_modules/playwright/lib/transform/esmLoader.js.txt +++ b/scraper-service/node_modules/playwright/lib/transform/esmLoader.js.txt @@ -1,5 +1,5 @@ # packages/playwright/lib/transform/esmLoader.js -# total: 249.0 KB +# total: 249.1 KB ## Inlined (47) 1.5 KB node_modules/balanced-match/index.js @@ -40,7 +40,7 @@ 0.2 KB packages/isomorphic/stringUtils.ts 6.3 KB packages/playwright/src/transform/compilationCache.ts 3.1 KB packages/playwright/src/transform/esmLoader.ts - 1.6 KB packages/playwright/src/transform/esmLoaderSync.ts + 1.7 KB packages/playwright/src/transform/esmLoaderSync.ts 1.1 KB packages/playwright/src/transform/pirates.ts 1.1 KB packages/playwright/src/transform/portTransport.ts 11.7 KB packages/playwright/src/transform/transform.ts diff --git a/scraper-service/node_modules/playwright/package.json b/scraper-service/node_modules/playwright/package.json index 22e5571..c077784 100644 --- a/scraper-service/node_modules/playwright/package.json +++ b/scraper-service/node_modules/playwright/package.json @@ -1,6 +1,6 @@ { "name": "playwright", - "version": "1.61.0", + "version": "1.61.1", "description": "A high-level API to automate web browsers", "repository": { "type": "git", @@ -53,7 +53,7 @@ }, "license": "Apache-2.0", "dependencies": { - "playwright-core": "1.61.0" + "playwright-core": "1.61.1" }, "optionalDependencies": { "fsevents": "2.3.2" diff --git a/scraper-service/package-lock.json b/scraper-service/package-lock.json index fa37c78..87d9218 100644 --- a/scraper-service/package-lock.json +++ b/scraper-service/package-lock.json @@ -9,7 +9,7 @@ "version": "1.0.0", "dependencies": { "express": "^4.18.2", - "playwright": "^1.48.0" + "playwright": "^1.61.1" } }, "node_modules/accepts": { @@ -580,12 +580,11 @@ "license": "MIT" }, "node_modules/playwright": { - "version": "1.61.0", - "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.61.0.tgz", - "integrity": "sha512-Z+7BeeqQPRRzklHsVFP4KTGIyMxKUmfeRA4WisM6G3/XW6nwGeX6fX9qYaDa+CiUqpOkb2f6X3nar05R3kSuJQ==", - "license": "Apache-2.0", + "version": "1.61.1", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.61.1.tgz", + "integrity": "sha512-DWnY5o3YbLWK4GovuAVwpqL+1VwGNdUGrRr++8j8PtQQzvAVZUIMjKQ90fY689sEJZJBbZVw1rXaOKSTitkzPQ==", "dependencies": { - "playwright-core": "1.61.0" + "playwright-core": "1.61.1" }, "bin": { "playwright": "cli.js" @@ -598,10 +597,9 @@ } }, "node_modules/playwright-core": { - "version": "1.61.0", - "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.61.0.tgz", - "integrity": "sha512-caX7TrY3Ml6egyDX0WUcTHDxodl/b51y5wJOdCEA36QviK/s2g081hvmGs8eaE3DWb6NYZQ6BjO/QkNRPenoPA==", - "license": "Apache-2.0", + "version": "1.61.1", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.61.1.tgz", + "integrity": "sha512-h7Qlt6m4REp25qvIdvbDtVmD4LqVXfpRxhORv9L0jzETM05p4fuPJ3dKyuSXQxDSbXnmS79HAgi9589lGSpLkg==", "bin": { "playwright-core": "cli.js" }, diff --git a/scraper-service/package.json b/scraper-service/package.json index c5afa35..9e5ff2f 100644 --- a/scraper-service/package.json +++ b/scraper-service/package.json @@ -8,6 +8,6 @@ }, "dependencies": { "express": "^4.18.2", - "playwright": "^1.48.0" + "playwright": "^1.61.1" } } diff --git a/src/app/(dashboard)/ai-assistant/page.tsx b/src/app/(dashboard)/ai-assistant/page.tsx index 9528f09..e70bc7f 100644 --- a/src/app/(dashboard)/ai-assistant/page.tsx +++ b/src/app/(dashboard)/ai-assistant/page.tsx @@ -18,8 +18,32 @@ export default function AIAssistantPage() { const [recentPrompts, setRecentPrompts] = useState([]) const [aiOnline, setAiOnline] = useState(null) const [searching, setSearching] = useState(false) + const [msgCount, setMsgCount] = useState(0) + const [tokenEstimate, setTokenEstimate] = useState("0") const aiChatRef = useRef(null) + useEffect(() => { + const saved = localStorage.getItem("ai-chat-messages") + if (saved) { + try { + const msgs = JSON.parse(saved) + if (Array.isArray(msgs)) { + const today = new Date() + today.setHours(0, 0, 0, 0) + const todayMsgs = msgs.filter((m: any) => { + if (!m.ts) return true + return new Date(m.ts).getTime() >= today.getTime() + }) + const userMsgs = todayMsgs.filter((m: any) => m.role === "user") + setMsgCount(userMsgs.length) + const totalChars = todayMsgs.reduce((s: number, m: any) => s + (m.content?.length || 0), 0) + const tokens = Math.round(totalChars / 4) + setTokenEstimate(tokens >= 1000 ? `${(tokens / 1000).toFixed(1)}k` : String(tokens)) + } + } catch {} + } + }, []) + const handleJobSelect = useCallback((job: typeof selectedJob) => { setSelectedJob(job) }, []) @@ -41,36 +65,72 @@ export default function AIAssistantPage() { const handleSearch = useCallback(async (job: NonNullable) => { setSearching(true) - const keyword = job.keywords[0] - aiChatRef.current?.addAssistantMessage(`🔍 Searching Facebook for **${job.job_title}** leads...`) + const prompt = `You are a lead generation assistant. Generate 5 realistic prospect profiles for my business targeting ${job.job_title} clients (${job.industry}). ${job.description ? "Service: " + job.description : ""} Keywords: ${job.keywords.join(", ")}. - const controller = new AbortController() - const timeoutId = setTimeout(() => controller.abort(), 360000) - const statusId = setTimeout(() => { - aiChatRef.current?.addAssistantMessage("⏳ Still searching Facebook (this can take up to 5 minutes)...") - }, 45000) +IMPORTANT: Return ONLY a valid JSON array. Do NOT use markdown, code blocks, or any formatting. Just raw JSON. + +Example format: +[{"companyName":"Acme Corp","contactName":"John Smith","email":"john@acme.com","phone":"555-0100","description":"Brief description"}] + +Use realistic company names, names, and emails for the ${job.industry} industry.` + aiChatRef.current?.addAssistantMessage(`🔍 Finding leads for **${job.job_title}**...`) try { - const res = await fetch(`http://localhost:3008/scrape/facebook?force=true&query=${encodeURIComponent(keyword)}`, { method: "POST", signal: controller.signal }) - clearTimeout(timeoutId) - clearTimeout(statusId) - const data = await res.json() - if (data.success && data.leads?.length > 0) { - const leadsText = data.leads.map((lead: any, i: number) => - `**${i + 1}.** ${lead.author || "Unknown"}\n> ${(lead.content || "").slice(0, 300)}\n> 🔗 ${lead.url || "(no link available)"}` - ).join("\n\n") - aiChatRef.current?.addAssistantMessage(`✅ Found **${data.leads.length}** leads:\n\n${leadsText}`) - } else { - const reason = data.error || data.flag_reason || "No leads found this time" - aiChatRef.current?.addAssistantMessage(`⚠️ ${reason}`) + const res = await fetch("/api/ai/chat", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ message: prompt }), + }) + + if (!res.ok) { + const data = await res.json().catch(() => ({})) + throw new Error(data.error || `Error ${res.status}`) } - } catch (err: any) { - clearTimeout(timeoutId) - clearTimeout(statusId) - const msg = err.name === "AbortError" - ? "❌ Scraper timed out after 5 minutes. Try again or check the scraper service." - : `❌ ${err instanceof Error ? err.message : "Search failed"}` - aiChatRef.current?.addAssistantMessage(msg) + + const data = await res.json() + const raw = data.response || "" + let leads: any[] + try { + let text = raw.trim() + const jsonMatch = text.match(/\[[\s\S]*\]/) + if (jsonMatch) text = jsonMatch[0] + leads = JSON.parse(text) + if (!Array.isArray(leads)) leads = [leads] + } catch (e) { + aiChatRef.current?.addAssistantMessage(`⚠️ AI response wasn't valid JSON. Showing raw output:\n\n${raw.slice(0, 1500)}`) + setSearching(false) + return + } + + let created = 0 + let lastError = "" + for (const lead of leads.slice(0, 5)) { + try { + const r = await fetch("/api/leads", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + companyName: lead.companyName || "Unknown Company", + contactName: lead.contactName || "", + email: lead.email || "", + phone: lead.phone || "", + description: lead.description || "", + status: "open", + }), + }) + if (r.ok) created++ + else { const e = await r.json().catch(() => ({})); lastError = e.error || r.statusText; } + } catch (e) { lastError = "Network error" } + } + + if (created > 0) { + aiChatRef.current?.addAssistantMessage(`✅ Created **${created}** lead${created > 1 ? "s" : ""} for **${job.job_title}**! Go to the Leads tab to see them.`) + } else { + aiChatRef.current?.addAssistantMessage(`❌ Failed to create leads. Last error: ${lastError || "Unknown"}. Make sure you're logged in with an account that has permission to create leads.`) + } + } catch (err) { + const msg = err instanceof Error ? err.message : "Search failed" + aiChatRef.current?.addAssistantMessage(`❌ Error: ${msg}. Make sure Ollama is running with the model loaded.`) } finally { setSearching(false) } @@ -89,6 +149,12 @@ export default function AIAssistantPage() { const next = [msg, ...prev.filter((p) => p !== msg)].slice(0, 3) return next }) + setMsgCount((prev) => prev + 1) + setTokenEstimate((prev) => { + const raw = parseInt(prev.replace("k", "")) * (prev.includes("k") ? 1000 : 1) + const updated = raw + Math.round(msg.length / 4) + return updated >= 1000 ? `${(updated / 1000).toFixed(1)}k` : String(updated) + }) }, []) return ( @@ -186,11 +252,11 @@ export default function AIAssistantPage() {
Messages today
-
24
+
{msgCount}
Tokens used
-
12.4k
+
{tokenEstimate}
diff --git a/src/components/ai/job-selector.tsx b/src/components/ai/job-selector.tsx index fa00790..8543c70 100644 --- a/src/components/ai/job-selector.tsx +++ b/src/components/ai/job-selector.tsx @@ -79,7 +79,7 @@ export function JobSelector({ onSelect, onSearch, searching }: JobSelectorProps) className="w-full flex items-center justify-center gap-2 mt-3 bg-primary hover:bg-primary/90 disabled:opacity-50 disabled:cursor-not-allowed text-primary-foreground text-sm font-semibold rounded-xl px-4 py-3 transition-all duration-200 shadow-lg shadow-primary/20" > {searching ? : } - {searching ? "Searching..." : "Search Facebook"} + {searching ? "Searching..." : "Search Leads"} )}