diff --git a/.gitignore b/.gitignore index c1b3cd5..91ec548 100644 --- a/.gitignore +++ b/.gitignore @@ -34,9 +34,24 @@ yarn-error.log* # env files (can opt-in for committing if needed) .env* +# runtime data +data/ +data/ai/ +data/ai/jobs.jsonl + +# logs +*.log +*.out +error-log + # vercel .vercel +# python +browser-use-service/venv/ +browser-use-service/__pycache__/ +browser-use-service/.env + # typescript *.tsbuildinfo next-env.d.ts diff --git a/browser-use-service/err.txt b/browser-use-service/err.txt new file mode 100644 index 0000000..a1f2d3d Binary files /dev/null and b/browser-use-service/err.txt differ diff --git a/browser-use-service/main.py b/browser-use-service/main.py new file mode 100644 index 0000000..726a364 --- /dev/null +++ b/browser-use-service/main.py @@ -0,0 +1,410 @@ +import os, json, asyncio, re, shutil, sqlite3, traceback, urllib.parse, random, time, logging +from datetime import datetime, timedelta +from bs4 import BeautifulSoup +import httpx +from fastapi import FastAPI +from pydantic import BaseModel +import uvicorn +from playwright.async_api import async_playwright + +logging.basicConfig(level=logging.INFO, format='%(asctime)s [%(levelname)s] %(message)s') +logger = logging.getLogger(__name__) + +app = FastAPI() +PORT = int(os.getenv("PORT", "3008")) + +OLLAMA_URL = os.getenv("OLLAMA_URL", "http://localhost:11434") +CLASSIFY_MODEL = os.getenv("CLASSIFY_MODEL", "dolphin-llama3:8b") + +FX_PROFILE = os.getenv('FX_PROFILE', '') +FX_COOKIE_DB = os.path.join(FX_PROFILE, 'cookies.sqlite') if FX_PROFILE else '' + +HEADERS = { + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36', + 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', +} + +last_scrape_time: datetime | None = None + +STRICT_KEYWORDS = [ + "website", "web design", "web develop", "web dev", + "build my website", "build a website", "create a website", + "need web", "looking for web", "new website", + "landing page", "wordpress", + "need a website", "my website", "business website", + "need a designer", "help with my website", + "redesign", "update my website", +] + +BROAD_KEYWORDS = STRICT_KEYWORDS + [ + "looking for", "need a", "need an", "looking to", + "help me build", "create my", "build me", + "design my", "make me a", "would like", + "need someone", "hire a", "looking to hire", + "want someone", "need help with", +] + +def kw_match_strict(text: str) -> bool: + t = text.lower() + return any(kw in t for kw in STRICT_KEYWORDS) + +def kw_match(text: str) -> bool: + t = text.lower() + return any(kw in t for kw in BROAD_KEYWORDS) + +FB_SEARCHES = [ + "looking for web developer", + "need a website designed", + "need website built South Africa", + "need someone to build my website", + "need web designer", + "looking for someone to create website", + "need ecommerce website built", + "want to hire web developer", + "website quote please", + "need wordpress website", + "I need a website for my business", + "need a site for my business", +] + +async def get_fb_cookies(): + if not FX_COOKIE_DB or not os.path.exists(FX_COOKIE_DB): + logger.warning("FX_COOKIE_DB not found or FX_PROFILE not set") + return [] + tmp = os.path.join(os.path.dirname(FX_COOKIE_DB), f'cookies_tmp_{os.getpid()}.sqlite') + for attempt in range(3): + try: + shutil.copy2(FX_COOKIE_DB, tmp) + break + except PermissionError: + if attempt < 2: + await asyncio.sleep(0.5) + else: + logger.error("Failed to copy cookie DB after 3 attempts") + return [] + try: + conn = sqlite3.connect(tmp) + c = conn.cursor() + c.execute("SELECT name, value, host, path FROM moz_cookies WHERE host LIKE '%facebook.com'") + rows = c.fetchall() + conn.close() + try: + os.remove(tmp) + except Exception: + pass + return [{ + "name": name, "value": value, + "domain": host if host.startswith('.') else f'.{host}', + "path": path, + "httpOnly": True, "secure": True, "sameSite": "Lax", + } for name, value, host, path in rows] + except Exception as e: + logger.error("Cookie DB read error: %s", e) + try: + os.remove(tmp) + except Exception: + pass + return [] + +WEEKDAY_ORDER = ['monday', 'tuesday', 'wednesday', 'thursday', 'friday', 'saturday', 'sunday'] + +def _parse_fb_date(block: list[str]) -> str: + for l in block: + l = l.strip() + m = re.match(r'(\d+)\s*(h|hr|hrs|hour|hours)\s*ago', l) + if m: + hours = int(m.group(1)) + d = datetime.now() - timedelta(hours=hours) + return d.strftime('%Y-%m-%d') + if l.lower() == 'yesterday': + d = datetime.now() - timedelta(days=1) + return d.strftime('%Y-%m-%d') + low = l.lower() + if low in WEEKDAY_ORDER: + day_idx = WEEKDAY_ORDER.index(low) + today_idx = datetime.now().weekday() + days_ago = (today_idx - day_idx) % 7 + d = datetime.now() - timedelta(days=days_ago) + return d.strftime('%Y-%m-%d') + for fmt in ['%Y-%m-%d', '%d %B %Y', '%B %d %Y', '%d %b %Y', '%b %d %Y', '%d %b', '%b %d']: + try: + dt = datetime.strptime(l, fmt) + return dt.strftime('%Y-%m-%d') + except ValueError: + pass + return datetime.now().strftime('%Y-%m-%d') + +def _extract_posts_from_text(raw: str, url: str) -> list[dict]: + lines = [l.strip() for l in raw.split('\n')] + blocks = [] + cur = [] + fb_run = 0 + for l in lines: + if 'facebook' in l.lower(): + fb_run += 1 + if cur and fb_run >= 2: + blocks.append(cur) + cur = [] + else: + fb_run = 0 + if l and len(l) >= 15: + words = re.findall(r'[A-Za-z]{2,}', l) + if len(words) >= 2: + cur.append(l) + if cur: + blocks.append(cur) + + posts = [] + seen_texts = set() + for block in blocks: + if not block: + continue + kw_indices = [i for i, l in enumerate(block) if kw_match(l)] + if not kw_indices: + continue + i = kw_indices[0] + start = max(0, i - 2) + end = min(len(block), i + 5) + snippet = ' '.join(block[start:end]) + if len(snippet) < 40: + continue + dekey = snippet[:80] + if dekey in seen_texts: + continue + seen_texts.add(dekey) + posts.append({ + "title": snippet[:300], + "content": snippet[:1000], + "author": block[start] if start < i else '', + "url": url, + "source": "facebook", + "date": _parse_fb_date(block), + }) + return posts + +async def search_facebook(page, query: str) -> list[dict]: + url = f'https://www.facebook.com/search/posts/?q={urllib.parse.quote(query)}' + try: + await page.goto(url, wait_until='domcontentloaded', timeout=30000) + await page.wait_for_timeout(6000) + except Exception as e: + logger.warning("Facebook search navigation failed for '%s': %s", query, e) + return [] + + try: + raw = await page.evaluate('document.body.innerText') + except Exception as e: + logger.warning("Failed to evaluate page text: %s", e) + return [] + return _extract_posts_from_text(raw, url) + +async def scrape_facebook() -> list[dict]: + all_posts = [] + fb_cookies = await get_fb_cookies() + if not fb_cookies: + logger.warning("No Facebook cookies available") + return [] + try: + async with async_playwright() as pw: + browser = await pw.chromium.launch(headless=True) + context = await browser.new_context( + user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:150.0) Gecko/20100101 Firefox/150.0', + viewport={'width': 1280, 'height': 800}, + ) + for c in fb_cookies: + try: + await context.add_cookies([c]) + except Exception as e: + logger.warning("Failed to inject cookie %s: %s", c.get('name'), e) + + page = await context.new_page() + await page.goto('https://www.facebook.com/', wait_until='domcontentloaded', timeout=30000) + await page.wait_for_timeout(5000) + + url = page.url + if '/login' in url.lower(): + logger.warning("Facebook login page detected — cookies may be expired") + return [] + + for query in FB_SEARCHES[:6]: + try: + posts = await search_facebook(page, query) + all_posts.extend(posts) + delay = random.uniform(3, 7) + await page.wait_for_timeout(int(delay * 1000)) + except Exception as e: + logger.warning("Facebook search '%s' failed: %s", query, e) + except Exception as e: + logger.error("Facebook scrape failed: %s", e) + return [] + + seen = set() + deduped = [] + for p in all_posts: + key = p.get('content', '')[:100] + if key not in seen: + seen.add(key) + deduped.append(p) + return deduped[:20] + +async def ask_ollama(prompt: str) -> str: + async with httpx.AsyncClient(timeout=120) as c: + r = await c.post(f"{OLLAMA_URL}/api/chat", json={ + "model": CLASSIFY_MODEL, + "messages": [ + {"role": "system", "content": "You classify forum posts. Return only valid JSON."}, + {"role": "user", "content": prompt} + ], + "stream": False, + "options": {"temperature": 0.05, "num_predict": 1024}, + }) + r.raise_for_status() + data = r.json() + return data["message"]["content"] + +async def scrape_warriorforum() -> list[dict]: + results = [] + try: + async with httpx.AsyncClient(headers=HEADERS, timeout=15, follow_redirects=True) as c: + r = await c.get('https://www.warriorforum.com/wanted-members-looking-hire-you/') + soup = BeautifulSoup(r.text, 'lxml') + for row in soup.find_all('td', class_='FlexTable-item--title'): + a = row.find('a', href=True) + if not a: + continue + href = a['href'] + title = a.text.strip() + if not title or len(title) < 15 or not kw_match_strict(title): + continue + author_div = row.find('div', class_='FlexTable-item-author') + author = author_div.get_text(strip=True).lstrip('by') if author_div else '' + date_str = '' + date_div = row.find('div', class_=lambda c: c and 'media--available' in c) + if date_div: + txt = date_div.get_text(strip=True) + m = re.search(r'(\d{10})', txt) + if m: + date_str = datetime.utcfromtimestamp(int(m.group(1))).strftime('%Y-%m-%d') + if not href.startswith('http'): + href = 'https://www.warriorforum.com' + href + results.append({ + "title": title, "url": href, + "author": author, + "content": title[:300], + "source": "warriorforum", + "date": date_str + }) + except Exception: + logger.error("WarriorForum scrape failed", exc_info=True) + return results + +async def classify_leads(results: list[dict]) -> list[dict]: + if not results: + return [] + briefs = [r["title"][:200] for r in results] + prompt = f"""Classify each post as LEAD or NOT. +LEAD = someone REQUESTING/POSTING/WANTING a website built, designed, or created for them. +LEAD examples: "Need a website for my business", "Looking for web developer to build my site", "I need someone to create my website", "Want a new website for my company", "Looking for someone to design my WordPress site" + +NOT LEAD: +- Offering web design services: "I build websites", "I offer web design", "Affordable web design packages" +- Already have a website and need marketing, SEO, content, video, link building, email marketing, affiliates +- Recruiting employees, hiring staff, looking for business partners +- Selling products, promoting services, affiliate offers +- "Need web hosting", "Looking for a partner", "Looking for content writer", "Video spokesperson" + +For each numbered post, answer ONLY "yes" (LEAD) or "no" (NOT LEAD): +{chr(10).join(f'{i+1}. {t}' for i, t in enumerate(briefs))} +Return a JSON array like ["yes","no","yes"] matching the order above.""" + ai_succeeded = False + try: + raw = await ask_ollama(prompt) + raw = raw.strip() + # Strip markdown code fences + if raw.startswith("```"): + # Remove opening fence + first_nl = raw.find('\n') + if first_nl != -1: + raw = raw[first_nl + 1:] + # Remove closing fence + if raw.endswith("```"): + raw = raw[:-3] + raw = raw.strip() + answers = json.loads(raw) + if isinstance(answers, list) and len(answers) == len(results): + ai_succeeded = True + filtered = [] + for i, ans in enumerate(answers): + if isinstance(ans, str) and ans.lower() == 'yes': + filtered.append(results[i]) + if filtered: + return filtered[:10] + # AI successfully classified but returned all "no" — respect that decision + logger.info("AI classified all %d items as NOT LEAD — returning empty", len(results)) + return [] + except Exception as e: + logger.warning("AI classification failed, falling back to keyword filter: %s", e) + + # Fallback: only use keyword filter when AI failed + if not ai_succeeded: + filtered = [] + for r in results: + t = r['title'].lower() + if not kw_match_strict(t): + continue + if any(kw in t for kw in ['i build', 'i offer', 'i create', 'i am a', 'web developer available', + 'affordable web', 'web hosting', 'free website', + 'limited time', 'special offer', 'sign up now', + 'offer']): + continue + filtered.append(r) + return filtered[:10] + return [] + +@app.get("/health") +async def health(): + return {"status": "ok"} + +class ScrapeRequest(BaseModel): + query: str = "" + +@app.post("/scrape/requests") +async def scrape_requests(req: ScrapeRequest): + global last_scrape_time + now = datetime.now() + if last_scrape_time and (now - last_scrape_time) < timedelta(seconds=15): + return {"error": "rate_limited", "retry_after": 15} + last_scrape_time = now + results = await scrape_warriorforum() + if results: + results = await classify_leads(results) + return results[:10] + +@app.post("/scrape/facebook") +async def scrape_facebook_endpoint(): + results = await scrape_facebook() + if results: + results = await classify_leads(results) + return results[:15] + +@app.post("/scrape/all") +async def scrape_all(): + results = [] + try: + wf = await scrape_warriorforum() + if wf: + wf = await classify_leads(wf) + results.extend(wf[:5]) + except Exception as e: + logger.error("WarriorForum scrape in /scrape/all failed: %s", e) + try: + fb = await scrape_facebook() + if fb: + fb = await classify_leads(fb) + results.extend(fb[:8]) + except Exception as e: + logger.error("Facebook scrape in /scrape/all failed: %s", e) + return results[:12] + +if __name__ == "__main__": + uvicorn.run(app, host="0.0.0.0", port=PORT) diff --git a/browser-use-service/out.txt b/browser-use-service/out.txt new file mode 100644 index 0000000..e69de29 diff --git a/browser-use-service/run_log.txt b/browser-use-service/run_log.txt new file mode 100644 index 0000000..df792e6 --- /dev/null +++ b/browser-use-service/run_log.txt @@ -0,0 +1,7 @@ +python : INFO: Started server process [20044] + + CategoryInfo : NotSpecified: (INFO: Started server process [20044]:String) [], RemoteException + + FullyQualifiedErrorId : NativeCommandError + +INFO: Waiting for application startup. +INFO: Application startup complete. +INFO: Uvicorn running on http://0.0.0.0:3008 (Press CTRL+C to quit) diff --git a/browser-use-service/scrape_log.txt b/browser-use-service/scrape_log.txt new file mode 100644 index 0000000..a356eb3 --- /dev/null +++ b/browser-use-service/scrape_log.txt @@ -0,0 +1,6 @@ +[2026-06-22T19:24:48.030482] Browser Use service starting on port 3008 +[2026-06-22T19:27:19.231701] Browser Use service starting on port 3008 +[2026-06-22T19:28:28.335765] Browser Use service starting on port 3008 +[2026-06-22T19:29:05.796265] Browser Use service starting on port 3008 +[2026-06-22T19:29:17.042807] Scraping WarriorForum... +[2026-06-22T19:29:19.075166] WarriorForum: 55 results diff --git a/browser-use-service/stderr.txt b/browser-use-service/stderr.txt new file mode 100644 index 0000000..85fb7e5 --- /dev/null +++ b/browser-use-service/stderr.txt @@ -0,0 +1,4 @@ +INFO: Started server process [24268] +INFO: Waiting for application startup. +INFO: Application startup complete. +INFO: Uvicorn running on http://0.0.0.0:3008 (Press CTRL+C to quit) diff --git a/browser-use-service/stdout.txt b/browser-use-service/stdout.txt new file mode 100644 index 0000000..66ec1d4 --- /dev/null +++ b/browser-use-service/stdout.txt @@ -0,0 +1,5 @@ +Browser Use service starting on port 3008 +INFO: 127.0.0.1:65003 - "GET /health HTTP/1.1" 200 OK +Scraping WarriorForum... +WarriorForum: 55 results +INFO: 127.0.0.1:54587 - "POST /scrape/all HTTP/1.1" 200 OK diff --git a/database/migrations/001_schema.sql b/database/migrations/001_schema.sql index 231cd74..9ae5d1c 100644 --- a/database/migrations/001_schema.sql +++ b/database/migrations/001_schema.sql @@ -22,7 +22,7 @@ $$ LANGUAGE plpgsql; -- 1. AUTHENTICATION & ROLE MANAGEMENT -- ============================================================================ -CREATE TABLE roles ( +CREATE TABLE IF NOT EXISTS roles ( id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), name VARCHAR(50) NOT NULL, display_name VARCHAR(100), @@ -35,7 +35,7 @@ CREATE TABLE roles ( CREATE UNIQUE INDEX uq_roles_name ON roles(name); -CREATE TABLE permissions ( +CREATE TABLE IF NOT EXISTS permissions ( id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), resource VARCHAR(100) NOT NULL, action VARCHAR(50) NOT NULL, @@ -46,7 +46,7 @@ CREATE TABLE permissions ( CREATE UNIQUE INDEX uq_permissions ON permissions(resource, action); -CREATE TABLE role_permissions ( +CREATE TABLE IF NOT EXISTS role_permissions ( role_id UUID NOT NULL REFERENCES roles(id) ON DELETE CASCADE, permission_id UUID NOT NULL REFERENCES permissions(id) ON DELETE CASCADE, granted_by UUID, @@ -54,7 +54,7 @@ CREATE TABLE role_permissions ( PRIMARY KEY (role_id, permission_id) ); -CREATE TABLE users ( +CREATE TABLE IF NOT EXISTS users ( id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), username VARCHAR(100) NOT NULL, email VARCHAR(255) NOT NULL, @@ -83,7 +83,7 @@ CREATE INDEX idx_users_is_active ON users(is_active) WHERE deleted_at IS NULL; CREATE INDEX idx_users_is_locked ON users(is_locked) WHERE is_locked = TRUE AND deleted_at IS NULL; CREATE INDEX idx_users_created_by ON users(created_by); -CREATE TABLE user_roles ( +CREATE TABLE IF NOT EXISTS user_roles ( user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, role_id UUID NOT NULL REFERENCES roles(id) ON DELETE CASCADE, assigned_by UUID REFERENCES users(id), @@ -98,7 +98,7 @@ CREATE INDEX idx_user_roles_role ON user_roles(role_id); -- 2. BAN & SUSPENSION SYSTEM -- ============================================================================ -CREATE TABLE banned_users ( +CREATE TABLE IF NOT EXISTS banned_users ( id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), banned_user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, banned_by UUID NOT NULL REFERENCES users(id), @@ -122,7 +122,7 @@ CREATE INDEX idx_banned_users_active ON banned_users(banned_user_id) WHERE is_reversed = FALSE; CREATE INDEX idx_banned_users_banned_by ON banned_users(banned_by); -CREATE TABLE suspended_users ( +CREATE TABLE IF NOT EXISTS suspended_users ( id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), suspended_user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, suspended_by UUID NOT NULL REFERENCES users(id), @@ -145,7 +145,7 @@ CREATE INDEX idx_suspended_users_suspended_by ON suspended_users(suspended_by); -- 3. SESSION & LOGIN MANAGEMENT -- ============================================================================ -CREATE TABLE sessions ( +CREATE TABLE IF NOT EXISTS sessions ( id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, token_hash VARCHAR(255) NOT NULL, @@ -161,7 +161,7 @@ CREATE INDEX idx_sessions_user ON sessions(user_id) WHERE is_active = TRUE; CREATE UNIQUE INDEX uq_sessions_token ON sessions(token_hash); CREATE INDEX idx_sessions_expires ON sessions(expires_at) WHERE is_active = TRUE; -CREATE TABLE login_attempts ( +CREATE TABLE IF NOT EXISTS login_attempts ( id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), user_id UUID REFERENCES users(id), username_attempted VARCHAR(100) NOT NULL, @@ -181,7 +181,7 @@ CREATE INDEX idx_login_attempts_time ON login_attempts(attempted_at DESC); -- 4. CUSTOMER MANAGEMENT -- ============================================================================ -CREATE TABLE customer_statuses ( +CREATE TABLE IF NOT EXISTS customer_statuses ( id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), name VARCHAR(100) NOT NULL, description TEXT, @@ -196,7 +196,7 @@ CREATE TABLE customer_statuses ( CREATE UNIQUE INDEX uq_customer_statuses_name ON customer_statuses(name) WHERE deleted_at IS NULL; -CREATE TABLE customers ( +CREATE TABLE IF NOT EXISTS customers ( id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), customer_type VARCHAR(20) NOT NULL, status_id UUID NOT NULL REFERENCES customer_statuses(id), @@ -218,7 +218,7 @@ CREATE INDEX idx_customers_score ON customers(score DESC) WHERE deleted_at IS NU CREATE INDEX idx_customers_tags ON customers USING GIN(tags) WHERE deleted_at IS NULL; CREATE INDEX idx_customers_created ON customers(created_at DESC) WHERE deleted_at IS NULL; -CREATE TABLE individual_customers ( +CREATE TABLE IF NOT EXISTS individual_customers ( customer_id UUID PRIMARY KEY REFERENCES customers(id) ON DELETE CASCADE, first_name VARCHAR(100) NOT NULL, last_name VARCHAR(100) NOT NULL, @@ -233,7 +233,7 @@ CREATE TABLE individual_customers ( CREATE INDEX idx_individual_names ON individual_customers(last_name, first_name); -CREATE TABLE company_customers ( +CREATE TABLE IF NOT EXISTS company_customers ( customer_id UUID PRIMARY KEY REFERENCES customers(id) ON DELETE CASCADE, company_name VARCHAR(255) NOT NULL, registration_number VARCHAR(100), @@ -250,7 +250,7 @@ CREATE TABLE company_customers ( CREATE INDEX idx_company_name ON company_customers(company_name); CREATE INDEX idx_company_industry ON company_customers(industry); -CREATE TABLE contact_information ( +CREATE TABLE IF NOT EXISTS contact_information ( id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), customer_id UUID NOT NULL REFERENCES customers(id) ON DELETE CASCADE, type VARCHAR(20) NOT NULL, @@ -270,7 +270,7 @@ CREATE INDEX idx_contacts_type ON contact_information(type) WHERE deleted_at IS CREATE INDEX idx_contacts_value ON contact_information(value) WHERE deleted_at IS NULL; CREATE UNIQUE INDEX uq_contacts_primary ON contact_information(customer_id, type) WHERE is_primary = TRUE AND deleted_at IS NULL; -CREATE TABLE addresses ( +CREATE TABLE IF NOT EXISTS addresses ( id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), customer_id UUID NOT NULL REFERENCES customers(id) ON DELETE CASCADE, address_type VARCHAR(20) NOT NULL, @@ -294,7 +294,7 @@ CREATE INDEX idx_addresses_country ON addresses(country) WHERE deleted_at IS NUL CREATE INDEX idx_addresses_city ON addresses(city) WHERE deleted_at IS NULL; CREATE UNIQUE INDEX uq_addresses_primary ON addresses(customer_id, address_type) WHERE is_primary = TRUE AND deleted_at IS NULL; -CREATE TABLE customer_notes ( +CREATE TABLE IF NOT EXISTS customer_notes ( id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), customer_id UUID NOT NULL REFERENCES customers(id) ON DELETE CASCADE, author_id UUID NOT NULL REFERENCES users(id), @@ -315,7 +315,7 @@ CREATE INDEX idx_notes_pinned ON customer_notes(is_pinned) WHERE is_pinned = TRU -- 5. CUSTOMER & INCIDENT REPORTS -- ============================================================================ -CREATE TABLE customer_reports ( +CREATE TABLE IF NOT EXISTS customer_reports ( id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), customer_id UUID NOT NULL REFERENCES customers(id), reported_by UUID NOT NULL REFERENCES users(id), @@ -339,7 +339,7 @@ CREATE INDEX idx_customer_reports_reported_by ON customer_reports(reported_by); CREATE INDEX idx_customer_reports_status ON customer_reports(status); CREATE INDEX idx_customer_reports_severity ON customer_reports(severity); -CREATE TABLE incident_reports ( +CREATE TABLE IF NOT EXISTS incident_reports ( id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), reported_by UUID NOT NULL REFERENCES users(id), target_user_id UUID REFERENCES users(id), @@ -368,7 +368,7 @@ CREATE INDEX idx_incident_reports_severity ON incident_reports(severity); -- 6. LEAD MANAGEMENT -- ============================================================================ -CREATE TABLE lead_sources ( +CREATE TABLE IF NOT EXISTS lead_sources ( id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), name VARCHAR(100) NOT NULL, description TEXT, @@ -380,7 +380,7 @@ CREATE TABLE lead_sources ( CREATE UNIQUE INDEX uq_lead_sources_name ON lead_sources(name) WHERE deleted_at IS NULL; -CREATE TABLE lead_stages ( +CREATE TABLE IF NOT EXISTS lead_stages ( id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), name VARCHAR(100) NOT NULL, description TEXT, @@ -395,7 +395,7 @@ CREATE TABLE lead_stages ( CREATE UNIQUE INDEX uq_lead_stages_name ON lead_stages(name) WHERE deleted_at IS NULL; CREATE INDEX idx_lead_stages_sort ON lead_stages(sort_order); -CREATE TABLE leads ( +CREATE TABLE IF NOT EXISTS leads ( id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), source_id UUID REFERENCES lead_sources(id), stage_id UUID NOT NULL REFERENCES lead_stages(id), @@ -431,7 +431,7 @@ CREATE INDEX idx_leads_converted ON leads(converted_customer_id) WHERE converted CREATE INDEX idx_leads_email ON leads(email) WHERE deleted_at IS NULL; CREATE INDEX idx_leads_active ON leads(stage_id, assigned_to) WHERE deleted_at IS NULL AND converted_customer_id IS NULL; -CREATE TABLE lead_conversions ( +CREATE TABLE IF NOT EXISTS lead_conversions ( id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), lead_id UUID NOT NULL REFERENCES leads(id) ON DELETE CASCADE, customer_id UUID NOT NULL REFERENCES customers(id) ON DELETE CASCADE, @@ -447,7 +447,7 @@ CREATE INDEX idx_lead_conversions_customer ON lead_conversions(customer_id); -- 7. PRODUCT CATALOG -- ============================================================================ -CREATE TABLE product_categories ( +CREATE TABLE IF NOT EXISTS product_categories ( id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), name VARCHAR(255) NOT NULL, description TEXT, @@ -462,7 +462,7 @@ CREATE TABLE product_categories ( CREATE UNIQUE INDEX uq_product_categories_name ON product_categories(name) WHERE deleted_at IS NULL; CREATE INDEX idx_product_categories_parent ON product_categories(parent_id); -CREATE TABLE products ( +CREATE TABLE IF NOT EXISTS products ( id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), category_id UUID REFERENCES product_categories(id), name VARCHAR(255) NOT NULL, @@ -487,7 +487,7 @@ CREATE INDEX idx_products_active ON products(is_active) WHERE deleted_at IS NULL -- 8. SALES PIPELINE -- ============================================================================ -CREATE TABLE deal_stages ( +CREATE TABLE IF NOT EXISTS deal_stages ( id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), name VARCHAR(100) NOT NULL, description TEXT, @@ -502,7 +502,7 @@ CREATE TABLE deal_stages ( CREATE UNIQUE INDEX uq_deal_stages_name ON deal_stages(name) WHERE deleted_at IS NULL; CREATE INDEX idx_deal_stages_sort ON deal_stages(sort_order); -CREATE TABLE opportunities ( +CREATE TABLE IF NOT EXISTS opportunities ( id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), customer_id UUID NOT NULL REFERENCES customers(id), lead_id UUID REFERENCES leads(id), @@ -530,7 +530,7 @@ CREATE INDEX idx_opportunities_won ON opportunities(is_won) WHERE deleted_at IS CREATE INDEX idx_opportunities_created ON opportunities(created_at DESC) WHERE deleted_at IS NULL; CREATE INDEX idx_opportunities_owner_stage ON opportunities(owner_id, stage_id) WHERE deleted_at IS NULL AND is_won IS NULL; -CREATE TABLE opportunity_products ( +CREATE TABLE IF NOT EXISTS opportunity_products ( id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), opportunity_id UUID NOT NULL REFERENCES opportunities(id) ON DELETE CASCADE, product_id UUID NOT NULL REFERENCES products(id), @@ -549,7 +549,7 @@ CREATE INDEX idx_opp_products_product ON opportunity_products(product_id); -- 9. COMMUNICATION TRACKING -- ============================================================================ -CREATE TABLE communication_types ( +CREATE TABLE IF NOT EXISTS communication_types ( id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), name VARCHAR(50) NOT NULL, description TEXT, @@ -561,7 +561,7 @@ CREATE TABLE communication_types ( CREATE UNIQUE INDEX uq_comm_types_name ON communication_types(name); -CREATE TABLE communications ( +CREATE TABLE IF NOT EXISTS communications ( id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), customer_id UUID NOT NULL REFERENCES customers(id), opportunity_id UUID REFERENCES opportunities(id), @@ -585,7 +585,7 @@ CREATE INDEX idx_communications_type ON communications(type_id) WHERE deleted_at CREATE INDEX idx_communications_created_by ON communications(created_by) WHERE deleted_at IS NULL; CREATE INDEX idx_communications_started ON communications(started_at) WHERE deleted_at IS NULL; -CREATE TABLE communication_participants ( +CREATE TABLE IF NOT EXISTS communication_participants ( id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), communication_id UUID NOT NULL REFERENCES communications(id) ON DELETE CASCADE, user_id UUID REFERENCES users(id), @@ -602,7 +602,7 @@ CREATE INDEX idx_comm_participants_user ON communication_participants(user_id); -- 10. TASKS AND ACTIVITIES -- ============================================================================ -CREATE TABLE task_priorities ( +CREATE TABLE IF NOT EXISTS task_priorities ( id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), name VARCHAR(50) NOT NULL, color VARCHAR(7), @@ -613,7 +613,7 @@ CREATE TABLE task_priorities ( CREATE UNIQUE INDEX uq_task_priorities_name ON task_priorities(name); -CREATE TABLE tasks ( +CREATE TABLE IF NOT EXISTS tasks ( id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), customer_id UUID REFERENCES customers(id), opportunity_id UUID REFERENCES opportunities(id), @@ -645,7 +645,7 @@ CREATE INDEX idx_tasks_reminder ON tasks(reminder_at) WHERE reminder_sent = FALS -- 11. INVOICE SUPPORT -- ============================================================================ -CREATE TABLE invoice_statuses ( +CREATE TABLE IF NOT EXISTS invoice_statuses ( id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), name VARCHAR(50) NOT NULL, description TEXT, @@ -657,7 +657,7 @@ CREATE TABLE invoice_statuses ( CREATE UNIQUE INDEX uq_invoice_statuses_name ON invoice_statuses(name); -CREATE TABLE invoices ( +CREATE TABLE IF NOT EXISTS invoices ( id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), customer_id UUID NOT NULL REFERENCES customers(id), opportunity_id UUID REFERENCES opportunities(id), @@ -689,7 +689,7 @@ CREATE INDEX idx_invoices_issued ON invoices(issued_date DESC) WHERE deleted_at CREATE INDEX idx_invoices_due ON invoices(due_date) WHERE deleted_at IS NULL; -- Skipped: cannot use NOW() in partial index predicate (not IMMUTABLE) -CREATE TABLE invoice_items ( +CREATE TABLE IF NOT EXISTS invoice_items ( id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), invoice_id UUID NOT NULL REFERENCES invoices(id) ON DELETE CASCADE, product_id UUID REFERENCES products(id), @@ -705,7 +705,7 @@ CREATE TABLE invoice_items ( CREATE INDEX idx_invoice_items_invoice ON invoice_items(invoice_id); CREATE INDEX idx_invoice_items_product ON invoice_items(product_id); -CREATE TABLE payments ( +CREATE TABLE IF NOT EXISTS payments ( id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), invoice_id UUID NOT NULL REFERENCES invoices(id), amount DECIMAL(15,2) NOT NULL CHECK (amount > 0), @@ -729,7 +729,7 @@ CREATE INDEX idx_payments_transaction ON payments(transaction_id) WHERE transact -- 12. AUDIT LOGGING -- ============================================================================ -CREATE TABLE audit_logs ( +CREATE TABLE IF NOT EXISTS audit_logs ( id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), table_name VARCHAR(100) NOT NULL, record_id UUID NOT NULL, @@ -755,7 +755,7 @@ CREATE INDEX idx_audit_session ON audit_logs(session_id); -- 13. DUPLICATE DETECTION -- ============================================================================ -CREATE TABLE customer_duplicates ( +CREATE TABLE IF NOT EXISTS customer_duplicates ( id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), customer_id UUID NOT NULL REFERENCES customers(id) ON DELETE CASCADE, duplicate_customer_id UUID NOT NULL REFERENCES customers(id) ON DELETE CASCADE, @@ -810,11 +810,11 @@ BEGIN ) LOOP EXECUTE format( - 'CREATE TRIGGER trg_%I_updated_at + 'DROP TRIGGER IF EXISTS trg_%I_updated_at ON %I; CREATE TRIGGER trg_%I_updated_at BEFORE UPDATE ON %I FOR EACH ROW EXECUTE FUNCTION update_updated_at_column()', - t, t + t, t, t, t ); END LOOP; END; @@ -885,6 +885,7 @@ BEGIN END; $$ LANGUAGE plpgsql SECURITY DEFINER; +DROP TRIGGER IF EXISTS trg_enforce_create_user ON users; CREATE TRIGGER trg_enforce_create_user BEFORE INSERT ON users FOR EACH ROW @@ -935,6 +936,7 @@ BEGIN END; $$ LANGUAGE plpgsql SECURITY DEFINER; +DROP TRIGGER IF EXISTS trg_enforce_role_assignment ON user_roles; CREATE TRIGGER trg_enforce_role_assignment BEFORE INSERT ON user_roles FOR EACH ROW @@ -973,7 +975,7 @@ BEGIN audit_action, CASE WHEN audit_action IN ('UPDATE', 'DELETE') THEN old_row ELSE NULL END, CASE WHEN audit_action IN ('CREATE', 'UPDATE') THEN new_row ELSE NULL END, - NULL + NULLIF(current_setting('app.current_user_id', true), '') ); RETURN COALESCE(NEW, OLD); @@ -998,6 +1000,7 @@ BEGIN END; $$ LANGUAGE plpgsql SECURITY DEFINER; +DROP TRIGGER IF EXISTS trg_audit_ban ON banned_users; CREATE TRIGGER trg_audit_ban AFTER INSERT OR UPDATE ON banned_users FOR EACH ROW @@ -1021,6 +1024,7 @@ BEGIN END; $$ LANGUAGE plpgsql SECURITY DEFINER; +DROP TRIGGER IF EXISTS trg_audit_suspension ON suspended_users; CREATE TRIGGER trg_audit_suspension AFTER INSERT OR UPDATE ON suspended_users FOR EACH ROW @@ -1051,6 +1055,7 @@ BEGIN END; $$ LANGUAGE plpgsql SECURITY DEFINER; +DROP TRIGGER IF EXISTS trg_audit_login ON login_attempts; CREATE TRIGGER trg_audit_login AFTER INSERT ON login_attempts FOR EACH ROW diff --git a/database/migrations/003_chat.sql b/database/migrations/003_chat.sql index 9221490..68cefa2 100644 --- a/database/migrations/003_chat.sql +++ b/database/migrations/003_chat.sql @@ -29,6 +29,8 @@ CREATE INDEX IF NOT EXISTS idx_messages_conversation_id ON messages(conversation CREATE INDEX IF NOT EXISTS idx_messages_created_at ON messages(created_at); CREATE INDEX IF NOT EXISTS idx_conversation_participants_user_id ON conversation_participants(user_id); +CREATE INDEX IF NOT EXISTS idx_messages_conversation_created ON messages(conversation_id, created_at DESC); + -- Seed conversations between superadmin and other users INSERT INTO conversations (id, created_at) VALUES ('c0000000-0000-0000-0000-000000000001', NOW() - INTERVAL '2 hours'), diff --git a/database/migrations/run_all.sql b/database/migrations/run_all.sql index ededccc..a7c3b72 100644 --- a/database/migrations/run_all.sql +++ b/database/migrations/run_all.sql @@ -4,6 +4,9 @@ -- Usage: psql -U postgres -d crm -f run_all.sql -- ============================================================================ +BEGIN; +\set ON_ERROR_STOP on + \echo '=== Running 001_schema.sql (Tables + Constraints + Functions + Views) ===' \i 001_schema.sql @@ -35,9 +38,4 @@ \i 010_chat_notifications.sql \echo '=== Migration Complete ===' -\echo '' -\echo 'Test accounts created:' -\echo ' superadmin_demo / SuperAdmin@2026' -\echo ' admin_demo / AdminAccess@2026' -\echo ' sales_demo / SalesAccess@2026' -\echo ' dev_demo / DevTesting@2026' +COMMIT; diff --git a/fix.md b/fix.md index 1c6a94c..88b2025 100644 --- a/fix.md +++ b/fix.md @@ -1,60 +1,71 @@ -# Facebook Scraper - Configuration Needed +# Scrapers - Configuration & Status -## Proxy Configuration +## Facebook Scraper **File:** `rust-ai/src/main.rs` -**Lines:** 25-27 - -The scraper needs real proxy URLs. The dummy placeholder `"http://0.0.0.0:0"` will fail to connect (expected — no crash, just error logs): +**Lines:** ~70-85 +Target URL (line 72): ```rust -const PROXIES: &[&str] = &[ - "http://0.0.0.0:0", // <- Replace with real proxy(es) -]; +let url = "https://www.facebook.com/search/top/?q=need%20website%20create"; ``` -Replace with your actual proxies, e.g.: -```rust -const PROXIES: &[&str] = &[ - "http://user:pass@192.168.1.1:8080", - "http://user:pass@192.168.1.2:8080", -]; +**Status:** Uses direct connection (no proxy) — your home IP. Facebook blocks datacenter IPs. May work from a residential connection. + +**Test with curl:** +``` +curl.exe -s "https://www.facebook.com/search/top/?q=need%20website%20create" -H "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36" --max-time 10 2>$null ``` -## User Agents (Optional) +**Expected log output when blocked:** +``` +ERROR crm_ai: Facebook scraper error: error sending request for url (https://www.facebook.com/search/top/?q=need%20website%20create) +``` + +## Reddit Scraper **File:** `rust-ai/src/main.rs` -**Lines:** 32-36 +**Lines:** ~90-120 -Add more realistic user agents here if needed. +Uses `old.reddit.com` (older design that allows scraping without captchas). -## Scraper Target URL +Search queries (currently 6, lines 93-100): +- `r/southafrica` — "need website", "web developer" +- Global — '"need a website"', "website quote" +- `r/forhire` — "website" +- `r/smallbusiness` — "website" + +**Status:** Working. Reddit results appear as `INFO LEAD:` entries in the server log. + +**Test with curl:** +``` +curl.exe -s "https://old.reddit.com/r/southafrica/search?q=need+website&sort=new&restrict_sr=on&t=week" -H "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36" +``` + +**Expected log output when working:** +``` +INFO crm_ai: LEAD: [Post Title] -> https://old.reddit.com/r/.../... +``` + +## Background Loop **File:** `rust-ai/src/main.rs` -**Line:** 68 +**Lines:** ~370-383 -Currently fetches `https://www.facebook.com/search/top/?q=need%20website%20create`. Change if needed. +Both scrapers run together every 60-180 seconds on a `spawn_blocking` thread. -## Background Thread +## What You Need to Do -**File:** `rust-ai/src/main.rs` -**Lines:** 355-371 +### Facebook +- Try running from your home PC instead — your residential IP may not be blocked +- If still blocked, you need residential proxies (BrightData, IPRoyal, Oxylabs) +- Configure them in the `PROXIES` array at line 25 -Runs in a `tokio::task::spawn_blocking` thread (changed from `tokio::spawn` because the scraper uses `reqwest::blocking::Client` + `thread::sleep`). +### Reddit +- Works out of the box via `old.reddit.com` +- If it stops working, Reddit IP-blocked you — use proxies or switch to Playwright -## Expected Behavior Until Configured +### To add more search queries +Edit the `searches` array in `run_reddit_scraper()` (line 93). -Until real proxies are set, the Rust server will log this every 1-5 seconds (not a crash): -``` -ERROR crm_ai: Facebook scraper error: error sending request for url (https://www.facebook.com/messages) -``` -Once you add working proxies, these errors stop and actual scraping begins. - -## How it works - -- Runs every 1-5 seconds on a background blocking thread (line 355-371) -- Rotates through proxies and user agents for each request -- Parses conversation HTML via `scraper` crate -- Designed to detect job posts like "need a website" and notify sales reps -- Error handling added to prevent server crash (uses `match` instead of `.unwrap()` at line 69) diff --git a/next.config.ts b/next.config.ts index fc53880..2e27cc2 100644 --- a/next.config.ts +++ b/next.config.ts @@ -2,7 +2,7 @@ import type { NextConfig } from "next" const nextConfig: NextConfig = { eslint: { - ignoreDuringBuilds: true, + ignoreDuringBuilds: false, }, images: { remotePatterns: [ diff --git a/package.json b/package.json index 57b42d6..8aee1d3 100644 --- a/package.json +++ b/package.json @@ -4,13 +4,14 @@ "private": true, "scripts": { "dev": "npm run dev:precheck & npm run dev:ollama & npm run dev:start", - "dev:start": "concurrently -n AI,NEXT -c cyan,green \"npm run dev:rust\" \"npm run dev:next\"", + "dev:start": "concurrently -n AI,BROWSE,NEXT -c cyan,magenta,green \"npm run dev:rust\" \"npm run dev:browser-use\" \"npm run dev:next\"", "dev:next": "next dev -p 3006", - "dev:precheck": "powershell -NoProfile -Command \"$targetPorts=3001,3006; netstat -ano | Select-String LISTENING | ForEach-Object { $line=$_.ToString(); foreach($p in $targetPorts){ if($line -match ('[:]'+$p+'\\s')){ $foundPid=($line -split '\\s+')[-1]; try{ Stop-Process -Id $foundPid -Force -ErrorAction SilentlyContinue; Write-Host ('Freed port '+$p) }catch{} } } }; exit 0\"", + "dev:precheck": "powershell -NoProfile -Command \"Get-NetTCPConnection -State Listen | Where-Object { $_.LocalPort -in 3001,3006,3008 } | ForEach-Object { Stop-Process -Id $_.OwningProcess -Force -ErrorAction SilentlyContinue; Write-Host ('Freed port '+$_.LocalPort) }\"", "dev:ollama": "powershell -NoProfile -Command \"if (-not (Get-Process ollama -ErrorAction SilentlyContinue)) { Start-Process ollama -ArgumentList 'serve' -WindowStyle Hidden; Start-Sleep 3 }; exit 0\"", "dev:rust": "cd rust-ai && cargo run", + "dev:browser-use": "cd browser-use-service && python main.py", "build": "next build", - "start": "npm run dev:next", + "start": "next start -p 3006", "lint": "eslint" }, "dependencies": { @@ -36,7 +37,6 @@ "bcryptjs": "^3.0.3", "class-variance-authority": "^0.7.1", "clsx": "^2.1.1", - "emoji-mart": "^5.6.0", "framer-motion": "^11.15.0", "jose": "^6.2.3", "lucide-react": "^0.468.0", diff --git a/rust-ai/Cargo.lock b/rust-ai/Cargo.lock index 6d3a2fb..1778fb6 100644 --- a/rust-ai/Cargo.lock +++ b/rust-ai/Cargo.lock @@ -34,7 +34,7 @@ checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn", ] [[package]] @@ -73,7 +73,7 @@ dependencies = [ "http-body-util", "hyper", "hyper-util", - "itoa 1.0.18", + "itoa", "matchit", "memchr", "mime", @@ -119,12 +119,6 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" -[[package]] -name = "bitflags" -version = "1.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" - [[package]] name = "bitflags" version = "2.13.0" @@ -226,12 +220,6 @@ dependencies = [ "crossbeam-utils", ] -[[package]] -name = "convert_case" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e" - [[package]] name = "core-foundation" version = "0.9.4" @@ -298,9 +286,9 @@ dependencies = [ "axum", "chrono", "dotenvy", + "jsonwebtoken", "rand 0.8.6", "reqwest", - "scraper", "serde", "serde_json", "sqlx", @@ -345,33 +333,6 @@ dependencies = [ "hybrid-array", ] -[[package]] -name = "cssparser" -version = "0.27.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "754b69d351cdc2d8ee09ae203db831e005560fc6030da058f86ad60c92a9cb0a" -dependencies = [ - "cssparser-macros", - "dtoa-short", - "itoa 0.4.8", - "matches", - "phf", - "proc-macro2", - "quote", - "smallvec", - "syn 1.0.109", -] - -[[package]] -name = "cssparser-macros" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13b588ba4ac1a99f7f2964d24b3d896ddc6bf847ee3855dbd4366f058cfcd331" -dependencies = [ - "quote", - "syn 2.0.118", -] - [[package]] name = "ctutils" version = "0.4.2" @@ -382,17 +343,10 @@ dependencies = [ ] [[package]] -name = "derive_more" -version = "0.99.20" +name = "deranged" +version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6edb4b64a43d977b8e99788fe3a04d483834fba1215a7e02caa415b626497f7f" -dependencies = [ - "convert_case", - "proc-macro2", - "quote", - "rustc_version", - "syn 2.0.118", -] +checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" [[package]] name = "digest" @@ -423,7 +377,7 @@ checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn", ] [[package]] @@ -432,27 +386,6 @@ version = "0.15.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" -[[package]] -name = "dtoa" -version = "1.0.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c3cf4824e2d5f025c7b531afcb2325364084a16806f6d47fbc1f5fbd9960590" - -[[package]] -name = "dtoa-short" -version = "0.3.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd1511a7b6a56299bd043a9c167a6d2bfb37bf84a6dfceaba651168adfb43c87" -dependencies = [ - "dtoa", -] - -[[package]] -name = "ego-tree" -version = "0.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12a0bb14ac04a9fcf170d0bbbef949b44cc492f4452bd20c095636956f653642" - [[package]] name = "either" version = "1.16.0" @@ -567,16 +500,6 @@ dependencies = [ "percent-encoding", ] -[[package]] -name = "futf" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df420e2e84819663797d1ec6544b13c5be84629e7bb00dc960d6917db2987843" -dependencies = [ - "mac", - "new_debug_unreachable", -] - [[package]] name = "futures-channel" version = "0.3.32" @@ -648,15 +571,6 @@ dependencies = [ "slab", ] -[[package]] -name = "fxhash" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c" -dependencies = [ - "byteorder", -] - [[package]] name = "generic-array" version = "0.14.7" @@ -667,26 +581,6 @@ dependencies = [ "version_check", ] -[[package]] -name = "getopts" -version = "0.2.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfe4fbac503b8d1f88e6676011885f34b7174f46e59956bba534ba83abded4df" -dependencies = [ - "unicode-width", -] - -[[package]] -name = "getrandom" -version = "0.1.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce" -dependencies = [ - "cfg-if", - "libc", - "wasi 0.9.0+wasi-snapshot-preview1", -] - [[package]] name = "getrandom" version = "0.2.17" @@ -694,8 +588,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" dependencies = [ "cfg-if", + "js-sys", "libc", - "wasi 0.11.1+wasi-snapshot-preview1", + "wasi", + "wasm-bindgen", ] [[package]] @@ -785,20 +681,6 @@ dependencies = [ "digest 0.11.3", ] -[[package]] -name = "html5ever" -version = "0.25.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5c13fb08e5d4dfc151ee5e88bae63f7773d61852f3bdc73c9f4b9e1bde03148" -dependencies = [ - "log", - "mac", - "markup5ever", - "proc-macro2", - "quote", - "syn 1.0.109", -] - [[package]] name = "http" version = "1.4.2" @@ -806,7 +688,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6970f50e31d6fc17d3fa27329444bfa74e196cf62e95052a3f6fee181dba6425" dependencies = [ "bytes", - "itoa 1.0.18", + "itoa", ] [[package]] @@ -868,7 +750,7 @@ dependencies = [ "http-body", "httparse", "httpdate", - "itoa 1.0.18", + "itoa", "pin-project-lite", "smallvec", "tokio", @@ -1074,12 +956,6 @@ version = "2.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" -[[package]] -name = "itoa" -version = "0.4.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4" - [[package]] name = "itoa" version = "1.0.18" @@ -1097,6 +973,21 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "jsonwebtoken" +version = "9.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a87cc7a48537badeae96744432de36f4be2b4a34a05a5ef32e9dd8a1c169dde" +dependencies = [ + "base64", + "js-sys", + "pem", + "ring", + "serde", + "serde_json", + "simple_asn1", +] + [[package]] name = "lazy_static" version = "1.5.0" @@ -1146,26 +1037,6 @@ version = "0.4.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" -[[package]] -name = "mac" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4" - -[[package]] -name = "markup5ever" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a24f40fb03852d1cdd84330cddcaf98e9ec08a7b7768e952fad3b4cf048ec8fd" -dependencies = [ - "log", - "phf", - "phf_codegen", - "string_cache", - "string_cache_codegen", - "tendril", -] - [[package]] name = "matchers" version = "0.2.0" @@ -1175,12 +1046,6 @@ dependencies = [ "regex-automata", ] -[[package]] -name = "matches" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2532096657941c2fea9c289d370a250971c689d4f143798ff67113ec042024a5" - [[package]] name = "matchit" version = "0.7.3" @@ -1216,7 +1081,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "02bd0af71c67b473010cbbc60715ee815645a4dc942899111f494b4b737d6fda" dependencies = [ "libc", - "wasi 0.11.1+wasi-snapshot-preview1", + "wasi", "windows-sys 0.61.2", ] @@ -1237,18 +1102,6 @@ dependencies = [ "tempfile", ] -[[package]] -name = "new_debug_unreachable" -version = "1.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086" - -[[package]] -name = "nodrop" -version = "0.1.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72ef4a56884ca558e5ddb05a1d1e7e1bfd9a68d9ed024c21704cc98872dae1bb" - [[package]] name = "nu-ansi-term" version = "0.50.3" @@ -1258,6 +1111,31 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "num-bigint" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-conv" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" + +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -1279,7 +1157,7 @@ version = "0.10.81" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77823a27f0babb03091cb9ed9ef80af3b39dbc82f97e8fa530374b7dafd87a45" dependencies = [ - "bitflags 2.13.0", + "bitflags", "cfg-if", "foreign-types", "libc", @@ -1295,7 +1173,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn", ] [[package]] @@ -1345,85 +1223,22 @@ dependencies = [ "windows-link", ] +[[package]] +name = "pem" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d30c53c26bc5b31a98cd02d20f25a7c8567146caf63ed593a9d87b2775291be" +dependencies = [ + "base64", + "serde_core", +] + [[package]] name = "percent-encoding" version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" -[[package]] -name = "phf" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3dfb61232e34fcb633f43d12c58f83c1df82962dcdfa565a4e866ffc17dafe12" -dependencies = [ - "phf_macros", - "phf_shared 0.8.0", - "proc-macro-hack", -] - -[[package]] -name = "phf_codegen" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cbffee61585b0411840d3ece935cce9cb6321f01c45477d30066498cd5e1a815" -dependencies = [ - "phf_generator 0.8.0", - "phf_shared 0.8.0", -] - -[[package]] -name = "phf_generator" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17367f0cc86f2d25802b2c26ee58a7b23faeccf78a396094c13dced0d0182526" -dependencies = [ - "phf_shared 0.8.0", - "rand 0.7.3", -] - -[[package]] -name = "phf_generator" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d" -dependencies = [ - "phf_shared 0.11.3", - "rand 0.8.6", -] - -[[package]] -name = "phf_macros" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f6fde18ff429ffc8fe78e2bf7f8b7a5a5a6e2a8b58bc5a9ac69198bbda9189c" -dependencies = [ - "phf_generator 0.8.0", - "phf_shared 0.8.0", - "proc-macro-hack", - "proc-macro2", - "quote", - "syn 1.0.109", -] - -[[package]] -name = "phf_shared" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c00cf8b9eafe68dde5e9eaa2cef8ee84a9336a47d566ec55ca16589633b65af7" -dependencies = [ - "siphasher 0.3.11", -] - -[[package]] -name = "phf_shared" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67eabc2ef2a60eb7faa00097bd1ffdb5bd28e62bf39990626a582201b7a754e5" -dependencies = [ - "siphasher 1.0.3", -] - [[package]] name = "pin-project-lite" version = "0.2.17" @@ -1445,6 +1260,12 @@ dependencies = [ "zerovec", ] +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + [[package]] name = "ppv-lite86" version = "0.2.21" @@ -1454,18 +1275,6 @@ dependencies = [ "zerocopy", ] -[[package]] -name = "precomputed-hash" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c" - -[[package]] -name = "proc-macro-hack" -version = "0.5.20+deprecated" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068" - [[package]] name = "proc-macro2" version = "1.0.106" @@ -1490,20 +1299,6 @@ version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" -[[package]] -name = "rand" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" -dependencies = [ - "getrandom 0.1.16", - "libc", - "rand_chacha 0.2.2", - "rand_core 0.5.1", - "rand_hc", - "rand_pcg", -] - [[package]] name = "rand" version = "0.8.6" @@ -1511,7 +1306,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5ca0ecfa931c29007047d1bc58e623ab12e5590e8c7cc53200d5202b69266d8a" dependencies = [ "libc", - "rand_chacha 0.3.1", + "rand_chacha", "rand_core 0.6.4", ] @@ -1526,16 +1321,6 @@ dependencies = [ "rand_core 0.10.1", ] -[[package]] -name = "rand_chacha" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402" -dependencies = [ - "ppv-lite86", - "rand_core 0.5.1", -] - [[package]] name = "rand_chacha" version = "0.3.1" @@ -1546,15 +1331,6 @@ dependencies = [ "rand_core 0.6.4", ] -[[package]] -name = "rand_core" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" -dependencies = [ - "getrandom 0.1.16", -] - [[package]] name = "rand_core" version = "0.6.4" @@ -1570,31 +1346,13 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" -[[package]] -name = "rand_hc" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c" -dependencies = [ - "rand_core 0.5.1", -] - -[[package]] -name = "rand_pcg" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16abd0c1b639e9eb4d7c50c0b8100b0d0f849be2349829c740fe8e6eb4816429" -dependencies = [ - "rand_core 0.5.1", -] - [[package]] name = "redox_syscall" version = "0.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" dependencies = [ - "bitflags 2.13.0", + "bitflags", ] [[package]] @@ -1670,22 +1428,13 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "rustc_version" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" -dependencies = [ - "semver", -] - [[package]] name = "rustix" version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" dependencies = [ - "bitflags 2.13.0", + "bitflags", "errno", "libc", "linux-raw-sys", @@ -1752,29 +1501,13 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" -[[package]] -name = "scraper" -version = "0.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48e02aa790c80c2e494130dec6a522033b6a23603ffc06360e9fe6c611ea2c12" -dependencies = [ - "cssparser", - "ego-tree", - "getopts", - "html5ever", - "matches", - "selectors", - "smallvec", - "tendril", -] - [[package]] name = "security-framework" version = "3.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" dependencies = [ - "bitflags 2.13.0", + "bitflags", "core-foundation 0.10.1", "core-foundation-sys", "libc", @@ -1791,32 +1524,6 @@ dependencies = [ "libc", ] -[[package]] -name = "selectors" -version = "0.22.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df320f1889ac4ba6bc0cdc9c9af7af4bd64bb927bccdf32d81140dc1f9be12fe" -dependencies = [ - "bitflags 1.3.2", - "cssparser", - "derive_more", - "fxhash", - "log", - "matches", - "phf", - "phf_codegen", - "precomputed-hash", - "servo_arc", - "smallvec", - "thin-slice", -] - -[[package]] -name = "semver" -version = "1.0.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" - [[package]] name = "serde" version = "1.0.228" @@ -1844,7 +1551,7 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn", ] [[package]] @@ -1853,7 +1560,7 @@ version = "1.0.150" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" dependencies = [ - "itoa 1.0.18", + "itoa", "memchr", "serde", "serde_core", @@ -1866,7 +1573,7 @@ version = "0.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457" dependencies = [ - "itoa 1.0.18", + "itoa", "serde", "serde_core", ] @@ -1878,21 +1585,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" dependencies = [ "form_urlencoded", - "itoa 1.0.18", + "itoa", "ryu", "serde", ] -[[package]] -name = "servo_arc" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d98238b800e0d1576d8b6e3de32827c2d74bee68bb97748dcf5071fb53965432" -dependencies = [ - "nodrop", - "stable_deref_trait", -] - [[package]] name = "sha1" version = "0.11.0" @@ -1952,16 +1649,16 @@ dependencies = [ ] [[package]] -name = "siphasher" -version = "0.3.11" +name = "simple_asn1" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38b58827f4464d87d377d175e90bf58eb00fd8716ff0a62f80356b5e61555d0d" - -[[package]] -name = "siphasher" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ee5873ec9cce0195efcb7a4e9507a04cd49aec9c83d0389df45b1ef7ba2e649" +checksum = "0d585997b0ac10be3c5ee635f1bab02d512760d14b7c468801ac8a01d9ae5f1d" +dependencies = [ + "num-bigint", + "num-traits", + "thiserror", + "time", +] [[package]] name = "slab" @@ -2056,7 +1753,7 @@ dependencies = [ "quote", "sqlx-core", "sqlx-macros-core", - "syn 2.0.118", + "syn", ] [[package]] @@ -2079,7 +1776,7 @@ dependencies = [ "sqlx-mysql", "sqlx-postgres", "sqlx-sqlite", - "syn 2.0.118", + "syn", "thiserror", "tokio", "url", @@ -2091,7 +1788,7 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "90b8020fe17c5f2c245bfa2505d7ef59c5604839527c740266ad2214acebea27" dependencies = [ - "bitflags 2.13.0", + "bitflags", "byteorder", "bytes", "chrono", @@ -2121,7 +1818,7 @@ checksum = "87a2bdd6e83f6b3ea525ca9fee568030508b58355a43d0b2c1674d5f79dcd65e" dependencies = [ "atoi", "base64", - "bitflags 2.13.0", + "bitflags", "byteorder", "chrono", "crc", @@ -2133,7 +1830,7 @@ dependencies = [ "hex", "hkdf", "hmac", - "itoa 1.0.18", + "itoa", "log", "md-5", "memchr", @@ -2182,31 +1879,6 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" -[[package]] -name = "string_cache" -version = "0.8.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf776ba3fa74f83bf4b63c3dcbbf82173db2632ed8452cb2d891d33f459de70f" -dependencies = [ - "new_debug_unreachable", - "parking_lot", - "phf_shared 0.11.3", - "precomputed-hash", - "serde", -] - -[[package]] -name = "string_cache_codegen" -version = "0.5.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c711928715f1fe0fe509c53b43e993a9a557babc2d0a3567d0a3006f1ac931a0" -dependencies = [ - "phf_generator 0.11.3", - "phf_shared 0.11.3", - "proc-macro2", - "quote", -] - [[package]] name = "stringprep" version = "0.1.5" @@ -2224,17 +1896,6 @@ version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" -[[package]] -name = "syn" -version = "1.0.109" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - [[package]] name = "syn" version = "2.0.118" @@ -2263,7 +1924,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn", ] [[package]] @@ -2272,7 +1933,7 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" dependencies = [ - "bitflags 2.13.0", + "bitflags", "core-foundation 0.9.4", "system-configuration-sys", ] @@ -2300,23 +1961,6 @@ dependencies = [ "windows-sys 0.61.2", ] -[[package]] -name = "tendril" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d24a120c5fc464a3458240ee02c299ebcb9d67b5249c8848b09d639dca8d7bb0" -dependencies = [ - "futf", - "mac", - "utf-8", -] - -[[package]] -name = "thin-slice" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8eaa81235c7058867fa8c0e7314f33dcce9c215f535d1913822a2b3f5e289f3c" - [[package]] name = "thiserror" version = "2.0.18" @@ -2334,7 +1978,7 @@ checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn", ] [[package]] @@ -2346,6 +1990,36 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "time" +version = "0.3.51" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85c17d80feb7334b40c484e45ed1a5273dfd8bfda537c3be2e74a06a6686f327" +dependencies = [ + "deranged", + "num-conv", + "powerfmt", + "serde_core", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" + +[[package]] +name = "time-macros" +version = "0.2.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcef1a61bdb119096e153208ec5cbec23944ce8bca13be5c7f60c634f7403935" +dependencies = [ + "num-conv", + "time-core", +] + [[package]] name = "tinystr" version = "0.8.3" @@ -2380,7 +2054,6 @@ dependencies = [ "bytes", "libc", "mio", - "parking_lot", "pin-project-lite", "signal-hook-registry", "socket2", @@ -2396,7 +2069,7 @@ checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn", ] [[package]] @@ -2465,7 +2138,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e9cd434a998747dd2c4276bc96ee2e0c7a2eadf3cae88e52be55a05fa9053f5" dependencies = [ - "bitflags 2.13.0", + "bitflags", "bytes", "http", "http-body", @@ -2481,7 +2154,7 @@ version = "0.6.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" dependencies = [ - "bitflags 2.13.0", + "bitflags", "bytes", "futures-util", "http", @@ -2525,7 +2198,7 @@ checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn", ] [[package]] @@ -2606,12 +2279,6 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7df058c713841ad818f1dc5d3fd88063241cc61f49f5fbea4b951e8cf5a8d71d" -[[package]] -name = "unicode-width" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" - [[package]] name = "untrusted" version = "0.9.0" @@ -2630,12 +2297,6 @@ dependencies = [ "serde", ] -[[package]] -name = "utf-8" -version = "0.7.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" - [[package]] name = "utf8_iter" version = "1.0.4" @@ -2681,12 +2342,6 @@ dependencies = [ "try-lock", ] -[[package]] -name = "wasi" -version = "0.9.0+wasi-snapshot-preview1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" - [[package]] name = "wasi" version = "0.11.1+wasi-snapshot-preview1" @@ -2735,7 +2390,7 @@ dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn 2.0.118", + "syn", "wasm-bindgen-shared", ] @@ -2785,7 +2440,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn", ] [[package]] @@ -2796,7 +2451,7 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn", ] [[package]] @@ -2941,7 +2596,7 @@ checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn", "synstructure", ] @@ -2962,7 +2617,7 @@ checksum = "1ae7f38b72ec2a254e2b87ef277cf2cd4fb97cbebf944faa6f33354da0867930" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn", ] [[package]] @@ -2982,7 +2637,7 @@ checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn", "synstructure", ] @@ -3022,7 +2677,7 @@ checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn", ] [[package]] diff --git a/rust-ai/Cargo.toml b/rust-ai/Cargo.toml index 3e8583a..e818717 100644 --- a/rust-ai/Cargo.toml +++ b/rust-ai/Cargo.toml @@ -6,7 +6,7 @@ description = "AI Sales Assistant backend for Coast IT CRM" [dependencies] axum = "0.7" -tokio = { version = "1", features = ["full"] } +tokio = { version = "1", features = ["rt-multi-thread", "macros", "sync", "time", "net", "process"] } reqwest = { version = "0.12", features = ["json", "blocking"] } serde = { version = "1", features = ["derive"] } serde_json = "1" @@ -17,5 +17,5 @@ tracing = "0.1" tracing-subscriber = { version = "0.3", features = ["env-filter"] } tower-http = { version = "0.5", features = ["cors"] } dotenvy = "0.15" -scraper = "0.12" -rand = "0.8" \ No newline at end of file +rand = "0.8" +jsonwebtoken = "9" \ No newline at end of file diff --git a/rust-ai/src/instructions.rs b/rust-ai/src/instructions.rs deleted file mode 100644 index 00fc80d..0000000 --- a/rust-ai/src/instructions.rs +++ /dev/null @@ -1,85 +0,0 @@ -use std::fs; -use std::path::PathBuf; -use std::sync::Arc; -use tokio::sync::RwLock; -use tracing::{error, info, warn}; - -/// Manages the ai.md self-improvement file. -/// Loaded on startup and periodically refreshed. -pub struct InstructionsManager { - path: PathBuf, - current: RwLock, -} - -impl InstructionsManager { - pub fn new(path: PathBuf) -> Self { - let initial = fs::read_to_string(&path).unwrap_or_default(); - if initial.is_empty() { - warn!("ai.md is empty or missing at {:?}", path); - } else { - info!("Loaded ai.md ({} bytes)", initial.len()); - } - Self { - path, - current: RwLock::new(initial), - } - } - - /// Read the current instructions - pub async fn get(&self) -> String { - self.current.read().await.clone() - } - - /// Append a new entry to the Improvement Log and optionally update the instructions. - /// Returns the updated full content. - pub async fn update(&self, entry: &str, new_content: Option<&str>) -> Result { - if let Some(content) = new_content { - // Full replacement - fs::write(&self.path, content).map_err(|e| format!("Write failed: {}", e))?; - let mut current = self.current.write().await; - *current = content.to_string(); - info!("ai.md fully replaced ({} bytes)", content.len()); - Ok(content.to_string()) - } else { - // Append to Improvement Log only - let mut current = self.current.write().await; - let log_entry = format!("\n- {} — {}", chrono::Utc::now().format("%Y-%m-%d %H:%M"), entry); - // Find the Improvement Log section and append - if let Some(pos) = current.rfind("\n## Improvement Log") { - // Find the next section after Improvement Log, or end of file - let after_log = ¤t[pos..]; - if let Some(section_start) = after_log[1..].find("\n## ") { - let insert_at = pos + 1 + section_start; - current.insert_str(insert_at, &format!("{}\n", log_entry)); - } else { - current.push_str(&format!("{}\n", log_entry)); - } - } else { - current.push_str(&format!("\n## Improvement Log\n{}", log_entry)); - } - fs::write(&self.path, &*current).map_err(|e| format!("Write failed: {}", e))?; - info!("ai.md improvement log appended: {}", entry); - Ok(current.clone()) - } - } - - /// Reload from disk - pub async fn reload(&self) { - match fs::read_to_string(&self.path) { - Ok(content) => { - let len = content.len(); - let mut current = self.current.write().await; - *current = content; - info!("ai.md reloaded from disk ({} bytes)", len); - } - Err(e) => error!("Failed to reload ai.md: {}", e), - } - } -} - -/// Wrapper for thread-safe sharing -pub type SharedInstructions = Arc; - -pub fn create_shared(path: PathBuf) -> SharedInstructions { - Arc::new(InstructionsManager::new(path)) -} diff --git a/rust-ai/src/main.rs b/rust-ai/src/main.rs index 4c9fdfe..174c1f5 100644 --- a/rust-ai/src/main.rs +++ b/rust-ai/src/main.rs @@ -1,93 +1,67 @@ use axum::{ extract::State, - http::Method, + http::{HeaderMap, Method, StatusCode}, routing::{get, post}, Json, Router, }; +use tower_http::cors::{CorsLayer, AllowOrigin, Any}; +use jsonwebtoken::{decode, DecodingKey, Validation, Algorithm}; use serde::{Deserialize, Serialize}; use sqlx::postgres::PgPoolOptions; +use std::collections::HashMap; use std::fs; -use std::sync::{Arc, Mutex}; -use tower_http::cors::{Any, CorsLayer}; -use tracing::{error, info}; +use std::sync::Arc; +use tokio::sync::Mutex; +use tracing::{error, info, warn}; use uuid::Uuid; -use reqwest::blocking::Client; -use scraper::{Html, Selector}; -use rand::rngs::StdRng; -use rand::SeedableRng; use rand::Rng; use std::time::Duration; -use std::thread; +use std::time::{SystemTime, UNIX_EPOCH}; -// ── Facebook Scraper ─────────────────────────────────────────── +// ── JWT Claims ──────────────────────────────────────────────── -// List of proxies -const PROXIES: &[&str] = &[ - "http://user:pass@192.168.1.1:8080", - "http://user:pass@192.168.1.2:8080", - "http://user:pass@192.168.1.3:8080", - "http://user:pass@192.168.1.4:8080", - "http://user:pass@192.168.1.5:8080", -]; - -// List of user agents -const USER_AGENTS: &[&str] = &[ - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3", - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.1.2 Safari/605.1.15", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:61.0) Gecko/20100101 Firefox/61.0", - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.1.2 Safari/605.1.15", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36", - // Add more user agents here -]; - -fn get_random_proxy() -> String { - let mut rng = rand::thread_rng(); - PROXIES[rng.gen_range(0..PROXIES.len())].to_string() +#[derive(Debug, Deserialize)] +struct Claims { + #[serde(rename = "userId")] + user_id: String, + role: String, } -fn get_random_user_agent() -> String { - let mut rng = rand::thread_rng(); - USER_AGENTS[rng.gen_range(0..USER_AGENTS.len())].to_string() +fn verify_jwt(token: &str, secret: &str) -> Option { + let key = DecodingKey::from_secret(secret.as_bytes()); + let validation = Validation::new(Algorithm::HS256); + decode::(token, &key, &validation).ok().map(|d| d.claims) } -fn scrape_conversations(url: &str) -> Result> { - let proxy = get_random_proxy(); - let user_agent = get_random_user_agent(); - let client = Client::builder() - .proxy(reqwest::Proxy::all(proxy)?) - .build()?; +// ── Rate limiter ────────────────────────────────────────────── - let response = client.get(url) - .header("User-Agent", user_agent) - .send()?; - - if response.status().is_success() { - let html = response.text()?; - Ok(Html::parse_document(&html)) - } else { - Err(format!("Failed to retrieve the page: {}", response.status()).into()) - } +struct RateLimiter { + buckets: Mutex>>, + max_requests: usize, + window_secs: u64, } -fn run_facebook_scraper() { - let url = "https://www.facebook.com/search/top/?q=need%20website%20create"; - match scrape_conversations(url) { - Ok(soup) => { - // Example: Extract conversation data - let selector = Selector::parse("div.conversation").unwrap(); - for element in soup.select(&selector) { - println!("Conversation: {}", element.inner_html()); - } - } - Err(e) => { - error!("Facebook scraper error: {}", e); +impl RateLimiter { + fn new(max_requests: usize, window_secs: u64) -> Self { + Self { + buckets: Mutex::new(HashMap::new()), + max_requests, + window_secs, } } - // Introduce random delay - let mut rng = rand::thread_rng(); - let delay = rng.gen_range(1..5); // Delay between 1 to 5 seconds - thread::sleep(Duration::from_secs(delay)); + async fn check(&self, key: &str) -> bool { + let now = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs(); + let mut buckets = self.buckets.lock().await; + let timestamps = buckets.entry(key.to_string()).or_default(); + timestamps.retain(|t| now - *t <= self.window_secs); + if timestamps.len() >= self.max_requests { + false + } else { + timestamps.push(now); + true + } + } } // ── Shared state ─────────────────────────────────────────────── @@ -97,6 +71,48 @@ struct AppState { ollama_url: String, model: String, jobs: Vec, + leads: Arc>, + http_client: reqwest::Client, + jwt_secret: String, + rate_limiter: RateLimiter, +} + +#[derive(Debug, Clone, Serialize)] +struct Lead { + title: String, + url: String, + source: String, + found_at: u64, + author: String, + date: String, + content: String, +} + +struct LeadStore { + leads: Vec, + max_size: usize, +} + +impl LeadStore { + fn new(max_size: usize) -> Self { + Self { leads: Vec::new(), max_size } + } + + fn push(&mut self, lead: Lead) { + if !self.leads.iter().any(|l| l.url == lead.url) { + self.leads.insert(0, lead); + self.leads.truncate(self.max_size); + } + } + + fn recent(&self, max_age_secs: u64, limit: usize) -> Vec { + let now = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs(); + self.leads.iter() + .filter(|l| now.saturating_sub(l.found_at) <= max_age_secs) + .take(limit) + .cloned() + .collect() + } } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -110,8 +126,6 @@ struct Job { #[derive(Debug, Deserialize)] struct ChatRequest { message: String, - user_id: String, - user_role: String, } #[derive(Debug, Serialize)] @@ -162,23 +176,73 @@ struct OllamaResponseMessage { content: String, } -// ── System prompt builder ───────────────────────────────────── +// ── Helpers ──────────────────────────────────────────────────── -fn build_system_prompt(jobs: &[Job]) -> String { +fn truncate(s: &str, max: usize) -> String { + s.chars().take(max).collect() +} + +fn extract_claims(headers: &HeaderMap, state: &AppState) -> Result { + let auth_header = headers.get("Authorization").and_then(|v| v.to_str().ok()).unwrap_or(""); + let token = auth_header.strip_prefix("Bearer ").unwrap_or(""); + let claims = verify_jwt(token, &state.jwt_secret).ok_or_else(|| { + (StatusCode::UNAUTHORIZED, "Unauthorized".to_string()) + })?; + match claims.role.as_str() { + "sales" | "admin" | "super_admin" => Ok(claims), + _ => Err((StatusCode::FORBIDDEN, "Forbidden".to_string())), + } +} + +fn format_leads_output(leads: &[Lead]) -> String { + if leads.is_empty() { + return "No new requests found yet.".to_string(); + } + leads + .iter() + .enumerate() + .map(|(i, l)| { + let author = if l.author.is_empty() { "Unknown" } else { &l.author }; + let date = truncate(&l.date, 10); + format!( + "{}. {}\n {}\n {}\n {}", + i + 1, + author, + date, + l.title, + l.url + ) + }) + .collect::>() + .join("\n") +} + +fn build_system_prompt(jobs: &[Job], leads: &[Lead]) -> String { let job_list: Vec = jobs .iter() .map(|j| format!("- {} ({}): {}", j.job_title, j.industry, j.description)) .collect(); let job_list_str = job_list.join("\n"); + let lead_summary: Vec = leads + .iter() + .map(|l| format!("{} | {} | {}", l.author, l.title, l.url)) + .collect(); + let lead_summary_str = if lead_summary.is_empty() { + "None yet.".to_string() + } else { + lead_summary.join("\n") + }; + format!( - "You are a Sales AI Assistant for Coast IT CRM. Your role is to help salespeople \ - with tips, strategies, and guidance.\n\n\ + "You are a Sales AI Assistant for Coast IT CRM.\n\n\ Available job categories to target:\n{}\n\n\ - Provide concise, actionable sales advice. When asked about a specific job category, \ - give targeted tips on finding and engaging prospects in that field. \ - Keep responses under 300 words unless asked for detail.", - job_list_str + Recent leads context:\n{}\n\n\ + Rules:\n\ + - When asked about leads, answer concisely under 150 words.\n\ + - If asked to suggest a sales strategy, give brief actionable advice.\n\ + - Be direct and professional. No fluff.", + job_list_str, lead_summary_str ) } @@ -186,90 +250,119 @@ fn build_system_prompt(jobs: &[Job]) -> String { async fn handle_chat( State(state): State>, + headers: HeaderMap, Json(req): Json, -) -> Result, (axum::http::StatusCode, String)> { - // Validate role - match req.user_role.as_str() { - "sales" | "admin" | "super_admin" => {} - _ => return Err((axum::http::StatusCode::FORBIDDEN, "Forbidden".to_string())), +) -> Result, (StatusCode, String)> { + let claims = extract_claims(&headers, &state)?; + + if !state.rate_limiter.check(&claims.user_id).await { + return Err((StatusCode::TOO_MANY_REQUESTS, "Rate limit exceeded".to_string())); } - let system_prompt = build_system_prompt(&state.jobs); + let msg_lower = req.message.to_lowercase(); + let msg_words: Vec<&str> = msg_lower.split_whitespace().collect(); + let has_listing = msg_words.iter().any(|w| ["listings", "listing", "leads", "links", "lists"].contains(w)); + let has_show = msg_lower.contains("show me") || msg_lower.contains("give me") || msg_lower.contains("pull"); + let has_job = msg_words.contains(&"jobs") || msg_words.contains(&"job"); + if has_listing || (has_show && has_job) || (has_show && msg_lower.contains("links")) || msg_lower.contains("recent leads") { + let now = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs(); + let service_url = "http://localhost:3008/scrape/all".to_string(); + if let Ok(resp) = state.http_client + .post(&service_url) + .timeout(Duration::from_secs(120)) + .send() + .await + { + if let Ok(leads_data) = resp.json::>().await { + info!("Scraped {} leads from {}", leads_data.len(), service_url); + let mut store = state.leads.lock().await; + for item in &leads_data { + store.push(Lead { + title: truncate(item["title"].as_str().unwrap_or(""), 120), + url: item["url"].as_str().unwrap_or("").to_string(), + source: item["source"].as_str().unwrap_or("unknown").to_string(), + found_at: now, + author: truncate(item["author"].as_str().unwrap_or(""), 60), + date: truncate(item["date"].as_str().unwrap_or(""), 30), + content: truncate(item["content"].as_str().unwrap_or(""), 300), + }); + } + } + } else { + error!("Scraper service unreachable: {}", service_url); + } + let recent_leads = state.leads.lock().await.recent(604800, 20); + let response = format_leads_output(&recent_leads); + let _ = sqlx::query( + "INSERT INTO ai_conversations (id, user_id, role, message, response) VALUES ($1, $2, $3, $4, $5)", + ) + .bind(Uuid::new_v4()) + .bind(Uuid::parse_str(&claims.user_id).unwrap_or(Uuid::nil())) + .bind(&claims.role) + .bind(&req.message) + .bind(&response) + .execute(&state.db) + .await; + return Ok(Json(ChatResponse { response })); + } + + let recent_leads = state.leads.lock().await.recent(604800, 20); + let system_prompt = build_system_prompt(&state.jobs, &recent_leads); let message_text = req.message.clone(); let ollama_req = OllamaRequest { model: state.model.clone(), messages: vec![ - OllamaChatMessage { - role: "system".to_string(), - content: system_prompt, - }, - OllamaChatMessage { - role: "user".to_string(), - content: req.message.clone(), - }, + OllamaChatMessage { role: "system".to_string(), content: system_prompt }, + OllamaChatMessage { role: "user".to_string(), content: req.message.clone() }, ], stream: false, - options: OllamaOptions { - temperature: 0.7, - num_predict: 1024, - }, + options: OllamaOptions { temperature: 0.7, num_predict: 1024 }, }; - let client = reqwest::Client::new(); - let resp = client + let resp = state.http_client .post(format!("{}/api/chat", state.ollama_url)) .json(&ollama_req) .send() .await .map_err(|e| { error!("Ollama request failed: {}", e); - ( - axum::http::StatusCode::SERVICE_UNAVAILABLE, - "AI service unavailable".to_string(), - ) + (StatusCode::SERVICE_UNAVAILABLE, "AI service unavailable".to_string()) })?; let ollama_resp: OllamaResponse = resp.json().await.map_err(|e| { error!("Failed to parse Ollama response: {}", e); - ( - axum::http::StatusCode::SERVICE_UNAVAILABLE, - "AI response parse error".to_string(), - ) + (StatusCode::SERVICE_UNAVAILABLE, "AI response parse error".to_string()) })?; - let response_text = ollama_resp - .message - .map(|m| m.content) - .unwrap_or_default(); + let response_text = ollama_resp.message.map(|m| m.content).unwrap_or_default(); - // Store in database - let user_id = Uuid::parse_str(&req.user_id).unwrap_or(Uuid::nil()); let _ = sqlx::query( "INSERT INTO ai_conversations (id, user_id, role, message, response) VALUES ($1, $2, $3, $4, $5)", ) .bind(Uuid::new_v4()) - .bind(user_id) - .bind(&req.user_role) + .bind(Uuid::parse_str(&claims.user_id).unwrap_or(Uuid::nil())) + .bind(&claims.role) .bind(&message_text) .bind(&response_text) .execute(&state.db) .await; - Ok(Json(ChatResponse { - response: response_text, - })) + Ok(Json(ChatResponse { response: response_text })) } // ── Jobs handler ─────────────────────────────────────────────── async fn handle_jobs( State(state): State>, -) -> Json { - Json(JobsResponse { - jobs: state.jobs.clone(), - }) + headers: HeaderMap, +) -> Result, (StatusCode, String)> { + let _claims = extract_claims(&headers, &state)?; + if !state.rate_limiter.check(&_claims.user_id).await { + return Err((StatusCode::TOO_MANY_REQUESTS, "Rate limit exceeded".to_string())); + } + Ok(Json(JobsResponse { jobs: state.jobs.clone() })) } // ── Health handler ───────────────────────────────────────────── @@ -296,52 +389,50 @@ async fn main() { dotenvy::dotenv().ok(); - let database_url = - std::env::var("DATABASE_URL").expect("DATABASE_URL must be set"); - let ollama_url = - std::env::var("OLLAMA_BASE_URL").unwrap_or_else(|_| "http://localhost:11434".to_string()); - let model = std::env::var("AI_MODEL").unwrap_or_else(|_| "sam860/dolphin3-llama3.2:3b".to_string()); - let host = std::env::var("AI_HOST").unwrap_or_else(|_| "0.0.0.0".to_string()); - let port: u16 = std::env::var("AI_PORT") - .unwrap_or_else(|_| "3001".to_string()) - .parse() - .expect("AI_PORT must be a number"); + let database_url = std::env::var("DATABASE_URL").expect("DATABASE_URL must be set"); + let jwt_secret = std::env::var("JWT_SECRET").expect("JWT_SECRET must be set"); + let ollama_url = std::env::var("OLLAMA_BASE_URL").unwrap_or_else(|_| "http://localhost:11434".to_string()); + let model = std::env::var("AI_MODEL").unwrap_or_else(|_| "dolphin-phi".to_string()); + let host = std::env::var("AI_HOST").unwrap_or_else(|_| "127.0.0.1".to_string()); + let port: u16 = std::env::var("AI_PORT").unwrap_or_else(|_| "3001".to_string()).parse().expect("AI_PORT must be a number"); - // Load jobs let jobs_path = std::env::var("JOBS_PATH").unwrap_or_else(|_| "data/ai/jobs.jsonl".to_string()); let jobs_content = fs::read_to_string(&jobs_path).unwrap_or_default(); - let jobs: Vec = jobs_content - .lines() - .filter(|l| !l.trim().is_empty()) - .filter_map(|l| serde_json::from_str(l).ok()) - .collect(); + let jobs: Vec = jobs_content.lines().filter(|l| !l.trim().is_empty()).filter_map(|l| serde_json::from_str(l).ok()).collect(); - info!( - "Loaded {} job categories, model: {}, Ollama: {}", - jobs.len(), - model, - ollama_url - ); + info!("Loaded {} job categories, model: {}, Ollama: {}", jobs.len(), model, ollama_url); - // Connect to database let db = PgPoolOptions::new() - .max_connections(5) + .max_connections(20) .connect(&database_url) .await .expect("Failed to connect to database"); info!("Connected to PostgreSQL"); + let http_client = reqwest::Client::builder() + .timeout(Duration::from_secs(120)) + .build() + .expect("Failed to build HTTP client"); + + let lead_store = Arc::new(Mutex::new(LeadStore::new(100))); + let state = Arc::new(AppState { db, ollama_url, model, jobs, + leads: lead_store.clone(), + http_client, + jwt_secret, + rate_limiter: RateLimiter::new(30, 60), }); - // CORS layer let cors = CorsLayer::new() - .allow_origin(Any) + .allow_origin(AllowOrigin::list([ + "http://localhost:3006".parse().unwrap(), + "http://127.0.0.1:3006".parse().unwrap(), + ])) .allow_methods([Method::GET, Method::POST]) .allow_headers(Any); @@ -359,24 +450,57 @@ async fn main() { .await .expect("Failed to bind address"); - // Start Facebook scraper in a separate thread - let rng = Arc::new(Mutex::new(StdRng::from_entropy())); - tokio::task::spawn_blocking(move || { - loop { - { - let _lock = rng.lock().unwrap(); - run_facebook_scraper(); + let bg_leads = lead_store.clone(); + let bg_url = "http://localhost:3008/scrape/all".to_string(); + tokio::spawn(async move { + let client = match reqwest::Client::builder() + .timeout(Duration::from_secs(120)) + .build() + { + Ok(c) => c, + Err(e) => { + error!("Failed to build background HTTP client: {} — scraper disabled", e); + return; } - // Introduce random delay - let delay = { - let mut rng = rng.lock().unwrap(); - rng.gen_range(1..5) - }; - thread::sleep(Duration::from_secs(delay)); + }; + loop { + let now = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs(); + match client.post(&bg_url).send().await { + Ok(resp) => { + if resp.status().is_success() { + match resp.json::>().await { + Ok(data) => { + let mut store = bg_leads.lock().await; + for item in &data { + store.push(Lead { + title: truncate(item["title"].as_str().unwrap_or(""), 120), + url: item["url"].as_str().unwrap_or("").to_string(), + source: item["source"].as_str().unwrap_or("unknown").to_string(), + found_at: now, + author: truncate(item["author"].as_str().unwrap_or(""), 60), + date: truncate(item["date"].as_str().unwrap_or(""), 30), + content: truncate(item["content"].as_str().unwrap_or(""), 300), + }); + } + } + Err(e) => { + warn!("Failed to parse scraper JSON: {}", e); + } + } + } else { + warn!("Scraper returned status: {}", resp.status()); + } + } + Err(e) => { + warn!("Scraper request failed: {}", e); + } + } + let delay = rand::thread_rng().gen_range(120..300); + tokio::time::sleep(Duration::from_secs(delay)).await; } }); axum::serve(listener, app) .await .expect("Server failed"); -} \ No newline at end of file +} diff --git a/scraper-service/index.js b/scraper-service/index.js new file mode 100644 index 0000000..46565c9 --- /dev/null +++ b/scraper-service/index.js @@ -0,0 +1,100 @@ +const express = require('express'); +const { chromium } = require('playwright'); + +const app = express(); +app.use(express.json()); +const PORT = process.env.PORT || 3007; + +app.get('/health', (req, res) => { + res.json({ status: 'ok' }); +}); + +app.post('/scrape/indeed', async (req, res) => { + const results = []; + let browser; + try { + browser = await chromium.launch({ headless: true }); + const context = await browser.newContext({ + userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', + }); + + const queries = [ + 'need a website built', + 'looking for someone to build my website', + 'need web designer', + 'looking for web developer', + 'need help with my website', + 'need ecommerce website', + 'need wordpress website', + 'looking for website designer', + ]; + + for (const q of queries) { + const url = `https://www.indeed.com/jobs?q=${encodeURIComponent(q)}&sort=date`; + try { + const page = await context.newPage(); + await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 20000 }); + await page.waitForTimeout(2000); + + const items = await page.evaluate(() => { + const cards = document.querySelectorAll('.job_seen_beacon'); + return Array.from(cards).slice(0, 8).map(card => { + const titleEl = card.querySelector('.jcs-JobTitle'); + const companyEl = card.querySelector('[data-testid="inlineHeader-companyName"]'); + const snippetEl = card.querySelector('.job-snippet'); + return { + title: titleEl?.textContent?.trim() || '', + url: titleEl?.href || '', + company: companyEl?.textContent?.trim() || '', + snippet: snippetEl?.textContent?.trim()?.slice(0, 200) || '', + }; + }); + }); + + for (const item of items) { + if (item.title && item.url && !results.some(r => r.url === item.url)) { + results.push({ + title: item.title, + url: item.url, + author: item.company, + date: '', + content: item.snippet, + source: 'indeed', + }); + } + } + await page.close(); + } catch (e) { + console.error(`Indeed failed ${q}:`, e.message); + } + } + + await browser.close(); + res.json(results.slice(0, 50)); + } catch (e) { + if (browser) await browser.close().catch(() => {}); + console.error('Indeed error:', e.message); + res.status(500).json({ error: e.message }); + } +}); + +app.post('/scrape/all', async (req, res) => { + try { + const [indeed] = await Promise.allSettled([ + fetch(`http://localhost:${PORT}/scrape/indeed`, { method: 'POST' }).then(r => r.json()).catch(() => []), + ]); + + const all = [ + ...(indeed.status === 'fulfilled' ? indeed.value : []), + ]; + console.log(`Scrape all: ${all.length} leads`); + res.json(all); + } catch (e) { + console.error('Scrape all error:', e.message); + res.status(500).json({ error: e.message }); + } +}); + +app.listen(PORT, () => { + console.log(`Scraper service running on port ${PORT}`); +}); diff --git a/scraper-service/node_modules/.bin/mime b/scraper-service/node_modules/.bin/mime new file mode 100644 index 0000000..7751de3 --- /dev/null +++ b/scraper-service/node_modules/.bin/mime @@ -0,0 +1,16 @@ +#!/bin/sh +basedir=$(dirname "$(echo "$0" | sed -e 's,\\,/,g')") + +case `uname` in + *CYGWIN*|*MINGW*|*MSYS*) + if command -v cygpath > /dev/null 2>&1; then + basedir=`cygpath -w "$basedir"` + fi + ;; +esac + +if [ -x "$basedir/node" ]; then + exec "$basedir/node" "$basedir/../mime/cli.js" "$@" +else + exec node "$basedir/../mime/cli.js" "$@" +fi diff --git a/scraper-service/node_modules/.bin/mime.cmd b/scraper-service/node_modules/.bin/mime.cmd new file mode 100644 index 0000000..54491f1 --- /dev/null +++ b/scraper-service/node_modules/.bin/mime.cmd @@ -0,0 +1,17 @@ +@ECHO off +GOTO start +:find_dp0 +SET dp0=%~dp0 +EXIT /b +:start +SETLOCAL +CALL :find_dp0 + +IF EXIST "%dp0%\node.exe" ( + SET "_prog=%dp0%\node.exe" +) ELSE ( + SET "_prog=node" + SET PATHEXT=%PATHEXT:;.JS;=;% +) + +endLocal & goto #_undefined_# 2>NUL || title %COMSPEC% & "%_prog%" "%dp0%\..\mime\cli.js" %* diff --git a/scraper-service/node_modules/.bin/mime.ps1 b/scraper-service/node_modules/.bin/mime.ps1 new file mode 100644 index 0000000..2222f40 --- /dev/null +++ b/scraper-service/node_modules/.bin/mime.ps1 @@ -0,0 +1,28 @@ +#!/usr/bin/env pwsh +$basedir=Split-Path $MyInvocation.MyCommand.Definition -Parent + +$exe="" +if ($PSVersionTable.PSVersion -lt "6.0" -or $IsWindows) { + # Fix case when both the Windows and Linux builds of Node + # are installed in the same directory + $exe=".exe" +} +$ret=0 +if (Test-Path "$basedir/node$exe") { + # Support pipeline input + if ($MyInvocation.ExpectingInput) { + $input | & "$basedir/node$exe" "$basedir/../mime/cli.js" $args + } else { + & "$basedir/node$exe" "$basedir/../mime/cli.js" $args + } + $ret=$LASTEXITCODE +} else { + # Support pipeline input + if ($MyInvocation.ExpectingInput) { + $input | & "node$exe" "$basedir/../mime/cli.js" $args + } else { + & "node$exe" "$basedir/../mime/cli.js" $args + } + $ret=$LASTEXITCODE +} +exit $ret diff --git a/scraper-service/node_modules/.bin/playwright b/scraper-service/node_modules/.bin/playwright new file mode 100644 index 0000000..98e2503 --- /dev/null +++ b/scraper-service/node_modules/.bin/playwright @@ -0,0 +1,16 @@ +#!/bin/sh +basedir=$(dirname "$(echo "$0" | sed -e 's,\\,/,g')") + +case `uname` in + *CYGWIN*|*MINGW*|*MSYS*) + if command -v cygpath > /dev/null 2>&1; then + basedir=`cygpath -w "$basedir"` + fi + ;; +esac + +if [ -x "$basedir/node" ]; then + exec "$basedir/node" "$basedir/../playwright/cli.js" "$@" +else + exec node "$basedir/../playwright/cli.js" "$@" +fi diff --git a/scraper-service/node_modules/.bin/playwright-core b/scraper-service/node_modules/.bin/playwright-core new file mode 100644 index 0000000..bc2c5c8 --- /dev/null +++ b/scraper-service/node_modules/.bin/playwright-core @@ -0,0 +1,16 @@ +#!/bin/sh +basedir=$(dirname "$(echo "$0" | sed -e 's,\\,/,g')") + +case `uname` in + *CYGWIN*|*MINGW*|*MSYS*) + if command -v cygpath > /dev/null 2>&1; then + basedir=`cygpath -w "$basedir"` + fi + ;; +esac + +if [ -x "$basedir/node" ]; then + exec "$basedir/node" "$basedir/../playwright-core/cli.js" "$@" +else + exec node "$basedir/../playwright-core/cli.js" "$@" +fi diff --git a/scraper-service/node_modules/.bin/playwright-core.cmd b/scraper-service/node_modules/.bin/playwright-core.cmd new file mode 100644 index 0000000..1128204 --- /dev/null +++ b/scraper-service/node_modules/.bin/playwright-core.cmd @@ -0,0 +1,17 @@ +@ECHO off +GOTO start +:find_dp0 +SET dp0=%~dp0 +EXIT /b +:start +SETLOCAL +CALL :find_dp0 + +IF EXIST "%dp0%\node.exe" ( + SET "_prog=%dp0%\node.exe" +) ELSE ( + SET "_prog=node" + SET PATHEXT=%PATHEXT:;.JS;=;% +) + +endLocal & goto #_undefined_# 2>NUL || title %COMSPEC% & "%_prog%" "%dp0%\..\playwright-core\cli.js" %* diff --git a/scraper-service/node_modules/.bin/playwright-core.ps1 b/scraper-service/node_modules/.bin/playwright-core.ps1 new file mode 100644 index 0000000..e914b99 --- /dev/null +++ b/scraper-service/node_modules/.bin/playwright-core.ps1 @@ -0,0 +1,28 @@ +#!/usr/bin/env pwsh +$basedir=Split-Path $MyInvocation.MyCommand.Definition -Parent + +$exe="" +if ($PSVersionTable.PSVersion -lt "6.0" -or $IsWindows) { + # Fix case when both the Windows and Linux builds of Node + # are installed in the same directory + $exe=".exe" +} +$ret=0 +if (Test-Path "$basedir/node$exe") { + # Support pipeline input + if ($MyInvocation.ExpectingInput) { + $input | & "$basedir/node$exe" "$basedir/../playwright-core/cli.js" $args + } else { + & "$basedir/node$exe" "$basedir/../playwright-core/cli.js" $args + } + $ret=$LASTEXITCODE +} else { + # Support pipeline input + if ($MyInvocation.ExpectingInput) { + $input | & "node$exe" "$basedir/../playwright-core/cli.js" $args + } else { + & "node$exe" "$basedir/../playwright-core/cli.js" $args + } + $ret=$LASTEXITCODE +} +exit $ret diff --git a/scraper-service/node_modules/.bin/playwright.cmd b/scraper-service/node_modules/.bin/playwright.cmd new file mode 100644 index 0000000..88713a4 --- /dev/null +++ b/scraper-service/node_modules/.bin/playwright.cmd @@ -0,0 +1,17 @@ +@ECHO off +GOTO start +:find_dp0 +SET dp0=%~dp0 +EXIT /b +:start +SETLOCAL +CALL :find_dp0 + +IF EXIST "%dp0%\node.exe" ( + SET "_prog=%dp0%\node.exe" +) ELSE ( + SET "_prog=node" + SET PATHEXT=%PATHEXT:;.JS;=;% +) + +endLocal & goto #_undefined_# 2>NUL || title %COMSPEC% & "%_prog%" "%dp0%\..\playwright\cli.js" %* diff --git a/scraper-service/node_modules/.bin/playwright.ps1 b/scraper-service/node_modules/.bin/playwright.ps1 new file mode 100644 index 0000000..efa8f92 --- /dev/null +++ b/scraper-service/node_modules/.bin/playwright.ps1 @@ -0,0 +1,28 @@ +#!/usr/bin/env pwsh +$basedir=Split-Path $MyInvocation.MyCommand.Definition -Parent + +$exe="" +if ($PSVersionTable.PSVersion -lt "6.0" -or $IsWindows) { + # Fix case when both the Windows and Linux builds of Node + # are installed in the same directory + $exe=".exe" +} +$ret=0 +if (Test-Path "$basedir/node$exe") { + # Support pipeline input + if ($MyInvocation.ExpectingInput) { + $input | & "$basedir/node$exe" "$basedir/../playwright/cli.js" $args + } else { + & "$basedir/node$exe" "$basedir/../playwright/cli.js" $args + } + $ret=$LASTEXITCODE +} else { + # Support pipeline input + if ($MyInvocation.ExpectingInput) { + $input | & "node$exe" "$basedir/../playwright/cli.js" $args + } else { + & "node$exe" "$basedir/../playwright/cli.js" $args + } + $ret=$LASTEXITCODE +} +exit $ret diff --git a/scraper-service/node_modules/.package-lock.json b/scraper-service/node_modules/.package-lock.json new file mode 100644 index 0000000..7873a80 --- /dev/null +++ b/scraper-service/node_modules/.package-lock.json @@ -0,0 +1,850 @@ +{ + "name": "scraper-service", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "node_modules/accepts": { + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz", + "integrity": "sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==", + "license": "MIT", + "dependencies": { + "mime-types": "~2.1.34", + "negotiator": "0.6.3" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/array-flatten": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", + "integrity": "sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg==", + "license": "MIT" + }, + "node_modules/body-parser": { + "version": "1.20.5", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.5.tgz", + "integrity": "sha512-3grm+/2tUOvu2cjJkvsIxrv/wVpfXQW4PsQHYm7yk4vfpu7Ekl6nEsYBoJUL6qDwZUx8wUhQ8tR2qz+ad9c9OA==", + "license": "MIT", + "dependencies": { + "bytes": "~3.1.2", + "content-type": "~1.0.5", + "debug": "2.6.9", + "depd": "2.0.0", + "destroy": "~1.2.0", + "http-errors": "~2.0.1", + "iconv-lite": "~0.4.24", + "on-finished": "~2.4.1", + "qs": "~6.15.1", + "raw-body": "~2.5.3", + "type-is": "~1.6.18", + "unpipe": "~1.0.0" + }, + "engines": { + "node": ">= 0.8", + "npm": "1.2.8000 || >= 1.4.16" + } + }, + "node_modules/bytes": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", + "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/call-bind-apply-helpers": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", + "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/call-bound": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/call-bound/-/call-bound-1.0.4.tgz", + "integrity": "sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "get-intrinsic": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/content-disposition": { + "version": "0.5.4", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz", + "integrity": "sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==", + "license": "MIT", + "dependencies": { + "safe-buffer": "5.2.1" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/content-type": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.5.tgz", + "integrity": "sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/cookie": { + "version": "0.7.2", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.7.2.tgz", + "integrity": "sha512-yki5XnKuf750l50uGTllt6kKILY4nQ1eNIQatoXEByZ5dWgnKqbnqmTrBE5B4N7lrMJKQ2ytWMiTO2o0v6Ew/w==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/cookie-signature": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.7.tgz", + "integrity": "sha512-NXdYc3dLr47pBkpUCHtKSwIOQXLVn8dZEuywboCOJY/osA0wFSLlSawr3KN8qXJEyX66FcONTH8EIlVuK0yyFA==", + "license": "MIT" + }, + "node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "license": "MIT", + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/depd": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", + "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/destroy": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.2.0.tgz", + "integrity": "sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg==", + "license": "MIT", + "engines": { + "node": ">= 0.8", + "npm": "1.2.8000 || >= 1.4.16" + } + }, + "node_modules/dunder-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", + "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.1", + "es-errors": "^1.3.0", + "gopd": "^1.2.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/ee-first": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", + "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==", + "license": "MIT" + }, + "node_modules/encodeurl": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-2.0.0.tgz", + "integrity": "sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/es-define-property": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", + "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-errors": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", + "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-object-atoms": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.2.tgz", + "integrity": "sha512-HWcBoN6NileqtSydK2FqHbS/LoDd2pqrnQHLyJzBj4kOp/ky2MWMN694xOfkK8/SnUsW2DH7EfyVlydKCsm1Zw==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/escape-html": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", + "integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==", + "license": "MIT" + }, + "node_modules/etag": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", + "integrity": "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/express": { + "version": "4.22.2", + "resolved": "https://registry.npmjs.org/express/-/express-4.22.2.tgz", + "integrity": "sha512-IuL+Elrou2ZvCFHs18/CIzy2Nzvo25nZ1/D2eIZlz7c+QUayAcYoiM2BthCjs+EBHVpjYjcuLDAiCWgeIX3X1Q==", + "license": "MIT", + "dependencies": { + "accepts": "~1.3.8", + "array-flatten": "1.1.1", + "body-parser": "~1.20.5", + "content-disposition": "~0.5.4", + "content-type": "~1.0.4", + "cookie": "~0.7.1", + "cookie-signature": "~1.0.6", + "debug": "2.6.9", + "depd": "2.0.0", + "encodeurl": "~2.0.0", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "finalhandler": "~1.3.1", + "fresh": "~0.5.2", + "http-errors": "~2.0.0", + "merge-descriptors": "1.0.3", + "methods": "~1.1.2", + "on-finished": "~2.4.1", + "parseurl": "~1.3.3", + "path-to-regexp": "~0.1.12", + "proxy-addr": "~2.0.7", + "qs": "~6.15.1", + "range-parser": "~1.2.1", + "safe-buffer": "5.2.1", + "send": "~0.19.0", + "serve-static": "~1.16.2", + "setprototypeof": "1.2.0", + "statuses": "~2.0.1", + "type-is": "~1.6.18", + "utils-merge": "1.0.1", + "vary": "~1.1.2" + }, + "engines": { + "node": ">= 0.10.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/finalhandler": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.3.2.tgz", + "integrity": "sha512-aA4RyPcd3badbdABGDuTXCMTtOneUCAYH/gxoYRTZlIJdF0YPWuGqiAsIrhNnnqdXGswYk6dGujem4w80UJFhg==", + "license": "MIT", + "dependencies": { + "debug": "2.6.9", + "encodeurl": "~2.0.0", + "escape-html": "~1.0.3", + "on-finished": "~2.4.1", + "parseurl": "~1.3.3", + "statuses": "~2.0.2", + "unpipe": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/forwarded": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", + "integrity": "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/fresh": { + "version": "0.5.2", + "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", + "integrity": "sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/function-bind": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-intrinsic": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", + "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "es-define-property": "^1.0.1", + "es-errors": "^1.3.0", + "es-object-atoms": "^1.1.1", + "function-bind": "^1.1.2", + "get-proto": "^1.0.1", + "gopd": "^1.2.0", + "has-symbols": "^1.1.0", + "hasown": "^2.0.2", + "math-intrinsics": "^1.1.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", + "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", + "license": "MIT", + "dependencies": { + "dunder-proto": "^1.0.1", + "es-object-atoms": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/gopd": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", + "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-symbols": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", + "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/hasown": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.4.tgz", + "integrity": "sha512-T2UbfbBEF32wiepXIsMlTW9+dDYC6wMh/t/vYA4tuOMKqWz/n3vr1NFSxQiyP+zk2mXsoMA/i/7qV6LKut1t1A==", + "license": "MIT", + "dependencies": { + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/http-errors": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz", + "integrity": "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==", + "license": "MIT", + "dependencies": { + "depd": "~2.0.0", + "inherits": "~2.0.4", + "setprototypeof": "~1.2.0", + "statuses": "~2.0.2", + "toidentifier": "~1.0.1" + }, + "engines": { + "node": ">= 0.8" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/iconv-lite": { + "version": "0.4.24", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", + "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "license": "ISC" + }, + "node_modules/ipaddr.js": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", + "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==", + "license": "MIT", + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/math-intrinsics": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", + "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/media-typer": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", + "integrity": "sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/merge-descriptors": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.3.tgz", + "integrity": "sha512-gaNvAS7TZ897/rVaZ0nMtAyxNyi/pdbjbAwUpFQpN70GqnVfOiXpeUUMKRBmzXaSQ8DdTX4/0ms62r2K+hE6mQ==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/methods": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz", + "integrity": "sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz", + "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==", + "license": "MIT", + "bin": { + "mime": "cli.js" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/mime-db": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types": { + "version": "2.1.35", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "license": "MIT", + "dependencies": { + "mime-db": "1.52.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", + "license": "MIT" + }, + "node_modules/negotiator": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", + "integrity": "sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/object-inspect": { + "version": "1.13.4", + "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.4.tgz", + "integrity": "sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/on-finished": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", + "integrity": "sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==", + "license": "MIT", + "dependencies": { + "ee-first": "1.1.1" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/parseurl": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", + "integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/path-to-regexp": { + "version": "0.1.13", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.13.tgz", + "integrity": "sha512-A/AGNMFN3c8bOlvV9RreMdrv7jsmF9XIfDeCd87+I8RNg6s78BhJxMu69NEMHBSJFxKidViTEdruRwEk/WIKqA==", + "license": "MIT" + }, + "node_modules/playwright": { + "version": "1.61.0", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.61.0.tgz", + "integrity": "sha512-Z+7BeeqQPRRzklHsVFP4KTGIyMxKUmfeRA4WisM6G3/XW6nwGeX6fX9qYaDa+CiUqpOkb2f6X3nar05R3kSuJQ==", + "license": "Apache-2.0", + "dependencies": { + "playwright-core": "1.61.0" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "fsevents": "2.3.2" + } + }, + "node_modules/playwright-core": { + "version": "1.61.0", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.61.0.tgz", + "integrity": "sha512-caX7TrY3Ml6egyDX0WUcTHDxodl/b51y5wJOdCEA36QviK/s2g081hvmGs8eaE3DWb6NYZQ6BjO/QkNRPenoPA==", + "license": "Apache-2.0", + "bin": { + "playwright-core": "cli.js" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/proxy-addr": { + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", + "integrity": "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==", + "license": "MIT", + "dependencies": { + "forwarded": "0.2.0", + "ipaddr.js": "1.9.1" + }, + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/qs": { + "version": "6.15.2", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.15.2.tgz", + "integrity": "sha512-Rzq0KEyX/w/tEybncDgdkZrJgVUsUMk3xjh3t5bv3S1HTAtg+uOYt72+ZfwiQwKdysThkTBdL/rTi6HDmX9Ddw==", + "license": "BSD-3-Clause", + "dependencies": { + "side-channel": "^1.1.0" + }, + "engines": { + "node": ">=0.6" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/range-parser": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", + "integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/raw-body": { + "version": "2.5.3", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.3.tgz", + "integrity": "sha512-s4VSOf6yN0rvbRZGxs8Om5CWj6seneMwK3oDb4lWDH0UPhWcxwOWw5+qk24bxq87szX1ydrwylIOp2uG1ojUpA==", + "license": "MIT", + "dependencies": { + "bytes": "~3.1.2", + "http-errors": "~2.0.1", + "iconv-lite": "~0.4.24", + "unpipe": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", + "license": "MIT" + }, + "node_modules/send": { + "version": "0.19.2", + "resolved": "https://registry.npmjs.org/send/-/send-0.19.2.tgz", + "integrity": "sha512-VMbMxbDeehAxpOtWJXlcUS5E8iXh6QmN+BkRX1GARS3wRaXEEgzCcB10gTQazO42tpNIya8xIyNx8fll1OFPrg==", + "license": "MIT", + "dependencies": { + "debug": "2.6.9", + "depd": "2.0.0", + "destroy": "1.2.0", + "encodeurl": "~2.0.0", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "fresh": "~0.5.2", + "http-errors": "~2.0.1", + "mime": "1.6.0", + "ms": "2.1.3", + "on-finished": "~2.4.1", + "range-parser": "~1.2.1", + "statuses": "~2.0.2" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/send/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, + "node_modules/serve-static": { + "version": "1.16.3", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.16.3.tgz", + "integrity": "sha512-x0RTqQel6g5SY7Lg6ZreMmsOzncHFU7nhnRWkKgWuMTu5NN0DR5oruckMqRvacAN9d5w6ARnRBXl9xhDCgfMeA==", + "license": "MIT", + "dependencies": { + "encodeurl": "~2.0.0", + "escape-html": "~1.0.3", + "parseurl": "~1.3.3", + "send": "~0.19.1" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/setprototypeof": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", + "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==", + "license": "ISC" + }, + "node_modules/side-channel": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.1.1.tgz", + "integrity": "sha512-6x6dK6zJdpTzF4sQeNYxwtvBzf6Eg4GtlesS94HOvTudUeyK2WXAaIfmDgsyslYrRBeFIlsi54AYsFGUuhmvrQ==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "object-inspect": "^1.13.4", + "side-channel-list": "^1.0.1", + "side-channel-map": "^1.0.1", + "side-channel-weakmap": "^1.0.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-list": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/side-channel-list/-/side-channel-list-1.0.1.tgz", + "integrity": "sha512-mjn/0bi/oUURjc5Xl7IaWi/OJJJumuoJFQJfDDyO46+hBWsfaVM65TBHq2eoZBhzl9EchxOijpkbRC8SVBQU0w==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "object-inspect": "^1.13.4" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-map": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/side-channel-map/-/side-channel-map-1.0.1.tgz", + "integrity": "sha512-VCjCNfgMsby3tTdo02nbjtM/ewra6jPHmpThenkTYh8pG9ucZ/1P8So4u4FGBek/BjpOVsDCMoLA/iuBKIFXRA==", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.5", + "object-inspect": "^1.13.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-weakmap": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/side-channel-weakmap/-/side-channel-weakmap-1.0.2.tgz", + "integrity": "sha512-WPS/HvHQTYnHisLo9McqBHOJk2FkHO/tlpvldyrnem4aeQp4hai3gythswg6p01oSoTl58rcpiFAjF2br2Ak2A==", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.5", + "object-inspect": "^1.13.3", + "side-channel-map": "^1.0.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/statuses": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", + "integrity": "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/toidentifier": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", + "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==", + "license": "MIT", + "engines": { + "node": ">=0.6" + } + }, + "node_modules/type-is": { + "version": "1.6.18", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz", + "integrity": "sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==", + "license": "MIT", + "dependencies": { + "media-typer": "0.3.0", + "mime-types": "~2.1.24" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/unpipe": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", + "integrity": "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/utils-merge": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", + "integrity": "sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA==", + "license": "MIT", + "engines": { + "node": ">= 0.4.0" + } + }, + "node_modules/vary": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", + "integrity": "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + } + } +} diff --git a/scraper-service/node_modules/accepts/HISTORY.md b/scraper-service/node_modules/accepts/HISTORY.md new file mode 100644 index 0000000..cb5990c --- /dev/null +++ b/scraper-service/node_modules/accepts/HISTORY.md @@ -0,0 +1,243 @@ +1.3.8 / 2022-02-02 +================== + + * deps: mime-types@~2.1.34 + - deps: mime-db@~1.51.0 + * deps: negotiator@0.6.3 + +1.3.7 / 2019-04-29 +================== + + * deps: negotiator@0.6.2 + - Fix sorting charset, encoding, and language with extra parameters + +1.3.6 / 2019-04-28 +================== + + * deps: mime-types@~2.1.24 + - deps: mime-db@~1.40.0 + +1.3.5 / 2018-02-28 +================== + + * deps: mime-types@~2.1.18 + - deps: mime-db@~1.33.0 + +1.3.4 / 2017-08-22 +================== + + * deps: mime-types@~2.1.16 + - deps: mime-db@~1.29.0 + +1.3.3 / 2016-05-02 +================== + + * deps: mime-types@~2.1.11 + - deps: mime-db@~1.23.0 + * deps: negotiator@0.6.1 + - perf: improve `Accept` parsing speed + - perf: improve `Accept-Charset` parsing speed + - perf: improve `Accept-Encoding` parsing speed + - perf: improve `Accept-Language` parsing speed + +1.3.2 / 2016-03-08 +================== + + * deps: mime-types@~2.1.10 + - Fix extension of `application/dash+xml` + - Update primary extension for `audio/mp4` + - deps: mime-db@~1.22.0 + +1.3.1 / 2016-01-19 +================== + + * deps: mime-types@~2.1.9 + - deps: mime-db@~1.21.0 + +1.3.0 / 2015-09-29 +================== + + * deps: mime-types@~2.1.7 + - deps: mime-db@~1.19.0 + * deps: negotiator@0.6.0 + - Fix including type extensions in parameters in `Accept` parsing + - Fix parsing `Accept` parameters with quoted equals + - Fix parsing `Accept` parameters with quoted semicolons + - Lazy-load modules from main entry point + - perf: delay type concatenation until needed + - perf: enable strict mode + - perf: hoist regular expressions + - perf: remove closures getting spec properties + - perf: remove a closure from media type parsing + - perf: remove property delete from media type parsing + +1.2.13 / 2015-09-06 +=================== + + * deps: mime-types@~2.1.6 + - deps: mime-db@~1.18.0 + +1.2.12 / 2015-07-30 +=================== + + * deps: mime-types@~2.1.4 + - deps: mime-db@~1.16.0 + +1.2.11 / 2015-07-16 +=================== + + * deps: mime-types@~2.1.3 + - deps: mime-db@~1.15.0 + +1.2.10 / 2015-07-01 +=================== + + * deps: mime-types@~2.1.2 + - deps: mime-db@~1.14.0 + +1.2.9 / 2015-06-08 +================== + + * deps: mime-types@~2.1.1 + - perf: fix deopt during mapping + +1.2.8 / 2015-06-07 +================== + + * deps: mime-types@~2.1.0 + - deps: mime-db@~1.13.0 + * perf: avoid argument reassignment & argument slice + * perf: avoid negotiator recursive construction + * perf: enable strict mode + * perf: remove unnecessary bitwise operator + +1.2.7 / 2015-05-10 +================== + + * deps: negotiator@0.5.3 + - Fix media type parameter matching to be case-insensitive + +1.2.6 / 2015-05-07 +================== + + * deps: mime-types@~2.0.11 + - deps: mime-db@~1.9.1 + * deps: negotiator@0.5.2 + - Fix comparing media types with quoted values + - Fix splitting media types with quoted commas + +1.2.5 / 2015-03-13 +================== + + * deps: mime-types@~2.0.10 + - deps: mime-db@~1.8.0 + +1.2.4 / 2015-02-14 +================== + + * Support Node.js 0.6 + * deps: mime-types@~2.0.9 + - deps: mime-db@~1.7.0 + * deps: negotiator@0.5.1 + - Fix preference sorting to be stable for long acceptable lists + +1.2.3 / 2015-01-31 +================== + + * deps: mime-types@~2.0.8 + - deps: mime-db@~1.6.0 + +1.2.2 / 2014-12-30 +================== + + * deps: mime-types@~2.0.7 + - deps: mime-db@~1.5.0 + +1.2.1 / 2014-12-30 +================== + + * deps: mime-types@~2.0.5 + - deps: mime-db@~1.3.1 + +1.2.0 / 2014-12-19 +================== + + * deps: negotiator@0.5.0 + - Fix list return order when large accepted list + - Fix missing identity encoding when q=0 exists + - Remove dynamic building of Negotiator class + +1.1.4 / 2014-12-10 +================== + + * deps: mime-types@~2.0.4 + - deps: mime-db@~1.3.0 + +1.1.3 / 2014-11-09 +================== + + * deps: mime-types@~2.0.3 + - deps: mime-db@~1.2.0 + +1.1.2 / 2014-10-14 +================== + + * deps: negotiator@0.4.9 + - Fix error when media type has invalid parameter + +1.1.1 / 2014-09-28 +================== + + * deps: mime-types@~2.0.2 + - deps: mime-db@~1.1.0 + * deps: negotiator@0.4.8 + - Fix all negotiations to be case-insensitive + - Stable sort preferences of same quality according to client order + +1.1.0 / 2014-09-02 +================== + + * update `mime-types` + +1.0.7 / 2014-07-04 +================== + + * Fix wrong type returned from `type` when match after unknown extension + +1.0.6 / 2014-06-24 +================== + + * deps: negotiator@0.4.7 + +1.0.5 / 2014-06-20 +================== + + * fix crash when unknown extension given + +1.0.4 / 2014-06-19 +================== + + * use `mime-types` + +1.0.3 / 2014-06-11 +================== + + * deps: negotiator@0.4.6 + - Order by specificity when quality is the same + +1.0.2 / 2014-05-29 +================== + + * Fix interpretation when header not in request + * deps: pin negotiator@0.4.5 + +1.0.1 / 2014-01-18 +================== + + * Identity encoding isn't always acceptable + * deps: negotiator@~0.4.0 + +1.0.0 / 2013-12-27 +================== + + * Genesis diff --git a/scraper-service/node_modules/accepts/LICENSE b/scraper-service/node_modules/accepts/LICENSE new file mode 100644 index 0000000..0616607 --- /dev/null +++ b/scraper-service/node_modules/accepts/LICENSE @@ -0,0 +1,23 @@ +(The MIT License) + +Copyright (c) 2014 Jonathan Ong +Copyright (c) 2015 Douglas Christopher Wilson + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +'Software'), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/scraper-service/node_modules/accepts/README.md b/scraper-service/node_modules/accepts/README.md new file mode 100644 index 0000000..82680c5 --- /dev/null +++ b/scraper-service/node_modules/accepts/README.md @@ -0,0 +1,140 @@ +# accepts + +[![NPM Version][npm-version-image]][npm-url] +[![NPM Downloads][npm-downloads-image]][npm-url] +[![Node.js Version][node-version-image]][node-version-url] +[![Build Status][github-actions-ci-image]][github-actions-ci-url] +[![Test Coverage][coveralls-image]][coveralls-url] + +Higher level content negotiation based on [negotiator](https://www.npmjs.com/package/negotiator). +Extracted from [koa](https://www.npmjs.com/package/koa) for general use. + +In addition to negotiator, it allows: + +- Allows types as an array or arguments list, ie `(['text/html', 'application/json'])` + as well as `('text/html', 'application/json')`. +- Allows type shorthands such as `json`. +- Returns `false` when no types match +- Treats non-existent headers as `*` + +## Installation + +This is a [Node.js](https://nodejs.org/en/) module available through the +[npm registry](https://www.npmjs.com/). Installation is done using the +[`npm install` command](https://docs.npmjs.com/getting-started/installing-npm-packages-locally): + +```sh +$ npm install accepts +``` + +## API + +```js +var accepts = require('accepts') +``` + +### accepts(req) + +Create a new `Accepts` object for the given `req`. + +#### .charset(charsets) + +Return the first accepted charset. If nothing in `charsets` is accepted, +then `false` is returned. + +#### .charsets() + +Return the charsets that the request accepts, in the order of the client's +preference (most preferred first). + +#### .encoding(encodings) + +Return the first accepted encoding. If nothing in `encodings` is accepted, +then `false` is returned. + +#### .encodings() + +Return the encodings that the request accepts, in the order of the client's +preference (most preferred first). + +#### .language(languages) + +Return the first accepted language. If nothing in `languages` is accepted, +then `false` is returned. + +#### .languages() + +Return the languages that the request accepts, in the order of the client's +preference (most preferred first). + +#### .type(types) + +Return the first accepted type (and it is returned as the same text as what +appears in the `types` array). If nothing in `types` is accepted, then `false` +is returned. + +The `types` array can contain full MIME types or file extensions. Any value +that is not a full MIME types is passed to `require('mime-types').lookup`. + +#### .types() + +Return the types that the request accepts, in the order of the client's +preference (most preferred first). + +## Examples + +### Simple type negotiation + +This simple example shows how to use `accepts` to return a different typed +respond body based on what the client wants to accept. The server lists it's +preferences in order and will get back the best match between the client and +server. + +```js +var accepts = require('accepts') +var http = require('http') + +function app (req, res) { + var accept = accepts(req) + + // the order of this list is significant; should be server preferred order + switch (accept.type(['json', 'html'])) { + case 'json': + res.setHeader('Content-Type', 'application/json') + res.write('{"hello":"world!"}') + break + case 'html': + res.setHeader('Content-Type', 'text/html') + res.write('hello, world!') + break + default: + // the fallback is text/plain, so no need to specify it above + res.setHeader('Content-Type', 'text/plain') + res.write('hello, world!') + break + } + + res.end() +} + +http.createServer(app).listen(3000) +``` + +You can test this out with the cURL program: +```sh +curl -I -H'Accept: text/html' http://localhost:3000/ +``` + +## License + +[MIT](LICENSE) + +[coveralls-image]: https://badgen.net/coveralls/c/github/jshttp/accepts/master +[coveralls-url]: https://coveralls.io/r/jshttp/accepts?branch=master +[github-actions-ci-image]: https://badgen.net/github/checks/jshttp/accepts/master?label=ci +[github-actions-ci-url]: https://github.com/jshttp/accepts/actions/workflows/ci.yml +[node-version-image]: https://badgen.net/npm/node/accepts +[node-version-url]: https://nodejs.org/en/download +[npm-downloads-image]: https://badgen.net/npm/dm/accepts +[npm-url]: https://npmjs.org/package/accepts +[npm-version-image]: https://badgen.net/npm/v/accepts diff --git a/scraper-service/node_modules/accepts/index.js b/scraper-service/node_modules/accepts/index.js new file mode 100644 index 0000000..e9b2f63 --- /dev/null +++ b/scraper-service/node_modules/accepts/index.js @@ -0,0 +1,238 @@ +/*! + * accepts + * Copyright(c) 2014 Jonathan Ong + * Copyright(c) 2015 Douglas Christopher Wilson + * MIT Licensed + */ + +'use strict' + +/** + * Module dependencies. + * @private + */ + +var Negotiator = require('negotiator') +var mime = require('mime-types') + +/** + * Module exports. + * @public + */ + +module.exports = Accepts + +/** + * Create a new Accepts object for the given req. + * + * @param {object} req + * @public + */ + +function Accepts (req) { + if (!(this instanceof Accepts)) { + return new Accepts(req) + } + + this.headers = req.headers + this.negotiator = new Negotiator(req) +} + +/** + * Check if the given `type(s)` is acceptable, returning + * the best match when true, otherwise `undefined`, in which + * case you should respond with 406 "Not Acceptable". + * + * The `type` value may be a single mime type string + * such as "application/json", the extension name + * such as "json" or an array `["json", "html", "text/plain"]`. When a list + * or array is given the _best_ match, if any is returned. + * + * Examples: + * + * // Accept: text/html + * this.types('html'); + * // => "html" + * + * // Accept: text/*, application/json + * this.types('html'); + * // => "html" + * this.types('text/html'); + * // => "text/html" + * this.types('json', 'text'); + * // => "json" + * this.types('application/json'); + * // => "application/json" + * + * // Accept: text/*, application/json + * this.types('image/png'); + * this.types('png'); + * // => undefined + * + * // Accept: text/*;q=.5, application/json + * this.types(['html', 'json']); + * this.types('html', 'json'); + * // => "json" + * + * @param {String|Array} types... + * @return {String|Array|Boolean} + * @public + */ + +Accepts.prototype.type = +Accepts.prototype.types = function (types_) { + var types = types_ + + // support flattened arguments + if (types && !Array.isArray(types)) { + types = new Array(arguments.length) + for (var i = 0; i < types.length; i++) { + types[i] = arguments[i] + } + } + + // no types, return all requested types + if (!types || types.length === 0) { + return this.negotiator.mediaTypes() + } + + // no accept header, return first given type + if (!this.headers.accept) { + return types[0] + } + + var mimes = types.map(extToMime) + var accepts = this.negotiator.mediaTypes(mimes.filter(validMime)) + var first = accepts[0] + + return first + ? types[mimes.indexOf(first)] + : false +} + +/** + * Return accepted encodings or best fit based on `encodings`. + * + * Given `Accept-Encoding: gzip, deflate` + * an array sorted by quality is returned: + * + * ['gzip', 'deflate'] + * + * @param {String|Array} encodings... + * @return {String|Array} + * @public + */ + +Accepts.prototype.encoding = +Accepts.prototype.encodings = function (encodings_) { + var encodings = encodings_ + + // support flattened arguments + if (encodings && !Array.isArray(encodings)) { + encodings = new Array(arguments.length) + for (var i = 0; i < encodings.length; i++) { + encodings[i] = arguments[i] + } + } + + // no encodings, return all requested encodings + if (!encodings || encodings.length === 0) { + return this.negotiator.encodings() + } + + return this.negotiator.encodings(encodings)[0] || false +} + +/** + * Return accepted charsets or best fit based on `charsets`. + * + * Given `Accept-Charset: utf-8, iso-8859-1;q=0.2, utf-7;q=0.5` + * an array sorted by quality is returned: + * + * ['utf-8', 'utf-7', 'iso-8859-1'] + * + * @param {String|Array} charsets... + * @return {String|Array} + * @public + */ + +Accepts.prototype.charset = +Accepts.prototype.charsets = function (charsets_) { + var charsets = charsets_ + + // support flattened arguments + if (charsets && !Array.isArray(charsets)) { + charsets = new Array(arguments.length) + for (var i = 0; i < charsets.length; i++) { + charsets[i] = arguments[i] + } + } + + // no charsets, return all requested charsets + if (!charsets || charsets.length === 0) { + return this.negotiator.charsets() + } + + return this.negotiator.charsets(charsets)[0] || false +} + +/** + * Return accepted languages or best fit based on `langs`. + * + * Given `Accept-Language: en;q=0.8, es, pt` + * an array sorted by quality is returned: + * + * ['es', 'pt', 'en'] + * + * @param {String|Array} langs... + * @return {Array|String} + * @public + */ + +Accepts.prototype.lang = +Accepts.prototype.langs = +Accepts.prototype.language = +Accepts.prototype.languages = function (languages_) { + var languages = languages_ + + // support flattened arguments + if (languages && !Array.isArray(languages)) { + languages = new Array(arguments.length) + for (var i = 0; i < languages.length; i++) { + languages[i] = arguments[i] + } + } + + // no languages, return all requested languages + if (!languages || languages.length === 0) { + return this.negotiator.languages() + } + + return this.negotiator.languages(languages)[0] || false +} + +/** + * Convert extnames to mime. + * + * @param {String} type + * @return {String} + * @private + */ + +function extToMime (type) { + return type.indexOf('/') === -1 + ? mime.lookup(type) + : type +} + +/** + * Check if mime is valid. + * + * @param {String} type + * @return {String} + * @private + */ + +function validMime (type) { + return typeof type === 'string' +} diff --git a/scraper-service/node_modules/accepts/package.json b/scraper-service/node_modules/accepts/package.json new file mode 100644 index 0000000..0f2d15d --- /dev/null +++ b/scraper-service/node_modules/accepts/package.json @@ -0,0 +1,47 @@ +{ + "name": "accepts", + "description": "Higher-level content negotiation", + "version": "1.3.8", + "contributors": [ + "Douglas Christopher Wilson ", + "Jonathan Ong (http://jongleberry.com)" + ], + "license": "MIT", + "repository": "jshttp/accepts", + "dependencies": { + "mime-types": "~2.1.34", + "negotiator": "0.6.3" + }, + "devDependencies": { + "deep-equal": "1.0.1", + "eslint": "7.32.0", + "eslint-config-standard": "14.1.1", + "eslint-plugin-import": "2.25.4", + "eslint-plugin-markdown": "2.2.1", + "eslint-plugin-node": "11.1.0", + "eslint-plugin-promise": "4.3.1", + "eslint-plugin-standard": "4.1.0", + "mocha": "9.2.0", + "nyc": "15.1.0" + }, + "files": [ + "LICENSE", + "HISTORY.md", + "index.js" + ], + "engines": { + "node": ">= 0.6" + }, + "scripts": { + "lint": "eslint .", + "test": "mocha --reporter spec --check-leaks --bail test/", + "test-ci": "nyc --reporter=lcov --reporter=text npm test", + "test-cov": "nyc --reporter=html --reporter=text npm test" + }, + "keywords": [ + "content", + "negotiation", + "accept", + "accepts" + ] +} diff --git a/scraper-service/node_modules/array-flatten/LICENSE b/scraper-service/node_modules/array-flatten/LICENSE new file mode 100644 index 0000000..983fbe8 --- /dev/null +++ b/scraper-service/node_modules/array-flatten/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2014 Blake Embrey (hello@blakeembrey.com) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/scraper-service/node_modules/array-flatten/README.md b/scraper-service/node_modules/array-flatten/README.md new file mode 100644 index 0000000..91fa5b6 --- /dev/null +++ b/scraper-service/node_modules/array-flatten/README.md @@ -0,0 +1,43 @@ +# Array Flatten + +[![NPM version][npm-image]][npm-url] +[![NPM downloads][downloads-image]][downloads-url] +[![Build status][travis-image]][travis-url] +[![Test coverage][coveralls-image]][coveralls-url] + +> Flatten an array of nested arrays into a single flat array. Accepts an optional depth. + +## Installation + +``` +npm install array-flatten --save +``` + +## Usage + +```javascript +var flatten = require('array-flatten') + +flatten([1, [2, [3, [4, [5], 6], 7], 8], 9]) +//=> [1, 2, 3, 4, 5, 6, 7, 8, 9] + +flatten([1, [2, [3, [4, [5], 6], 7], 8], 9], 2) +//=> [1, 2, 3, [4, [5], 6], 7, 8, 9] + +(function () { + flatten(arguments) //=> [1, 2, 3] +})(1, [2, 3]) +``` + +## License + +MIT + +[npm-image]: https://img.shields.io/npm/v/array-flatten.svg?style=flat +[npm-url]: https://npmjs.org/package/array-flatten +[downloads-image]: https://img.shields.io/npm/dm/array-flatten.svg?style=flat +[downloads-url]: https://npmjs.org/package/array-flatten +[travis-image]: https://img.shields.io/travis/blakeembrey/array-flatten.svg?style=flat +[travis-url]: https://travis-ci.org/blakeembrey/array-flatten +[coveralls-image]: https://img.shields.io/coveralls/blakeembrey/array-flatten.svg?style=flat +[coveralls-url]: https://coveralls.io/r/blakeembrey/array-flatten?branch=master diff --git a/scraper-service/node_modules/array-flatten/array-flatten.js b/scraper-service/node_modules/array-flatten/array-flatten.js new file mode 100644 index 0000000..089117b --- /dev/null +++ b/scraper-service/node_modules/array-flatten/array-flatten.js @@ -0,0 +1,64 @@ +'use strict' + +/** + * Expose `arrayFlatten`. + */ +module.exports = arrayFlatten + +/** + * Recursive flatten function with depth. + * + * @param {Array} array + * @param {Array} result + * @param {Number} depth + * @return {Array} + */ +function flattenWithDepth (array, result, depth) { + for (var i = 0; i < array.length; i++) { + var value = array[i] + + if (depth > 0 && Array.isArray(value)) { + flattenWithDepth(value, result, depth - 1) + } else { + result.push(value) + } + } + + return result +} + +/** + * Recursive flatten function. Omitting depth is slightly faster. + * + * @param {Array} array + * @param {Array} result + * @return {Array} + */ +function flattenForever (array, result) { + for (var i = 0; i < array.length; i++) { + var value = array[i] + + if (Array.isArray(value)) { + flattenForever(value, result) + } else { + result.push(value) + } + } + + return result +} + +/** + * Flatten an array, with the ability to define a depth. + * + * @param {Array} array + * @param {Number} depth + * @return {Array} + */ +function arrayFlatten (array, depth) { + if (depth == null) { + return flattenForever(array, []) + } + + return flattenWithDepth(array, [], depth) +} diff --git a/scraper-service/node_modules/array-flatten/package.json b/scraper-service/node_modules/array-flatten/package.json new file mode 100644 index 0000000..1a24e2a --- /dev/null +++ b/scraper-service/node_modules/array-flatten/package.json @@ -0,0 +1,39 @@ +{ + "name": "array-flatten", + "version": "1.1.1", + "description": "Flatten an array of nested arrays into a single flat array", + "main": "array-flatten.js", + "files": [ + "array-flatten.js", + "LICENSE" + ], + "scripts": { + "test": "istanbul cover _mocha -- -R spec" + }, + "repository": { + "type": "git", + "url": "git://github.com/blakeembrey/array-flatten.git" + }, + "keywords": [ + "array", + "flatten", + "arguments", + "depth" + ], + "author": { + "name": "Blake Embrey", + "email": "hello@blakeembrey.com", + "url": "http://blakeembrey.me" + }, + "license": "MIT", + "bugs": { + "url": "https://github.com/blakeembrey/array-flatten/issues" + }, + "homepage": "https://github.com/blakeembrey/array-flatten", + "devDependencies": { + "istanbul": "^0.3.13", + "mocha": "^2.2.4", + "pre-commit": "^1.0.7", + "standard": "^3.7.3" + } +} diff --git a/scraper-service/node_modules/body-parser/HISTORY.md b/scraper-service/node_modules/body-parser/HISTORY.md new file mode 100644 index 0000000..e5c4fde --- /dev/null +++ b/scraper-service/node_modules/body-parser/HISTORY.md @@ -0,0 +1,686 @@ +1.20.5 / 2026-04-24 +=================== +* refactor(json): simplify strict mode error string construction +* fix: extended urlencoded parsing of arrays with >100 elements (#716) +* deps: qs@~6.15.1 + +1.20.4 / 2025-12-01 +=================== + + * deps: qs@~6.14.0 + * deps: use tilde notation for dependencies + * deps: http-errors@~2.0.1 + * deps: raw-body@~2.5.3 + +1.20.3 / 2024-09-10 +=================== + + * deps: qs@6.13.0 + * add `depth` option to customize the depth level in the parser + * IMPORTANT: The default `depth` level for parsing URL-encoded data is now `32` (previously was `Infinity`) + +1.20.2 / 2023-02-21 +=================== + + * Fix strict json error message on Node.js 19+ + * deps: content-type@~1.0.5 + - perf: skip value escaping when unnecessary + * deps: raw-body@2.5.2 + +1.20.1 / 2022-10-06 +=================== + + * deps: qs@6.11.0 + * perf: remove unnecessary object clone + +1.20.0 / 2022-04-02 +=================== + + * Fix error message for json parse whitespace in `strict` + * Fix internal error when inflated body exceeds limit + * Prevent loss of async hooks context + * Prevent hanging when request already read + * deps: depd@2.0.0 + - Replace internal `eval` usage with `Function` constructor + - Use instance methods on `process` to check for listeners + * deps: http-errors@2.0.0 + - deps: depd@2.0.0 + - deps: statuses@2.0.1 + * deps: on-finished@2.4.1 + * deps: qs@6.10.3 + * deps: raw-body@2.5.1 + - deps: http-errors@2.0.0 + +1.19.2 / 2022-02-15 +=================== + + * deps: bytes@3.1.2 + * deps: qs@6.9.7 + * Fix handling of `__proto__` keys + * deps: raw-body@2.4.3 + - deps: bytes@3.1.2 + +1.19.1 / 2021-12-10 +=================== + + * deps: bytes@3.1.1 + * deps: http-errors@1.8.1 + - deps: inherits@2.0.4 + - deps: toidentifier@1.0.1 + - deps: setprototypeof@1.2.0 + * deps: qs@6.9.6 + * deps: raw-body@2.4.2 + - deps: bytes@3.1.1 + - deps: http-errors@1.8.1 + * deps: safe-buffer@5.2.1 + * deps: type-is@~1.6.18 + +1.19.0 / 2019-04-25 +=================== + + * deps: bytes@3.1.0 + - Add petabyte (`pb`) support + * deps: http-errors@1.7.2 + - Set constructor name when possible + - deps: setprototypeof@1.1.1 + - deps: statuses@'>= 1.5.0 < 2' + * deps: iconv-lite@0.4.24 + - Added encoding MIK + * deps: qs@6.7.0 + - Fix parsing array brackets after index + * deps: raw-body@2.4.0 + - deps: bytes@3.1.0 + - deps: http-errors@1.7.2 + - deps: iconv-lite@0.4.24 + * deps: type-is@~1.6.17 + - deps: mime-types@~2.1.24 + - perf: prevent internal `throw` on invalid type + +1.18.3 / 2018-05-14 +=================== + + * Fix stack trace for strict json parse error + * deps: depd@~1.1.2 + - perf: remove argument reassignment + * deps: http-errors@~1.6.3 + - deps: depd@~1.1.2 + - deps: setprototypeof@1.1.0 + - deps: statuses@'>= 1.3.1 < 2' + * deps: iconv-lite@0.4.23 + - Fix loading encoding with year appended + - Fix deprecation warnings on Node.js 10+ + * deps: qs@6.5.2 + * deps: raw-body@2.3.3 + - deps: http-errors@1.6.3 + - deps: iconv-lite@0.4.23 + * deps: type-is@~1.6.16 + - deps: mime-types@~2.1.18 + +1.18.2 / 2017-09-22 +=================== + + * deps: debug@2.6.9 + * perf: remove argument reassignment + +1.18.1 / 2017-09-12 +=================== + + * deps: content-type@~1.0.4 + - perf: remove argument reassignment + - perf: skip parameter parsing when no parameters + * deps: iconv-lite@0.4.19 + - Fix ISO-8859-1 regression + - Update Windows-1255 + * deps: qs@6.5.1 + - Fix parsing & compacting very deep objects + * deps: raw-body@2.3.2 + - deps: iconv-lite@0.4.19 + +1.18.0 / 2017-09-08 +=================== + + * Fix JSON strict violation error to match native parse error + * Include the `body` property on verify errors + * Include the `type` property on all generated errors + * Use `http-errors` to set status code on errors + * deps: bytes@3.0.0 + * deps: debug@2.6.8 + * deps: depd@~1.1.1 + - Remove unnecessary `Buffer` loading + * deps: http-errors@~1.6.2 + - deps: depd@1.1.1 + * deps: iconv-lite@0.4.18 + - Add support for React Native + - Add a warning if not loaded as utf-8 + - Fix CESU-8 decoding in Node.js 8 + - Improve speed of ISO-8859-1 encoding + * deps: qs@6.5.0 + * deps: raw-body@2.3.1 + - Use `http-errors` for standard emitted errors + - deps: bytes@3.0.0 + - deps: iconv-lite@0.4.18 + - perf: skip buffer decoding on overage chunk + * perf: prevent internal `throw` when missing charset + +1.17.2 / 2017-05-17 +=================== + + * deps: debug@2.6.7 + - Fix `DEBUG_MAX_ARRAY_LENGTH` + - deps: ms@2.0.0 + * deps: type-is@~1.6.15 + - deps: mime-types@~2.1.15 + +1.17.1 / 2017-03-06 +=================== + + * deps: qs@6.4.0 + - Fix regression parsing keys starting with `[` + +1.17.0 / 2017-03-01 +=================== + + * deps: http-errors@~1.6.1 + - Make `message` property enumerable for `HttpError`s + - deps: setprototypeof@1.0.3 + * deps: qs@6.3.1 + - Fix compacting nested arrays + +1.16.1 / 2017-02-10 +=================== + + * deps: debug@2.6.1 + - Fix deprecation messages in WebStorm and other editors + - Undeprecate `DEBUG_FD` set to `1` or `2` + +1.16.0 / 2017-01-17 +=================== + + * deps: debug@2.6.0 + - Allow colors in workers + - Deprecated `DEBUG_FD` environment variable + - Fix error when running under React Native + - Use same color for same namespace + - deps: ms@0.7.2 + * deps: http-errors@~1.5.1 + - deps: inherits@2.0.3 + - deps: setprototypeof@1.0.2 + - deps: statuses@'>= 1.3.1 < 2' + * deps: iconv-lite@0.4.15 + - Added encoding MS-31J + - Added encoding MS-932 + - Added encoding MS-936 + - Added encoding MS-949 + - Added encoding MS-950 + - Fix GBK/GB18030 handling of Euro character + * deps: qs@6.2.1 + - Fix array parsing from skipping empty values + * deps: raw-body@~2.2.0 + - deps: iconv-lite@0.4.15 + * deps: type-is@~1.6.14 + - deps: mime-types@~2.1.13 + +1.15.2 / 2016-06-19 +=================== + + * deps: bytes@2.4.0 + * deps: content-type@~1.0.2 + - perf: enable strict mode + * deps: http-errors@~1.5.0 + - Use `setprototypeof` module to replace `__proto__` setting + - deps: statuses@'>= 1.3.0 < 2' + - perf: enable strict mode + * deps: qs@6.2.0 + * deps: raw-body@~2.1.7 + - deps: bytes@2.4.0 + - perf: remove double-cleanup on happy path + * deps: type-is@~1.6.13 + - deps: mime-types@~2.1.11 + +1.15.1 / 2016-05-05 +=================== + + * deps: bytes@2.3.0 + - Drop partial bytes on all parsed units + - Fix parsing byte string that looks like hex + * deps: raw-body@~2.1.6 + - deps: bytes@2.3.0 + * deps: type-is@~1.6.12 + - deps: mime-types@~2.1.10 + +1.15.0 / 2016-02-10 +=================== + + * deps: http-errors@~1.4.0 + - Add `HttpError` export, for `err instanceof createError.HttpError` + - deps: inherits@2.0.1 + - deps: statuses@'>= 1.2.1 < 2' + * deps: qs@6.1.0 + * deps: type-is@~1.6.11 + - deps: mime-types@~2.1.9 + +1.14.2 / 2015-12-16 +=================== + + * deps: bytes@2.2.0 + * deps: iconv-lite@0.4.13 + * deps: qs@5.2.0 + * deps: raw-body@~2.1.5 + - deps: bytes@2.2.0 + - deps: iconv-lite@0.4.13 + * deps: type-is@~1.6.10 + - deps: mime-types@~2.1.8 + +1.14.1 / 2015-09-27 +=================== + + * Fix issue where invalid charset results in 400 when `verify` used + * deps: iconv-lite@0.4.12 + - Fix CESU-8 decoding in Node.js 4.x + * deps: raw-body@~2.1.4 + - Fix masking critical errors from `iconv-lite` + - deps: iconv-lite@0.4.12 + * deps: type-is@~1.6.9 + - deps: mime-types@~2.1.7 + +1.14.0 / 2015-09-16 +=================== + + * Fix JSON strict parse error to match syntax errors + * Provide static `require` analysis in `urlencoded` parser + * deps: depd@~1.1.0 + - Support web browser loading + * deps: qs@5.1.0 + * deps: raw-body@~2.1.3 + - Fix sync callback when attaching data listener causes sync read + * deps: type-is@~1.6.8 + - Fix type error when given invalid type to match against + - deps: mime-types@~2.1.6 + +1.13.3 / 2015-07-31 +=================== + + * deps: type-is@~1.6.6 + - deps: mime-types@~2.1.4 + +1.13.2 / 2015-07-05 +=================== + + * deps: iconv-lite@0.4.11 + * deps: qs@4.0.0 + - Fix dropping parameters like `hasOwnProperty` + - Fix user-visible incompatibilities from 3.1.0 + - Fix various parsing edge cases + * deps: raw-body@~2.1.2 + - Fix error stack traces to skip `makeError` + - deps: iconv-lite@0.4.11 + * deps: type-is@~1.6.4 + - deps: mime-types@~2.1.2 + - perf: enable strict mode + - perf: remove argument reassignment + +1.13.1 / 2015-06-16 +=================== + + * deps: qs@2.4.2 + - Downgraded from 3.1.0 because of user-visible incompatibilities + +1.13.0 / 2015-06-14 +=================== + + * Add `statusCode` property on `Error`s, in addition to `status` + * Change `type` default to `application/json` for JSON parser + * Change `type` default to `application/x-www-form-urlencoded` for urlencoded parser + * Provide static `require` analysis + * Use the `http-errors` module to generate errors + * deps: bytes@2.1.0 + - Slight optimizations + * deps: iconv-lite@0.4.10 + - The encoding UTF-16 without BOM now defaults to UTF-16LE when detection fails + - Leading BOM is now removed when decoding + * deps: on-finished@~2.3.0 + - Add defined behavior for HTTP `CONNECT` requests + - Add defined behavior for HTTP `Upgrade` requests + - deps: ee-first@1.1.1 + * deps: qs@3.1.0 + - Fix dropping parameters like `hasOwnProperty` + - Fix various parsing edge cases + - Parsed object now has `null` prototype + * deps: raw-body@~2.1.1 + - Use `unpipe` module for unpiping requests + - deps: iconv-lite@0.4.10 + * deps: type-is@~1.6.3 + - deps: mime-types@~2.1.1 + - perf: reduce try block size + - perf: remove bitwise operations + * perf: enable strict mode + * perf: remove argument reassignment + * perf: remove delete call + +1.12.4 / 2015-05-10 +=================== + + * deps: debug@~2.2.0 + * deps: qs@2.4.2 + - Fix allowing parameters like `constructor` + * deps: on-finished@~2.2.1 + * deps: raw-body@~2.0.1 + - Fix a false-positive when unpiping in Node.js 0.8 + - deps: bytes@2.0.1 + * deps: type-is@~1.6.2 + - deps: mime-types@~2.0.11 + +1.12.3 / 2015-04-15 +=================== + + * Slight efficiency improvement when not debugging + * deps: depd@~1.0.1 + * deps: iconv-lite@0.4.8 + - Add encoding alias UNICODE-1-1-UTF-7 + * deps: raw-body@1.3.4 + - Fix hanging callback if request aborts during read + - deps: iconv-lite@0.4.8 + +1.12.2 / 2015-03-16 +=================== + + * deps: qs@2.4.1 + - Fix error when parameter `hasOwnProperty` is present + +1.12.1 / 2015-03-15 +=================== + + * deps: debug@~2.1.3 + - Fix high intensity foreground color for bold + - deps: ms@0.7.0 + * deps: type-is@~1.6.1 + - deps: mime-types@~2.0.10 + +1.12.0 / 2015-02-13 +=================== + + * add `debug` messages + * accept a function for the `type` option + * use `content-type` to parse `Content-Type` headers + * deps: iconv-lite@0.4.7 + - Gracefully support enumerables on `Object.prototype` + * deps: raw-body@1.3.3 + - deps: iconv-lite@0.4.7 + * deps: type-is@~1.6.0 + - fix argument reassignment + - fix false-positives in `hasBody` `Transfer-Encoding` check + - support wildcard for both type and subtype (`*/*`) + - deps: mime-types@~2.0.9 + +1.11.0 / 2015-01-30 +=================== + + * make internal `extended: true` depth limit infinity + * deps: type-is@~1.5.6 + - deps: mime-types@~2.0.8 + +1.10.2 / 2015-01-20 +=================== + + * deps: iconv-lite@0.4.6 + - Fix rare aliases of single-byte encodings + * deps: raw-body@1.3.2 + - deps: iconv-lite@0.4.6 + +1.10.1 / 2015-01-01 +=================== + + * deps: on-finished@~2.2.0 + * deps: type-is@~1.5.5 + - deps: mime-types@~2.0.7 + +1.10.0 / 2014-12-02 +=================== + + * make internal `extended: true` array limit dynamic + +1.9.3 / 2014-11-21 +================== + + * deps: iconv-lite@0.4.5 + - Fix Windows-31J and X-SJIS encoding support + * deps: qs@2.3.3 + - Fix `arrayLimit` behavior + * deps: raw-body@1.3.1 + - deps: iconv-lite@0.4.5 + * deps: type-is@~1.5.3 + - deps: mime-types@~2.0.3 + +1.9.2 / 2014-10-27 +================== + + * deps: qs@2.3.2 + - Fix parsing of mixed objects and values + +1.9.1 / 2014-10-22 +================== + + * deps: on-finished@~2.1.1 + - Fix handling of pipelined requests + * deps: qs@2.3.0 + - Fix parsing of mixed implicit and explicit arrays + * deps: type-is@~1.5.2 + - deps: mime-types@~2.0.2 + +1.9.0 / 2014-09-24 +================== + + * include the charset in "unsupported charset" error message + * include the encoding in "unsupported content encoding" error message + * deps: depd@~1.0.0 + +1.8.4 / 2014-09-23 +================== + + * fix content encoding to be case-insensitive + +1.8.3 / 2014-09-19 +================== + + * deps: qs@2.2.4 + - Fix issue with object keys starting with numbers truncated + +1.8.2 / 2014-09-15 +================== + + * deps: depd@0.4.5 + +1.8.1 / 2014-09-07 +================== + + * deps: media-typer@0.3.0 + * deps: type-is@~1.5.1 + +1.8.0 / 2014-09-05 +================== + + * make empty-body-handling consistent between chunked requests + - empty `json` produces `{}` + - empty `raw` produces `new Buffer(0)` + - empty `text` produces `''` + - empty `urlencoded` produces `{}` + * deps: qs@2.2.3 + - Fix issue where first empty value in array is discarded + * deps: type-is@~1.5.0 + - fix `hasbody` to be true for `content-length: 0` + +1.7.0 / 2014-09-01 +================== + + * add `parameterLimit` option to `urlencoded` parser + * change `urlencoded` extended array limit to 100 + * respond with 413 when over `parameterLimit` in `urlencoded` + +1.6.7 / 2014-08-29 +================== + + * deps: qs@2.2.2 + - Remove unnecessary cloning + +1.6.6 / 2014-08-27 +================== + + * deps: qs@2.2.0 + - Array parsing fix + - Performance improvements + +1.6.5 / 2014-08-16 +================== + + * deps: on-finished@2.1.0 + +1.6.4 / 2014-08-14 +================== + + * deps: qs@1.2.2 + +1.6.3 / 2014-08-10 +================== + + * deps: qs@1.2.1 + +1.6.2 / 2014-08-07 +================== + + * deps: qs@1.2.0 + - Fix parsing array of objects + +1.6.1 / 2014-08-06 +================== + + * deps: qs@1.1.0 + - Accept urlencoded square brackets + - Accept empty values in implicit array notation + +1.6.0 / 2014-08-05 +================== + + * deps: qs@1.0.2 + - Complete rewrite + - Limits array length to 20 + - Limits object depth to 5 + - Limits parameters to 1,000 + +1.5.2 / 2014-07-27 +================== + + * deps: depd@0.4.4 + - Work-around v8 generating empty stack traces + +1.5.1 / 2014-07-26 +================== + + * deps: depd@0.4.3 + - Fix exception when global `Error.stackTraceLimit` is too low + +1.5.0 / 2014-07-20 +================== + + * deps: depd@0.4.2 + - Add `TRACE_DEPRECATION` environment variable + - Remove non-standard grey color from color output + - Support `--no-deprecation` argument + - Support `--trace-deprecation` argument + * deps: iconv-lite@0.4.4 + - Added encoding UTF-7 + * deps: raw-body@1.3.0 + - deps: iconv-lite@0.4.4 + - Added encoding UTF-7 + - Fix `Cannot switch to old mode now` error on Node.js 0.10+ + * deps: type-is@~1.3.2 + +1.4.3 / 2014-06-19 +================== + + * deps: type-is@1.3.1 + - fix global variable leak + +1.4.2 / 2014-06-19 +================== + + * deps: type-is@1.3.0 + - improve type parsing + +1.4.1 / 2014-06-19 +================== + + * fix urlencoded extended deprecation message + +1.4.0 / 2014-06-19 +================== + + * add `text` parser + * add `raw` parser + * check accepted charset in content-type (accepts utf-8) + * check accepted encoding in content-encoding (accepts identity) + * deprecate `bodyParser()` middleware; use `.json()` and `.urlencoded()` as needed + * deprecate `urlencoded()` without provided `extended` option + * lazy-load urlencoded parsers + * parsers split into files for reduced mem usage + * support gzip and deflate bodies + - set `inflate: false` to turn off + * deps: raw-body@1.2.2 + - Support all encodings from `iconv-lite` + +1.3.1 / 2014-06-11 +================== + + * deps: type-is@1.2.1 + - Switch dependency from mime to mime-types@1.0.0 + +1.3.0 / 2014-05-31 +================== + + * add `extended` option to urlencoded parser + +1.2.2 / 2014-05-27 +================== + + * deps: raw-body@1.1.6 + - assert stream encoding on node.js 0.8 + - assert stream encoding on node.js < 0.10.6 + - deps: bytes@1 + +1.2.1 / 2014-05-26 +================== + + * invoke `next(err)` after request fully read + - prevents hung responses and socket hang ups + +1.2.0 / 2014-05-11 +================== + + * add `verify` option + * deps: type-is@1.2.0 + - support suffix matching + +1.1.2 / 2014-05-11 +================== + + * improve json parser speed + +1.1.1 / 2014-05-11 +================== + + * fix repeated limit parsing with every request + +1.1.0 / 2014-05-10 +================== + + * add `type` option + * deps: pin for safety and consistency + +1.0.2 / 2014-04-14 +================== + + * use `type-is` module + +1.0.1 / 2014-03-20 +================== + + * lower default limits to 100kb diff --git a/scraper-service/node_modules/body-parser/LICENSE b/scraper-service/node_modules/body-parser/LICENSE new file mode 100644 index 0000000..386b7b6 --- /dev/null +++ b/scraper-service/node_modules/body-parser/LICENSE @@ -0,0 +1,23 @@ +(The MIT License) + +Copyright (c) 2014 Jonathan Ong +Copyright (c) 2014-2015 Douglas Christopher Wilson + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +'Software'), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/scraper-service/node_modules/body-parser/README.md b/scraper-service/node_modules/body-parser/README.md new file mode 100644 index 0000000..f6661b7 --- /dev/null +++ b/scraper-service/node_modules/body-parser/README.md @@ -0,0 +1,476 @@ +# body-parser + +[![NPM Version][npm-version-image]][npm-url] +[![NPM Downloads][npm-downloads-image]][npm-url] +[![Build Status][ci-image]][ci-url] +[![Test Coverage][coveralls-image]][coveralls-url] +[![OpenSSF Scorecard Badge][ossf-scorecard-badge]][ossf-scorecard-visualizer] + +Node.js body parsing middleware. + +Parse incoming request bodies in a middleware before your handlers, available +under the `req.body` property. + +**Note** As `req.body`'s shape is based on user-controlled input, all +properties and values in this object are untrusted and should be validated +before trusting. For example, `req.body.foo.toString()` may fail in multiple +ways, for example the `foo` property may not be there or may not be a string, +and `toString` may not be a function and instead a string or other user input. + +[Learn about the anatomy of an HTTP transaction in Node.js](https://nodejs.org/en/docs/guides/anatomy-of-an-http-transaction/). + +_This does not handle multipart bodies_, due to their complex and typically +large nature. For multipart bodies, you may be interested in the following +modules: + + * [busboy](https://www.npmjs.org/package/busboy#readme) and + [connect-busboy](https://www.npmjs.org/package/connect-busboy#readme) + * [multiparty](https://www.npmjs.org/package/multiparty#readme) and + [connect-multiparty](https://www.npmjs.org/package/connect-multiparty#readme) + * [formidable](https://www.npmjs.org/package/formidable#readme) + * [multer](https://www.npmjs.org/package/multer#readme) + +This module provides the following parsers: + + * [JSON body parser](#bodyparserjsonoptions) + * [Raw body parser](#bodyparserrawoptions) + * [Text body parser](#bodyparsertextoptions) + * [URL-encoded form body parser](#bodyparserurlencodedoptions) + +Other body parsers you might be interested in: + +- [body](https://www.npmjs.org/package/body#readme) +- [co-body](https://www.npmjs.org/package/co-body#readme) + +## Installation + +```sh +$ npm install body-parser +``` + +## API + +```js +var bodyParser = require('body-parser') +``` + +The `bodyParser` object exposes various factories to create middlewares. All +middlewares will populate the `req.body` property with the parsed body when +the `Content-Type` request header matches the `type` option, or an empty +object (`{}`) if there was no body to parse, the `Content-Type` was not matched, +or an error occurred. + +The various errors returned by this module are described in the +[errors section](#errors). + +### bodyParser.json([options]) + +Returns middleware that only parses `json` and only looks at requests where +the `Content-Type` header matches the `type` option. This parser accepts any +Unicode encoding of the body and supports automatic inflation of `gzip` and +`deflate` encodings. + +A new `body` object containing the parsed data is populated on the `request` +object after the middleware (i.e. `req.body`). + +#### Options + +The `json` function takes an optional `options` object that may contain any of +the following keys: + +##### inflate + +When set to `true`, then deflated (compressed) bodies will be inflated; when +`false`, deflated bodies are rejected. Defaults to `true`. + +##### limit + +Controls the maximum request body size. If this is a number, then the value +specifies the number of bytes; if it is a string, the value is passed to the +[bytes](https://www.npmjs.com/package/bytes) library for parsing. Defaults +to `'100kb'`. + +##### reviver + +The `reviver` option is passed directly to `JSON.parse` as the second +argument. You can find more information on this argument +[in the MDN documentation about JSON.parse](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/JSON/parse#Example.3A_Using_the_reviver_parameter). + +##### strict + +When set to `true`, will only accept arrays and objects; when `false` will +accept anything `JSON.parse` accepts. Defaults to `true`. + +##### type + +The `type` option is used to determine what media type the middleware will +parse. This option can be a string, array of strings, or a function. If not a +function, `type` option is passed directly to the +[type-is](https://www.npmjs.org/package/type-is#readme) library and this can +be an extension name (like `json`), a mime type (like `application/json`), or +a mime type with a wildcard (like `*/*` or `*/json`). If a function, the `type` +option is called as `fn(req)` and the request is parsed if it returns a truthy +value. Defaults to `application/json`. + +##### verify + +The `verify` option, if supplied, is called as `verify(req, res, buf, encoding)`, +where `buf` is a `Buffer` of the raw request body and `encoding` is the +encoding of the request. The parsing can be aborted by throwing an error. + +### bodyParser.raw([options]) + +Returns middleware that parses all bodies as a `Buffer` and only looks at +requests where the `Content-Type` header matches the `type` option. This +parser supports automatic inflation of `gzip` and `deflate` encodings. + +A new `body` object containing the parsed data is populated on the `request` +object after the middleware (i.e. `req.body`). This will be a `Buffer` object +of the body. + +#### Options + +The `raw` function takes an optional `options` object that may contain any of +the following keys: + +##### inflate + +When set to `true`, then deflated (compressed) bodies will be inflated; when +`false`, deflated bodies are rejected. Defaults to `true`. + +##### limit + +Controls the maximum request body size. If this is a number, then the value +specifies the number of bytes; if it is a string, the value is passed to the +[bytes](https://www.npmjs.com/package/bytes) library for parsing. Defaults +to `'100kb'`. + +##### type + +The `type` option is used to determine what media type the middleware will +parse. This option can be a string, array of strings, or a function. +If not a function, `type` option is passed directly to the +[type-is](https://www.npmjs.org/package/type-is#readme) library and this +can be an extension name (like `bin`), a mime type (like +`application/octet-stream`), or a mime type with a wildcard (like `*/*` or +`application/*`). If a function, the `type` option is called as `fn(req)` +and the request is parsed if it returns a truthy value. Defaults to +`application/octet-stream`. + +##### verify + +The `verify` option, if supplied, is called as `verify(req, res, buf, encoding)`, +where `buf` is a `Buffer` of the raw request body and `encoding` is the +encoding of the request. The parsing can be aborted by throwing an error. + +### bodyParser.text([options]) + +Returns middleware that parses all bodies as a string and only looks at +requests where the `Content-Type` header matches the `type` option. This +parser supports automatic inflation of `gzip` and `deflate` encodings. + +A new `body` string containing the parsed data is populated on the `request` +object after the middleware (i.e. `req.body`). This will be a string of the +body. + +#### Options + +The `text` function takes an optional `options` object that may contain any of +the following keys: + +##### defaultCharset + +Specify the default character set for the text content if the charset is not +specified in the `Content-Type` header of the request. Defaults to `utf-8`. + +##### inflate + +When set to `true`, then deflated (compressed) bodies will be inflated; when +`false`, deflated bodies are rejected. Defaults to `true`. + +##### limit + +Controls the maximum request body size. If this is a number, then the value +specifies the number of bytes; if it is a string, the value is passed to the +[bytes](https://www.npmjs.com/package/bytes) library for parsing. Defaults +to `'100kb'`. + +##### type + +The `type` option is used to determine what media type the middleware will +parse. This option can be a string, array of strings, or a function. If not +a function, `type` option is passed directly to the +[type-is](https://www.npmjs.org/package/type-is#readme) library and this can +be an extension name (like `txt`), a mime type (like `text/plain`), or a mime +type with a wildcard (like `*/*` or `text/*`). If a function, the `type` +option is called as `fn(req)` and the request is parsed if it returns a +truthy value. Defaults to `text/plain`. + +##### verify + +The `verify` option, if supplied, is called as `verify(req, res, buf, encoding)`, +where `buf` is a `Buffer` of the raw request body and `encoding` is the +encoding of the request. The parsing can be aborted by throwing an error. + +### bodyParser.urlencoded([options]) + +Returns middleware that only parses `urlencoded` bodies and only looks at +requests where the `Content-Type` header matches the `type` option. This +parser accepts only UTF-8 encoding of the body and supports automatic +inflation of `gzip` and `deflate` encodings. + +A new `body` object containing the parsed data is populated on the `request` +object after the middleware (i.e. `req.body`). This object will contain +key-value pairs, where the value can be a string or array (when `extended` is +`false`), or any type (when `extended` is `true`). + +#### Options + +The `urlencoded` function takes an optional `options` object that may contain +any of the following keys: + +##### extended + +The `extended` option allows to choose between parsing the URL-encoded data +with the `querystring` library (when `false`) or the `qs` library (when +`true`). The "extended" syntax allows for rich objects and arrays to be +encoded into the URL-encoded format, allowing for a JSON-like experience +with URL-encoded. For more information, please +[see the qs library](https://www.npmjs.org/package/qs#readme). + +Defaults to `true`, but using the default has been deprecated. Please +research into the difference between `qs` and `querystring` and choose the +appropriate setting. + +##### inflate + +When set to `true`, then deflated (compressed) bodies will be inflated; when +`false`, deflated bodies are rejected. Defaults to `true`. + +##### limit + +Controls the maximum request body size. If this is a number, then the value +specifies the number of bytes; if it is a string, the value is passed to the +[bytes](https://www.npmjs.com/package/bytes) library for parsing. Defaults +to `'100kb'`. + +##### parameterLimit + +The `parameterLimit` option controls the maximum number of parameters that +are allowed in the URL-encoded data. If a request contains more parameters +than this value, a 413 will be returned to the client. Defaults to `1000`. + +##### type + +The `type` option is used to determine what media type the middleware will +parse. This option can be a string, array of strings, or a function. If not +a function, `type` option is passed directly to the +[type-is](https://www.npmjs.org/package/type-is#readme) library and this can +be an extension name (like `urlencoded`), a mime type (like +`application/x-www-form-urlencoded`), or a mime type with a wildcard (like +`*/x-www-form-urlencoded`). If a function, the `type` option is called as +`fn(req)` and the request is parsed if it returns a truthy value. Defaults +to `application/x-www-form-urlencoded`. + +##### verify + +The `verify` option, if supplied, is called as `verify(req, res, buf, encoding)`, +where `buf` is a `Buffer` of the raw request body and `encoding` is the +encoding of the request. The parsing can be aborted by throwing an error. + +#### depth + +The `depth` option is used to configure the maximum depth of the `qs` library when `extended` is `true`. This allows you to limit the amount of keys that are parsed and can be useful to prevent certain types of abuse. Defaults to `32`. It is recommended to keep this value as low as possible. + +## Errors + +The middlewares provided by this module create errors using the +[`http-errors` module](https://www.npmjs.com/package/http-errors). The errors +will typically have a `status`/`statusCode` property that contains the suggested +HTTP response code, an `expose` property to determine if the `message` property +should be displayed to the client, a `type` property to determine the type of +error without matching against the `message`, and a `body` property containing +the read body, if available. + +The following are the common errors created, though any error can come through +for various reasons. + +### content encoding unsupported + +This error will occur when the request had a `Content-Encoding` header that +contained an encoding but the "inflation" option was set to `false`. The +`status` property is set to `415`, the `type` property is set to +`'encoding.unsupported'`, and the `charset` property will be set to the +encoding that is unsupported. + +### entity parse failed + +This error will occur when the request contained an entity that could not be +parsed by the middleware. The `status` property is set to `400`, the `type` +property is set to `'entity.parse.failed'`, and the `body` property is set to +the entity value that failed parsing. + +### entity verify failed + +This error will occur when the request contained an entity that could not be +failed verification by the defined `verify` option. The `status` property is +set to `403`, the `type` property is set to `'entity.verify.failed'`, and the +`body` property is set to the entity value that failed verification. + +### request aborted + +This error will occur when the request is aborted by the client before reading +the body has finished. The `received` property will be set to the number of +bytes received before the request was aborted and the `expected` property is +set to the number of expected bytes. The `status` property is set to `400` +and `type` property is set to `'request.aborted'`. + +### request entity too large + +This error will occur when the request body's size is larger than the "limit" +option. The `limit` property will be set to the byte limit and the `length` +property will be set to the request body's length. The `status` property is +set to `413` and the `type` property is set to `'entity.too.large'`. + +### request size did not match content length + +This error will occur when the request's length did not match the length from +the `Content-Length` header. This typically occurs when the request is malformed, +typically when the `Content-Length` header was calculated based on characters +instead of bytes. The `status` property is set to `400` and the `type` property +is set to `'request.size.invalid'`. + +### stream encoding should not be set + +This error will occur when something called the `req.setEncoding` method prior +to this middleware. This module operates directly on bytes only and you cannot +call `req.setEncoding` when using this module. The `status` property is set to +`500` and the `type` property is set to `'stream.encoding.set'`. + +### stream is not readable + +This error will occur when the request is no longer readable when this middleware +attempts to read it. This typically means something other than a middleware from +this module read the request body already and the middleware was also configured to +read the same request. The `status` property is set to `500` and the `type` +property is set to `'stream.not.readable'`. + +### too many parameters + +This error will occur when the content of the request exceeds the configured +`parameterLimit` for the `urlencoded` parser. The `status` property is set to +`413` and the `type` property is set to `'parameters.too.many'`. + +### unsupported charset "BOGUS" + +This error will occur when the request had a charset parameter in the +`Content-Type` header, but the `iconv-lite` module does not support it OR the +parser does not support it. The charset is contained in the message as well +as in the `charset` property. The `status` property is set to `415`, the +`type` property is set to `'charset.unsupported'`, and the `charset` property +is set to the charset that is unsupported. + +### unsupported content encoding "bogus" + +This error will occur when the request had a `Content-Encoding` header that +contained an unsupported encoding. The encoding is contained in the message +as well as in the `encoding` property. The `status` property is set to `415`, +the `type` property is set to `'encoding.unsupported'`, and the `encoding` +property is set to the encoding that is unsupported. + +### The input exceeded the depth + +This error occurs when using `bodyParser.urlencoded` with the `extended` property set to `true` and the input exceeds the configured `depth` option. The `status` property is set to `400`. It is recommended to review the `depth` option and evaluate if it requires a higher value. When the `depth` option is set to `32` (default value), the error will not be thrown. + +## Examples + +### Express/Connect top-level generic + +This example demonstrates adding a generic JSON and URL-encoded parser as a +top-level middleware, which will parse the bodies of all incoming requests. +This is the simplest setup. + +```js +var express = require('express') +var bodyParser = require('body-parser') + +var app = express() + +// parse application/x-www-form-urlencoded +app.use(bodyParser.urlencoded({ extended: false })) + +// parse application/json +app.use(bodyParser.json()) + +app.use(function (req, res) { + res.setHeader('Content-Type', 'text/plain') + res.write('you posted:\n') + res.end(JSON.stringify(req.body, null, 2)) +}) +``` + +### Express route-specific + +This example demonstrates adding body parsers specifically to the routes that +need them. In general, this is the most recommended way to use body-parser with +Express. + +```js +var express = require('express') +var bodyParser = require('body-parser') + +var app = express() + +// create application/json parser +var jsonParser = bodyParser.json() + +// create application/x-www-form-urlencoded parser +var urlencodedParser = bodyParser.urlencoded({ extended: false }) + +// POST /login gets urlencoded bodies +app.post('/login', urlencodedParser, function (req, res) { + res.send('welcome, ' + req.body.username) +}) + +// POST /api/users gets JSON bodies +app.post('/api/users', jsonParser, function (req, res) { + // create user in req.body +}) +``` + +### Change accepted type for parsers + +All the parsers accept a `type` option which allows you to change the +`Content-Type` that the middleware will parse. + +```js +var express = require('express') +var bodyParser = require('body-parser') + +var app = express() + +// parse various different custom JSON types as JSON +app.use(bodyParser.json({ type: 'application/*+json' })) + +// parse some custom thing into a Buffer +app.use(bodyParser.raw({ type: 'application/vnd.custom-type' })) + +// parse an HTML body into a string +app.use(bodyParser.text({ type: 'text/html' })) +``` + +## License + +[MIT](LICENSE) + +[ci-image]: https://badgen.net/github/checks/expressjs/body-parser/master?label=ci +[ci-url]: https://github.com/expressjs/body-parser/actions/workflows/ci.yml +[coveralls-image]: https://badgen.net/coveralls/c/github/expressjs/body-parser/master +[coveralls-url]: https://coveralls.io/r/expressjs/body-parser?branch=master +[node-version-image]: https://badgen.net/npm/node/body-parser +[node-version-url]: https://nodejs.org/en/download +[npm-downloads-image]: https://badgen.net/npm/dm/body-parser +[npm-url]: https://npmjs.org/package/body-parser +[npm-version-image]: https://badgen.net/npm/v/body-parser +[ossf-scorecard-badge]: https://api.scorecard.dev/projects/github.com/expressjs/body-parser/badge +[ossf-scorecard-visualizer]: https://ossf.github.io/scorecard-visualizer/#/projects/github.com/expressjs/body-parser \ No newline at end of file diff --git a/scraper-service/node_modules/body-parser/index.js b/scraper-service/node_modules/body-parser/index.js new file mode 100644 index 0000000..bb24d73 --- /dev/null +++ b/scraper-service/node_modules/body-parser/index.js @@ -0,0 +1,156 @@ +/*! + * body-parser + * Copyright(c) 2014-2015 Douglas Christopher Wilson + * MIT Licensed + */ + +'use strict' + +/** + * Module dependencies. + * @private + */ + +var deprecate = require('depd')('body-parser') + +/** + * Cache of loaded parsers. + * @private + */ + +var parsers = Object.create(null) + +/** + * @typedef Parsers + * @type {function} + * @property {function} json + * @property {function} raw + * @property {function} text + * @property {function} urlencoded + */ + +/** + * Module exports. + * @type {Parsers} + */ + +exports = module.exports = deprecate.function(bodyParser, + 'bodyParser: use individual json/urlencoded middlewares') + +/** + * JSON parser. + * @public + */ + +Object.defineProperty(exports, 'json', { + configurable: true, + enumerable: true, + get: createParserGetter('json') +}) + +/** + * Raw parser. + * @public + */ + +Object.defineProperty(exports, 'raw', { + configurable: true, + enumerable: true, + get: createParserGetter('raw') +}) + +/** + * Text parser. + * @public + */ + +Object.defineProperty(exports, 'text', { + configurable: true, + enumerable: true, + get: createParserGetter('text') +}) + +/** + * URL-encoded parser. + * @public + */ + +Object.defineProperty(exports, 'urlencoded', { + configurable: true, + enumerable: true, + get: createParserGetter('urlencoded') +}) + +/** + * Create a middleware to parse json and urlencoded bodies. + * + * @param {object} [options] + * @return {function} + * @deprecated + * @public + */ + +function bodyParser (options) { + // use default type for parsers + var opts = Object.create(options || null, { + type: { + configurable: true, + enumerable: true, + value: undefined, + writable: true + } + }) + + var _urlencoded = exports.urlencoded(opts) + var _json = exports.json(opts) + + return function bodyParser (req, res, next) { + _json(req, res, function (err) { + if (err) return next(err) + _urlencoded(req, res, next) + }) + } +} + +/** + * Create a getter for loading a parser. + * @private + */ + +function createParserGetter (name) { + return function get () { + return loadParser(name) + } +} + +/** + * Load a parser module. + * @private + */ + +function loadParser (parserName) { + var parser = parsers[parserName] + + if (parser !== undefined) { + return parser + } + + // this uses a switch for static require analysis + switch (parserName) { + case 'json': + parser = require('./lib/types/json') + break + case 'raw': + parser = require('./lib/types/raw') + break + case 'text': + parser = require('./lib/types/text') + break + case 'urlencoded': + parser = require('./lib/types/urlencoded') + break + } + + // store to prevent invoking require() + return (parsers[parserName] = parser) +} diff --git a/scraper-service/node_modules/body-parser/lib/read.js b/scraper-service/node_modules/body-parser/lib/read.js new file mode 100644 index 0000000..fce6283 --- /dev/null +++ b/scraper-service/node_modules/body-parser/lib/read.js @@ -0,0 +1,205 @@ +/*! + * body-parser + * Copyright(c) 2014-2015 Douglas Christopher Wilson + * MIT Licensed + */ + +'use strict' + +/** + * Module dependencies. + * @private + */ + +var createError = require('http-errors') +var destroy = require('destroy') +var getBody = require('raw-body') +var iconv = require('iconv-lite') +var onFinished = require('on-finished') +var unpipe = require('unpipe') +var zlib = require('zlib') + +/** + * Module exports. + */ + +module.exports = read + +/** + * Read a request into a buffer and parse. + * + * @param {object} req + * @param {object} res + * @param {function} next + * @param {function} parse + * @param {function} debug + * @param {object} options + * @private + */ + +function read (req, res, next, parse, debug, options) { + var length + var opts = options + var stream + + // flag as parsed + req._body = true + + // read options + var encoding = opts.encoding !== null + ? opts.encoding + : null + var verify = opts.verify + + try { + // get the content stream + stream = contentstream(req, debug, opts.inflate) + length = stream.length + stream.length = undefined + } catch (err) { + return next(err) + } + + // set raw-body options + opts.length = length + opts.encoding = verify + ? null + : encoding + + // assert charset is supported + if (opts.encoding === null && encoding !== null && !iconv.encodingExists(encoding)) { + return next(createError(415, 'unsupported charset "' + encoding.toUpperCase() + '"', { + charset: encoding.toLowerCase(), + type: 'charset.unsupported' + })) + } + + // read body + debug('read body') + getBody(stream, opts, function (error, body) { + if (error) { + var _error + + if (error.type === 'encoding.unsupported') { + // echo back charset + _error = createError(415, 'unsupported charset "' + encoding.toUpperCase() + '"', { + charset: encoding.toLowerCase(), + type: 'charset.unsupported' + }) + } else { + // set status code on error + _error = createError(400, error) + } + + // unpipe from stream and destroy + if (stream !== req) { + unpipe(req) + destroy(stream, true) + } + + // read off entire request + dump(req, function onfinished () { + next(createError(400, _error)) + }) + return + } + + // verify + if (verify) { + try { + debug('verify body') + verify(req, res, body, encoding) + } catch (err) { + next(createError(403, err, { + body: body, + type: err.type || 'entity.verify.failed' + })) + return + } + } + + // parse + var str = body + try { + debug('parse body') + str = typeof body !== 'string' && encoding !== null + ? iconv.decode(body, encoding) + : body + req.body = parse(str) + } catch (err) { + next(createError(400, err, { + body: str, + type: err.type || 'entity.parse.failed' + })) + return + } + + next() + }) +} + +/** + * Get the content stream of the request. + * + * @param {object} req + * @param {function} debug + * @param {boolean} [inflate=true] + * @return {object} + * @api private + */ + +function contentstream (req, debug, inflate) { + var encoding = (req.headers['content-encoding'] || 'identity').toLowerCase() + var length = req.headers['content-length'] + var stream + + debug('content-encoding "%s"', encoding) + + if (inflate === false && encoding !== 'identity') { + throw createError(415, 'content encoding unsupported', { + encoding: encoding, + type: 'encoding.unsupported' + }) + } + + switch (encoding) { + case 'deflate': + stream = zlib.createInflate() + debug('inflate body') + req.pipe(stream) + break + case 'gzip': + stream = zlib.createGunzip() + debug('gunzip body') + req.pipe(stream) + break + case 'identity': + stream = req + stream.length = length + break + default: + throw createError(415, 'unsupported content encoding "' + encoding + '"', { + encoding: encoding, + type: 'encoding.unsupported' + }) + } + + return stream +} + +/** + * Dump the contents of a request. + * + * @param {object} req + * @param {function} callback + * @api private + */ + +function dump (req, callback) { + if (onFinished.isFinished(req)) { + callback(null) + } else { + onFinished(req, callback) + req.resume() + } +} diff --git a/scraper-service/node_modules/body-parser/lib/types/json.js b/scraper-service/node_modules/body-parser/lib/types/json.js new file mode 100644 index 0000000..d1f510d --- /dev/null +++ b/scraper-service/node_modules/body-parser/lib/types/json.js @@ -0,0 +1,243 @@ +/*! + * body-parser + * Copyright(c) 2014 Jonathan Ong + * Copyright(c) 2014-2015 Douglas Christopher Wilson + * MIT Licensed + */ + +'use strict' + +/** + * Module dependencies. + * @private + */ + +var bytes = require('bytes') +var contentType = require('content-type') +var createError = require('http-errors') +var debug = require('debug')('body-parser:json') +var read = require('../read') +var typeis = require('type-is') + +/** + * Module exports. + */ + +module.exports = json + +/** + * RegExp to match the first non-space in a string. + * + * Allowed whitespace is defined in RFC 7159: + * + * ws = *( + * %x20 / ; Space + * %x09 / ; Horizontal tab + * %x0A / ; Line feed or New line + * %x0D ) ; Carriage return + */ + +var FIRST_CHAR_REGEXP = /^[\x20\x09\x0a\x0d]*([^\x20\x09\x0a\x0d])/ // eslint-disable-line no-control-regex + +var JSON_SYNTAX_CHAR = '#' +var JSON_SYNTAX_REGEXP = /#+/g + +/** + * Create a middleware to parse JSON bodies. + * + * @param {object} [options] + * @return {function} + * @public + */ + +function json (options) { + var opts = options || {} + + var limit = typeof opts.limit !== 'number' + ? bytes.parse(opts.limit || '100kb') + : opts.limit + var inflate = opts.inflate !== false + var reviver = opts.reviver + var strict = opts.strict !== false + var type = opts.type || 'application/json' + var verify = opts.verify || false + + if (verify !== false && typeof verify !== 'function') { + throw new TypeError('option verify must be function') + } + + // create the appropriate type checking function + var shouldParse = typeof type !== 'function' + ? typeChecker(type) + : type + + function parse (body) { + if (body.length === 0) { + // special-case empty json body, as it's a common client-side mistake + // TODO: maybe make this configurable or part of "strict" option + return {} + } + + if (strict) { + var first = firstchar(body) + + if (first !== '{' && first !== '[') { + debug('strict violation') + throw createStrictSyntaxError(body, first) + } + } + + try { + debug('parse json') + return JSON.parse(body, reviver) + } catch (e) { + throw normalizeJsonSyntaxError(e, { + message: e.message, + stack: e.stack + }) + } + } + + return function jsonParser (req, res, next) { + if (req._body) { + debug('body already parsed') + next() + return + } + + req.body = req.body || {} + + // skip requests without bodies + if (!typeis.hasBody(req)) { + debug('skip empty body') + next() + return + } + + debug('content-type %j', req.headers['content-type']) + + // determine if request should be parsed + if (!shouldParse(req)) { + debug('skip parsing') + next() + return + } + + // assert charset per RFC 7159 sec 8.1 + var charset = getCharset(req) || 'utf-8' + if (charset.slice(0, 4) !== 'utf-') { + debug('invalid charset') + next(createError(415, 'unsupported charset "' + charset.toUpperCase() + '"', { + charset: charset, + type: 'charset.unsupported' + })) + return + } + + // read + read(req, res, next, parse, debug, { + encoding: charset, + inflate: inflate, + limit: limit, + verify: verify + }) + } +} + +/** + * Create strict violation syntax error matching native error. + * + * @param {string} str + * @param {string} char + * @return {Error} + * @private + */ + +function createStrictSyntaxError (str, char) { + var index = str.indexOf(char) + var partial = '' + + if (index !== -1) { + partial = str.substring(0, index) + new Array(str.length - index + 1).join(JSON_SYNTAX_CHAR) + } + + try { + JSON.parse(partial); /* istanbul ignore next */ throw new SyntaxError('strict violation') + } catch (e) { + return normalizeJsonSyntaxError(e, { + message: e.message.replace(JSON_SYNTAX_REGEXP, function (placeholder) { + return str.substring(index, index + placeholder.length) + }), + stack: e.stack + }) + } +} + +/** + * Get the first non-whitespace character in a string. + * + * @param {string} str + * @return {function} + * @private + */ + +function firstchar (str) { + var match = FIRST_CHAR_REGEXP.exec(str) + + return match + ? match[1] + : undefined +} + +/** + * Get the charset of a request. + * + * @param {object} req + * @api private + */ + +function getCharset (req) { + try { + return (contentType.parse(req).parameters.charset || '').toLowerCase() + } catch (e) { + return undefined + } +} + +/** + * Normalize a SyntaxError for JSON.parse. + * + * @param {SyntaxError} error + * @param {object} obj + * @return {SyntaxError} + */ + +function normalizeJsonSyntaxError (error, obj) { + var keys = Object.getOwnPropertyNames(error) + + for (var i = 0; i < keys.length; i++) { + var key = keys[i] + if (key !== 'stack' && key !== 'message') { + delete error[key] + } + } + + // replace stack before message for Node.js 0.10 and below + error.stack = obj.stack.replace(error.message, obj.message) + error.message = obj.message + + return error +} + +/** + * Get the simple type checker. + * + * @param {string} type + * @return {function} + */ + +function typeChecker (type) { + return function checkType (req) { + return Boolean(typeis(req, type)) + } +} diff --git a/scraper-service/node_modules/body-parser/lib/types/raw.js b/scraper-service/node_modules/body-parser/lib/types/raw.js new file mode 100644 index 0000000..f5d1b67 --- /dev/null +++ b/scraper-service/node_modules/body-parser/lib/types/raw.js @@ -0,0 +1,101 @@ +/*! + * body-parser + * Copyright(c) 2014-2015 Douglas Christopher Wilson + * MIT Licensed + */ + +'use strict' + +/** + * Module dependencies. + */ + +var bytes = require('bytes') +var debug = require('debug')('body-parser:raw') +var read = require('../read') +var typeis = require('type-is') + +/** + * Module exports. + */ + +module.exports = raw + +/** + * Create a middleware to parse raw bodies. + * + * @param {object} [options] + * @return {function} + * @api public + */ + +function raw (options) { + var opts = options || {} + + var inflate = opts.inflate !== false + var limit = typeof opts.limit !== 'number' + ? bytes.parse(opts.limit || '100kb') + : opts.limit + var type = opts.type || 'application/octet-stream' + var verify = opts.verify || false + + if (verify !== false && typeof verify !== 'function') { + throw new TypeError('option verify must be function') + } + + // create the appropriate type checking function + var shouldParse = typeof type !== 'function' + ? typeChecker(type) + : type + + function parse (buf) { + return buf + } + + return function rawParser (req, res, next) { + if (req._body) { + debug('body already parsed') + next() + return + } + + req.body = req.body || {} + + // skip requests without bodies + if (!typeis.hasBody(req)) { + debug('skip empty body') + next() + return + } + + debug('content-type %j', req.headers['content-type']) + + // determine if request should be parsed + if (!shouldParse(req)) { + debug('skip parsing') + next() + return + } + + // read + read(req, res, next, parse, debug, { + encoding: null, + inflate: inflate, + limit: limit, + verify: verify + }) + } +} + +/** + * Get the simple type checker. + * + * @param {string} type + * @return {function} + */ + +function typeChecker (type) { + return function checkType (req) { + return Boolean(typeis(req, type)) + } +} diff --git a/scraper-service/node_modules/body-parser/lib/types/text.js b/scraper-service/node_modules/body-parser/lib/types/text.js new file mode 100644 index 0000000..083a009 --- /dev/null +++ b/scraper-service/node_modules/body-parser/lib/types/text.js @@ -0,0 +1,121 @@ +/*! + * body-parser + * Copyright(c) 2014-2015 Douglas Christopher Wilson + * MIT Licensed + */ + +'use strict' + +/** + * Module dependencies. + */ + +var bytes = require('bytes') +var contentType = require('content-type') +var debug = require('debug')('body-parser:text') +var read = require('../read') +var typeis = require('type-is') + +/** + * Module exports. + */ + +module.exports = text + +/** + * Create a middleware to parse text bodies. + * + * @param {object} [options] + * @return {function} + * @api public + */ + +function text (options) { + var opts = options || {} + + var defaultCharset = opts.defaultCharset || 'utf-8' + var inflate = opts.inflate !== false + var limit = typeof opts.limit !== 'number' + ? bytes.parse(opts.limit || '100kb') + : opts.limit + var type = opts.type || 'text/plain' + var verify = opts.verify || false + + if (verify !== false && typeof verify !== 'function') { + throw new TypeError('option verify must be function') + } + + // create the appropriate type checking function + var shouldParse = typeof type !== 'function' + ? typeChecker(type) + : type + + function parse (buf) { + return buf + } + + return function textParser (req, res, next) { + if (req._body) { + debug('body already parsed') + next() + return + } + + req.body = req.body || {} + + // skip requests without bodies + if (!typeis.hasBody(req)) { + debug('skip empty body') + next() + return + } + + debug('content-type %j', req.headers['content-type']) + + // determine if request should be parsed + if (!shouldParse(req)) { + debug('skip parsing') + next() + return + } + + // get charset + var charset = getCharset(req) || defaultCharset + + // read + read(req, res, next, parse, debug, { + encoding: charset, + inflate: inflate, + limit: limit, + verify: verify + }) + } +} + +/** + * Get the charset of a request. + * + * @param {object} req + * @api private + */ + +function getCharset (req) { + try { + return (contentType.parse(req).parameters.charset || '').toLowerCase() + } catch (e) { + return undefined + } +} + +/** + * Get the simple type checker. + * + * @param {string} type + * @return {function} + */ + +function typeChecker (type) { + return function checkType (req) { + return Boolean(typeis(req, type)) + } +} diff --git a/scraper-service/node_modules/body-parser/lib/types/urlencoded.js b/scraper-service/node_modules/body-parser/lib/types/urlencoded.js new file mode 100644 index 0000000..892e346 --- /dev/null +++ b/scraper-service/node_modules/body-parser/lib/types/urlencoded.js @@ -0,0 +1,299 @@ +/*! + * body-parser + * Copyright(c) 2014 Jonathan Ong + * Copyright(c) 2014-2015 Douglas Christopher Wilson + * MIT Licensed + */ + +'use strict' + +/** + * Module dependencies. + * @private + */ + +var bytes = require('bytes') +var contentType = require('content-type') +var createError = require('http-errors') +var debug = require('debug')('body-parser:urlencoded') +var deprecate = require('depd')('body-parser') +var read = require('../read') +var typeis = require('type-is') + +/** + * Module exports. + */ + +module.exports = urlencoded + +/** + * Cache of parser modules. + */ + +var parsers = Object.create(null) + +/** + * Create a middleware to parse urlencoded bodies. + * + * @param {object} [options] + * @return {function} + * @public + */ + +function urlencoded (options) { + var opts = options || {} + + // notice because option default will flip in next major + if (opts.extended === undefined) { + deprecate('undefined extended: provide extended option') + } + + var extended = opts.extended !== false + var inflate = opts.inflate !== false + var limit = typeof opts.limit !== 'number' + ? bytes.parse(opts.limit || '100kb') + : opts.limit + var type = opts.type || 'application/x-www-form-urlencoded' + var verify = opts.verify || false + + if (verify !== false && typeof verify !== 'function') { + throw new TypeError('option verify must be function') + } + + // create the appropriate query parser + var queryparse = extended + ? extendedparser(opts) + : simpleparser(opts) + + // create the appropriate type checking function + var shouldParse = typeof type !== 'function' + ? typeChecker(type) + : type + + function parse (body) { + return body.length + ? queryparse(body) + : {} + } + + return function urlencodedParser (req, res, next) { + if (req._body) { + debug('body already parsed') + next() + return + } + + req.body = req.body || {} + + // skip requests without bodies + if (!typeis.hasBody(req)) { + debug('skip empty body') + next() + return + } + + debug('content-type %j', req.headers['content-type']) + + // determine if request should be parsed + if (!shouldParse(req)) { + debug('skip parsing') + next() + return + } + + // assert charset + var charset = getCharset(req) || 'utf-8' + if (charset !== 'utf-8') { + debug('invalid charset') + next(createError(415, 'unsupported charset "' + charset.toUpperCase() + '"', { + charset: charset, + type: 'charset.unsupported' + })) + return + } + + // read + read(req, res, next, parse, debug, { + debug: debug, + encoding: charset, + inflate: inflate, + limit: limit, + verify: verify + }) + } +} + +/** + * Get the extended query parser. + * + * @param {object} options + */ + +function extendedparser (options) { + var parameterLimit = options.parameterLimit !== undefined + ? options.parameterLimit + : 1000 + var depth = options.depth !== undefined ? options.depth : 32 + var parse = parser('qs') + + if (isNaN(parameterLimit) || parameterLimit < 1) { + throw new TypeError('option parameterLimit must be a positive number') + } + + if (isNaN(depth) || depth < 0) { + throw new TypeError('option depth must be a zero or a positive number') + } + + if (isFinite(parameterLimit)) { + parameterLimit = parameterLimit | 0 + } + + return function queryparse (body) { + var paramCount = parameterCount(body, parameterLimit) + + if (paramCount === undefined) { + debug('too many parameters') + throw createError(413, 'too many parameters', { + type: 'parameters.too.many' + }) + } + + var arrayLimit = Math.max(100, paramCount) + + debug('parse extended urlencoding') + try { + return parse(body, { + allowPrototypes: true, + arrayLimit: arrayLimit, + depth: depth, + strictDepth: true, + parameterLimit: parameterLimit + }) + } catch (err) { + if (err instanceof RangeError) { + throw createError(400, 'The input exceeded the depth', { + type: 'querystring.parse.rangeError' + }) + } else { + throw err + } + } + } +} + +/** + * Get the charset of a request. + * + * @param {object} req + * @api private + */ + +function getCharset (req) { + try { + return (contentType.parse(req).parameters.charset || '').toLowerCase() + } catch (e) { + return undefined + } +} + +/** + * Count the number of parameters, stopping once limit reached + * + * @param {string} body + * @param {number} limit + * @api private + */ + +function parameterCount (body, limit) { + var count = 0 + var index = -1 + + do { + count++ + if (count > limit) { + return undefined + } + index = body.indexOf('&', index + 1) + } while (index !== -1) + + return count +} + +/** + * Get parser for module name dynamically. + * + * @param {string} name + * @return {function} + * @api private + */ + +function parser (name) { + var mod = parsers[name] + + if (mod !== undefined) { + return mod.parse + } + + // this uses a switch for static require analysis + switch (name) { + case 'qs': + mod = require('qs') + break + case 'querystring': + mod = require('querystring') + break + } + + // store to prevent invoking require() + parsers[name] = mod + + return mod.parse +} + +/** + * Get the simple query parser. + * + * @param {object} options + */ + +function simpleparser (options) { + var parameterLimit = options.parameterLimit !== undefined + ? options.parameterLimit + : 1000 + var parse = parser('querystring') + + if (isNaN(parameterLimit) || parameterLimit < 1) { + throw new TypeError('option parameterLimit must be a positive number') + } + + if (isFinite(parameterLimit)) { + parameterLimit = parameterLimit | 0 + } + + return function queryparse (body) { + var paramCount = parameterCount(body, parameterLimit) + + if (paramCount === undefined) { + debug('too many parameters') + throw createError(413, 'too many parameters', { + type: 'parameters.too.many' + }) + } + + debug('parse urlencoding') + return parse(body, undefined, undefined, { maxKeys: parameterLimit }) + } +} + +/** + * Get the simple type checker. + * + * @param {string} type + * @return {function} + */ + +function typeChecker (type) { + return function checkType (req) { + return Boolean(typeis(req, type)) + } +} diff --git a/scraper-service/node_modules/body-parser/package.json b/scraper-service/node_modules/body-parser/package.json new file mode 100644 index 0000000..a1d16f8 --- /dev/null +++ b/scraper-service/node_modules/body-parser/package.json @@ -0,0 +1,55 @@ +{ + "name": "body-parser", + "description": "Node.js body parsing middleware", + "version": "1.20.5", + "contributors": [ + "Douglas Christopher Wilson ", + "Jonathan Ong (http://jongleberry.com)" + ], + "license": "MIT", + "repository": "expressjs/body-parser", + "dependencies": { + "bytes": "~3.1.2", + "content-type": "~1.0.5", + "debug": "2.6.9", + "depd": "2.0.0", + "destroy": "~1.2.0", + "http-errors": "~2.0.1", + "iconv-lite": "~0.4.24", + "on-finished": "~2.4.1", + "qs": "~6.15.1", + "raw-body": "~2.5.3", + "type-is": "~1.6.18", + "unpipe": "~1.0.0" + }, + "devDependencies": { + "eslint": "8.34.0", + "eslint-config-standard": "14.1.1", + "eslint-plugin-import": "2.27.5", + "eslint-plugin-markdown": "3.0.0", + "eslint-plugin-node": "11.1.0", + "eslint-plugin-promise": "6.1.1", + "eslint-plugin-standard": "4.1.0", + "methods": "1.1.2", + "mocha": "10.2.0", + "nyc": "15.1.0", + "safe-buffer": "5.2.1", + "supertest": "6.3.3" + }, + "files": [ + "lib/", + "LICENSE", + "HISTORY.md", + "index.js" + ], + "engines": { + "node": ">= 0.8", + "npm": "1.2.8000 || >= 1.4.16" + }, + "scripts": { + "lint": "eslint .", + "test": "mocha --require test/support/env --reporter spec --check-leaks --bail test/", + "test-ci": "nyc --reporter=lcov --reporter=text npm test", + "test-cov": "nyc --reporter=html --reporter=text npm test" + } +} diff --git a/scraper-service/node_modules/bytes/History.md b/scraper-service/node_modules/bytes/History.md new file mode 100644 index 0000000..d60ce0e --- /dev/null +++ b/scraper-service/node_modules/bytes/History.md @@ -0,0 +1,97 @@ +3.1.2 / 2022-01-27 +================== + + * Fix return value for un-parsable strings + +3.1.1 / 2021-11-15 +================== + + * Fix "thousandsSeparator" incorrecting formatting fractional part + +3.1.0 / 2019-01-22 +================== + + * Add petabyte (`pb`) support + +3.0.0 / 2017-08-31 +================== + + * Change "kB" to "KB" in format output + * Remove support for Node.js 0.6 + * Remove support for ComponentJS + +2.5.0 / 2017-03-24 +================== + + * Add option "unit" + +2.4.0 / 2016-06-01 +================== + + * Add option "unitSeparator" + +2.3.0 / 2016-02-15 +================== + + * Drop partial bytes on all parsed units + * Fix non-finite numbers to `.format` to return `null` + * Fix parsing byte string that looks like hex + * perf: hoist regular expressions + +2.2.0 / 2015-11-13 +================== + + * add option "decimalPlaces" + * add option "fixedDecimals" + +2.1.0 / 2015-05-21 +================== + + * add `.format` export + * add `.parse` export + +2.0.2 / 2015-05-20 +================== + + * remove map recreation + * remove unnecessary object construction + +2.0.1 / 2015-05-07 +================== + + * fix browserify require + * remove node.extend dependency + +2.0.0 / 2015-04-12 +================== + + * add option "case" + * add option "thousandsSeparator" + * return "null" on invalid parse input + * support proper round-trip: bytes(bytes(num)) === num + * units no longer case sensitive when parsing + +1.0.0 / 2014-05-05 +================== + + * add negative support. fixes #6 + +0.3.0 / 2014-03-19 +================== + + * added terabyte support + +0.2.1 / 2013-04-01 +================== + + * add .component + +0.2.0 / 2012-10-28 +================== + + * bytes(200).should.eql('200b') + +0.1.0 / 2012-07-04 +================== + + * add bytes to string conversion [yields] diff --git a/scraper-service/node_modules/bytes/LICENSE b/scraper-service/node_modules/bytes/LICENSE new file mode 100644 index 0000000..63e95a9 --- /dev/null +++ b/scraper-service/node_modules/bytes/LICENSE @@ -0,0 +1,23 @@ +(The MIT License) + +Copyright (c) 2012-2014 TJ Holowaychuk +Copyright (c) 2015 Jed Watson + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +'Software'), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/scraper-service/node_modules/bytes/Readme.md b/scraper-service/node_modules/bytes/Readme.md new file mode 100644 index 0000000..5790e23 --- /dev/null +++ b/scraper-service/node_modules/bytes/Readme.md @@ -0,0 +1,152 @@ +# Bytes utility + +[![NPM Version][npm-image]][npm-url] +[![NPM Downloads][downloads-image]][downloads-url] +[![Build Status][ci-image]][ci-url] +[![Test Coverage][coveralls-image]][coveralls-url] + +Utility to parse a string bytes (ex: `1TB`) to bytes (`1099511627776`) and vice-versa. + +## Installation + +This is a [Node.js](https://nodejs.org/en/) module available through the +[npm registry](https://www.npmjs.com/). Installation is done using the +[`npm install` command](https://docs.npmjs.com/getting-started/installing-npm-packages-locally): + +```bash +$ npm install bytes +``` + +## Usage + +```js +var bytes = require('bytes'); +``` + +#### bytes(number|string value, [options]): number|string|null + +Default export function. Delegates to either `bytes.format` or `bytes.parse` based on the type of `value`. + +**Arguments** + +| Name | Type | Description | +|---------|----------|--------------------| +| value | `number`|`string` | Number value to format or string value to parse | +| options | `Object` | Conversion options for `format` | + +**Returns** + +| Name | Type | Description | +|---------|------------------|-------------------------------------------------| +| results | `string`|`number`|`null` | Return null upon error. Numeric value in bytes, or string value otherwise. | + +**Example** + +```js +bytes(1024); +// output: '1KB' + +bytes('1KB'); +// output: 1024 +``` + +#### bytes.format(number value, [options]): string|null + +Format the given value in bytes into a string. If the value is negative, it is kept as such. If it is a float, it is + rounded. + +**Arguments** + +| Name | Type | Description | +|---------|----------|--------------------| +| value | `number` | Value in bytes | +| options | `Object` | Conversion options | + +**Options** + +| Property | Type | Description | +|-------------------|--------|-----------------------------------------------------------------------------------------| +| decimalPlaces | `number`|`null` | Maximum number of decimal places to include in output. Default value to `2`. | +| fixedDecimals | `boolean`|`null` | Whether to always display the maximum number of decimal places. Default value to `false` | +| thousandsSeparator | `string`|`null` | Example of values: `' '`, `','` and `'.'`... Default value to `''`. | +| unit | `string`|`null` | The unit in which the result will be returned (B/KB/MB/GB/TB). Default value to `''` (which means auto detect). | +| unitSeparator | `string`|`null` | Separator to use between number and unit. Default value to `''`. | + +**Returns** + +| Name | Type | Description | +|---------|------------------|-------------------------------------------------| +| results | `string`|`null` | Return null upon error. String value otherwise. | + +**Example** + +```js +bytes.format(1024); +// output: '1KB' + +bytes.format(1000); +// output: '1000B' + +bytes.format(1000, {thousandsSeparator: ' '}); +// output: '1 000B' + +bytes.format(1024 * 1.7, {decimalPlaces: 0}); +// output: '2KB' + +bytes.format(1024, {unitSeparator: ' '}); +// output: '1 KB' +``` + +#### bytes.parse(string|number value): number|null + +Parse the string value into an integer in bytes. If no unit is given, or `value` +is a number, it is assumed the value is in bytes. + +Supported units and abbreviations are as follows and are case-insensitive: + + * `b` for bytes + * `kb` for kilobytes + * `mb` for megabytes + * `gb` for gigabytes + * `tb` for terabytes + * `pb` for petabytes + +The units are in powers of two, not ten. This means 1kb = 1024b according to this parser. + +**Arguments** + +| Name | Type | Description | +|---------------|--------|--------------------| +| value | `string`|`number` | String to parse, or number in bytes. | + +**Returns** + +| Name | Type | Description | +|---------|-------------|-------------------------| +| results | `number`|`null` | Return null upon error. Value in bytes otherwise. | + +**Example** + +```js +bytes.parse('1KB'); +// output: 1024 + +bytes.parse('1024'); +// output: 1024 + +bytes.parse(1024); +// output: 1024 +``` + +## License + +[MIT](LICENSE) + +[ci-image]: https://badgen.net/github/checks/visionmedia/bytes.js/master?label=ci +[ci-url]: https://github.com/visionmedia/bytes.js/actions?query=workflow%3Aci +[coveralls-image]: https://badgen.net/coveralls/c/github/visionmedia/bytes.js/master +[coveralls-url]: https://coveralls.io/r/visionmedia/bytes.js?branch=master +[downloads-image]: https://badgen.net/npm/dm/bytes +[downloads-url]: https://npmjs.org/package/bytes +[npm-image]: https://badgen.net/npm/v/bytes +[npm-url]: https://npmjs.org/package/bytes diff --git a/scraper-service/node_modules/bytes/index.js b/scraper-service/node_modules/bytes/index.js new file mode 100644 index 0000000..6f2d0f8 --- /dev/null +++ b/scraper-service/node_modules/bytes/index.js @@ -0,0 +1,170 @@ +/*! + * bytes + * Copyright(c) 2012-2014 TJ Holowaychuk + * Copyright(c) 2015 Jed Watson + * MIT Licensed + */ + +'use strict'; + +/** + * Module exports. + * @public + */ + +module.exports = bytes; +module.exports.format = format; +module.exports.parse = parse; + +/** + * Module variables. + * @private + */ + +var formatThousandsRegExp = /\B(?=(\d{3})+(?!\d))/g; + +var formatDecimalsRegExp = /(?:\.0*|(\.[^0]+)0+)$/; + +var map = { + b: 1, + kb: 1 << 10, + mb: 1 << 20, + gb: 1 << 30, + tb: Math.pow(1024, 4), + pb: Math.pow(1024, 5), +}; + +var parseRegExp = /^((-|\+)?(\d+(?:\.\d+)?)) *(kb|mb|gb|tb|pb)$/i; + +/** + * Convert the given value in bytes into a string or parse to string to an integer in bytes. + * + * @param {string|number} value + * @param {{ + * case: [string], + * decimalPlaces: [number] + * fixedDecimals: [boolean] + * thousandsSeparator: [string] + * unitSeparator: [string] + * }} [options] bytes options. + * + * @returns {string|number|null} + */ + +function bytes(value, options) { + if (typeof value === 'string') { + return parse(value); + } + + if (typeof value === 'number') { + return format(value, options); + } + + return null; +} + +/** + * Format the given value in bytes into a string. + * + * If the value is negative, it is kept as such. If it is a float, + * it is rounded. + * + * @param {number} value + * @param {object} [options] + * @param {number} [options.decimalPlaces=2] + * @param {number} [options.fixedDecimals=false] + * @param {string} [options.thousandsSeparator=] + * @param {string} [options.unit=] + * @param {string} [options.unitSeparator=] + * + * @returns {string|null} + * @public + */ + +function format(value, options) { + if (!Number.isFinite(value)) { + return null; + } + + var mag = Math.abs(value); + var thousandsSeparator = (options && options.thousandsSeparator) || ''; + var unitSeparator = (options && options.unitSeparator) || ''; + var decimalPlaces = (options && options.decimalPlaces !== undefined) ? options.decimalPlaces : 2; + var fixedDecimals = Boolean(options && options.fixedDecimals); + var unit = (options && options.unit) || ''; + + if (!unit || !map[unit.toLowerCase()]) { + if (mag >= map.pb) { + unit = 'PB'; + } else if (mag >= map.tb) { + unit = 'TB'; + } else if (mag >= map.gb) { + unit = 'GB'; + } else if (mag >= map.mb) { + unit = 'MB'; + } else if (mag >= map.kb) { + unit = 'KB'; + } else { + unit = 'B'; + } + } + + var val = value / map[unit.toLowerCase()]; + var str = val.toFixed(decimalPlaces); + + if (!fixedDecimals) { + str = str.replace(formatDecimalsRegExp, '$1'); + } + + if (thousandsSeparator) { + str = str.split('.').map(function (s, i) { + return i === 0 + ? s.replace(formatThousandsRegExp, thousandsSeparator) + : s + }).join('.'); + } + + return str + unitSeparator + unit; +} + +/** + * Parse the string value into an integer in bytes. + * + * If no unit is given, it is assumed the value is in bytes. + * + * @param {number|string} val + * + * @returns {number|null} + * @public + */ + +function parse(val) { + if (typeof val === 'number' && !isNaN(val)) { + return val; + } + + if (typeof val !== 'string') { + return null; + } + + // Test if the string passed is valid + var results = parseRegExp.exec(val); + var floatValue; + var unit = 'b'; + + if (!results) { + // Nothing could be extracted from the given string + floatValue = parseInt(val, 10); + unit = 'b' + } else { + // Retrieve the value and the unit + floatValue = parseFloat(results[1]); + unit = results[4].toLowerCase(); + } + + if (isNaN(floatValue)) { + return null; + } + + return Math.floor(map[unit] * floatValue); +} diff --git a/scraper-service/node_modules/bytes/package.json b/scraper-service/node_modules/bytes/package.json new file mode 100644 index 0000000..f2b6a8b --- /dev/null +++ b/scraper-service/node_modules/bytes/package.json @@ -0,0 +1,42 @@ +{ + "name": "bytes", + "description": "Utility to parse a string bytes to bytes and vice-versa", + "version": "3.1.2", + "author": "TJ Holowaychuk (http://tjholowaychuk.com)", + "contributors": [ + "Jed Watson ", + "Théo FIDRY " + ], + "license": "MIT", + "keywords": [ + "byte", + "bytes", + "utility", + "parse", + "parser", + "convert", + "converter" + ], + "repository": "visionmedia/bytes.js", + "devDependencies": { + "eslint": "7.32.0", + "eslint-plugin-markdown": "2.2.1", + "mocha": "9.2.0", + "nyc": "15.1.0" + }, + "files": [ + "History.md", + "LICENSE", + "Readme.md", + "index.js" + ], + "engines": { + "node": ">= 0.8" + }, + "scripts": { + "lint": "eslint .", + "test": "mocha --check-leaks --reporter spec", + "test-ci": "nyc --reporter=lcov --reporter=text npm test", + "test-cov": "nyc --reporter=html --reporter=text npm test" + } +} diff --git a/scraper-service/node_modules/call-bind-apply-helpers/.eslintrc b/scraper-service/node_modules/call-bind-apply-helpers/.eslintrc new file mode 100644 index 0000000..201e859 --- /dev/null +++ b/scraper-service/node_modules/call-bind-apply-helpers/.eslintrc @@ -0,0 +1,17 @@ +{ + "root": true, + + "extends": "@ljharb", + + "rules": { + "func-name-matching": 0, + "id-length": 0, + "new-cap": [2, { + "capIsNewExceptions": [ + "GetIntrinsic", + ], + }], + "no-extra-parens": 0, + "no-magic-numbers": 0, + }, +} diff --git a/scraper-service/node_modules/call-bind-apply-helpers/.github/FUNDING.yml b/scraper-service/node_modules/call-bind-apply-helpers/.github/FUNDING.yml new file mode 100644 index 0000000..0011e9d --- /dev/null +++ b/scraper-service/node_modules/call-bind-apply-helpers/.github/FUNDING.yml @@ -0,0 +1,12 @@ +# These are supported funding model platforms + +github: [ljharb] +patreon: # Replace with a single Patreon username +open_collective: # Replace with a single Open Collective username +ko_fi: # Replace with a single Ko-fi username +tidelift: npm/call-bind-apply-helpers +community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry +liberapay: # Replace with a single Liberapay username +issuehunt: # Replace with a single IssueHunt username +otechie: # Replace with a single Otechie username +custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2'] diff --git a/scraper-service/node_modules/call-bind-apply-helpers/.nycrc b/scraper-service/node_modules/call-bind-apply-helpers/.nycrc new file mode 100644 index 0000000..bdd626c --- /dev/null +++ b/scraper-service/node_modules/call-bind-apply-helpers/.nycrc @@ -0,0 +1,9 @@ +{ + "all": true, + "check-coverage": false, + "reporter": ["text-summary", "text", "html", "json"], + "exclude": [ + "coverage", + "test" + ] +} diff --git a/scraper-service/node_modules/call-bind-apply-helpers/CHANGELOG.md b/scraper-service/node_modules/call-bind-apply-helpers/CHANGELOG.md new file mode 100644 index 0000000..2484942 --- /dev/null +++ b/scraper-service/node_modules/call-bind-apply-helpers/CHANGELOG.md @@ -0,0 +1,30 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [v1.0.2](https://github.com/ljharb/call-bind-apply-helpers/compare/v1.0.1...v1.0.2) - 2025-02-12 + +### Commits + +- [types] improve inferred types [`e6f9586`](https://github.com/ljharb/call-bind-apply-helpers/commit/e6f95860a3c72879cb861a858cdfb8138fbedec1) +- [Dev Deps] update `@arethetypeswrong/cli`, `@ljharb/tsconfig`, `@types/tape`, `es-value-fixtures`, `for-each`, `has-strict-mode`, `object-inspect` [`e43d540`](https://github.com/ljharb/call-bind-apply-helpers/commit/e43d5409f97543bfbb11f345d47d8ce4e066d8c1) + +## [v1.0.1](https://github.com/ljharb/call-bind-apply-helpers/compare/v1.0.0...v1.0.1) - 2024-12-08 + +### Commits + +- [types] `reflectApply`: fix types [`4efc396`](https://github.com/ljharb/call-bind-apply-helpers/commit/4efc3965351a4f02cc55e836fa391d3d11ef2ef8) +- [Fix] `reflectApply`: oops, Reflect is not a function [`83cc739`](https://github.com/ljharb/call-bind-apply-helpers/commit/83cc7395de6b79b7730bdf092f1436f0b1263c75) +- [Dev Deps] update `@arethetypeswrong/cli` [`80bd5d3`](https://github.com/ljharb/call-bind-apply-helpers/commit/80bd5d3ae58b4f6b6995ce439dd5a1bcb178a940) + +## v1.0.0 - 2024-12-05 + +### Commits + +- Initial implementation, tests, readme [`7879629`](https://github.com/ljharb/call-bind-apply-helpers/commit/78796290f9b7430c9934d6f33d94ae9bc89fce04) +- Initial commit [`3f1dc16`](https://github.com/ljharb/call-bind-apply-helpers/commit/3f1dc164afc43285631b114a5f9dd9137b2b952f) +- npm init [`081df04`](https://github.com/ljharb/call-bind-apply-helpers/commit/081df048c312fcee400922026f6e97281200a603) +- Only apps should have lockfiles [`5b9ca0f`](https://github.com/ljharb/call-bind-apply-helpers/commit/5b9ca0fe8101ebfaf309c549caac4e0a017ed930) diff --git a/scraper-service/node_modules/call-bind-apply-helpers/LICENSE b/scraper-service/node_modules/call-bind-apply-helpers/LICENSE new file mode 100644 index 0000000..f82f389 --- /dev/null +++ b/scraper-service/node_modules/call-bind-apply-helpers/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2024 Jordan Harband + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/scraper-service/node_modules/call-bind-apply-helpers/README.md b/scraper-service/node_modules/call-bind-apply-helpers/README.md new file mode 100644 index 0000000..8fc0dae --- /dev/null +++ b/scraper-service/node_modules/call-bind-apply-helpers/README.md @@ -0,0 +1,62 @@ +# call-bind-apply-helpers [![Version Badge][npm-version-svg]][package-url] + +[![github actions][actions-image]][actions-url] +[![coverage][codecov-image]][codecov-url] +[![dependency status][deps-svg]][deps-url] +[![dev dependency status][dev-deps-svg]][dev-deps-url] +[![License][license-image]][license-url] +[![Downloads][downloads-image]][downloads-url] + +[![npm badge][npm-badge-png]][package-url] + +Helper functions around Function call/apply/bind, for use in `call-bind`. + +The only packages that should likely ever use this package directly are `call-bind` and `get-intrinsic`. +Please use `call-bind` unless you have a very good reason not to. + +## Getting started + +```sh +npm install --save call-bind-apply-helpers +``` + +## Usage/Examples + +```js +const assert = require('assert'); +const callBindBasic = require('call-bind-apply-helpers'); + +function f(a, b) { + assert.equal(this, 1); + assert.equal(a, 2); + assert.equal(b, 3); + assert.equal(arguments.length, 2); +} + +const fBound = callBindBasic([f, 1]); + +delete Function.prototype.call; +delete Function.prototype.bind; + +fBound(2, 3); +``` + +## Tests + +Clone the repo, `npm install`, and run `npm test` + +[package-url]: https://npmjs.org/package/call-bind-apply-helpers +[npm-version-svg]: https://versionbadg.es/ljharb/call-bind-apply-helpers.svg +[deps-svg]: https://david-dm.org/ljharb/call-bind-apply-helpers.svg +[deps-url]: https://david-dm.org/ljharb/call-bind-apply-helpers +[dev-deps-svg]: https://david-dm.org/ljharb/call-bind-apply-helpers/dev-status.svg +[dev-deps-url]: https://david-dm.org/ljharb/call-bind-apply-helpers#info=devDependencies +[npm-badge-png]: https://nodei.co/npm/call-bind-apply-helpers.png?downloads=true&stars=true +[license-image]: https://img.shields.io/npm/l/call-bind-apply-helpers.svg +[license-url]: LICENSE +[downloads-image]: https://img.shields.io/npm/dm/call-bind-apply-helpers.svg +[downloads-url]: https://npm-stat.com/charts.html?package=call-bind-apply-helpers +[codecov-image]: https://codecov.io/gh/ljharb/call-bind-apply-helpers/branch/main/graphs/badge.svg +[codecov-url]: https://app.codecov.io/gh/ljharb/call-bind-apply-helpers/ +[actions-image]: https://img.shields.io/endpoint?url=https://github-actions-badge-u3jn4tfpocch.runkit.sh/ljharb/call-bind-apply-helpers +[actions-url]: https://github.com/ljharb/call-bind-apply-helpers/actions diff --git a/scraper-service/node_modules/call-bind-apply-helpers/actualApply.d.ts b/scraper-service/node_modules/call-bind-apply-helpers/actualApply.d.ts new file mode 100644 index 0000000..b87286a --- /dev/null +++ b/scraper-service/node_modules/call-bind-apply-helpers/actualApply.d.ts @@ -0,0 +1 @@ +export = Reflect.apply; \ No newline at end of file diff --git a/scraper-service/node_modules/call-bind-apply-helpers/actualApply.js b/scraper-service/node_modules/call-bind-apply-helpers/actualApply.js new file mode 100644 index 0000000..ffa5135 --- /dev/null +++ b/scraper-service/node_modules/call-bind-apply-helpers/actualApply.js @@ -0,0 +1,10 @@ +'use strict'; + +var bind = require('function-bind'); + +var $apply = require('./functionApply'); +var $call = require('./functionCall'); +var $reflectApply = require('./reflectApply'); + +/** @type {import('./actualApply')} */ +module.exports = $reflectApply || bind.call($call, $apply); diff --git a/scraper-service/node_modules/call-bind-apply-helpers/applyBind.d.ts b/scraper-service/node_modules/call-bind-apply-helpers/applyBind.d.ts new file mode 100644 index 0000000..d176c1a --- /dev/null +++ b/scraper-service/node_modules/call-bind-apply-helpers/applyBind.d.ts @@ -0,0 +1,19 @@ +import actualApply from './actualApply'; + +type TupleSplitHead = T['length'] extends N + ? T + : T extends [...infer R, any] + ? TupleSplitHead + : never + +type TupleSplitTail = O['length'] extends N + ? T + : T extends [infer F, ...infer R] + ? TupleSplitTail<[...R], N, [...O, F]> + : never + +type TupleSplit = [TupleSplitHead, TupleSplitTail] + +declare function applyBind(...args: TupleSplit, 2>[1]): ReturnType; + +export = applyBind; \ No newline at end of file diff --git a/scraper-service/node_modules/call-bind-apply-helpers/applyBind.js b/scraper-service/node_modules/call-bind-apply-helpers/applyBind.js new file mode 100644 index 0000000..d2b7723 --- /dev/null +++ b/scraper-service/node_modules/call-bind-apply-helpers/applyBind.js @@ -0,0 +1,10 @@ +'use strict'; + +var bind = require('function-bind'); +var $apply = require('./functionApply'); +var actualApply = require('./actualApply'); + +/** @type {import('./applyBind')} */ +module.exports = function applyBind() { + return actualApply(bind, $apply, arguments); +}; diff --git a/scraper-service/node_modules/call-bind-apply-helpers/functionApply.d.ts b/scraper-service/node_modules/call-bind-apply-helpers/functionApply.d.ts new file mode 100644 index 0000000..1f6e11b --- /dev/null +++ b/scraper-service/node_modules/call-bind-apply-helpers/functionApply.d.ts @@ -0,0 +1 @@ +export = Function.prototype.apply; \ No newline at end of file diff --git a/scraper-service/node_modules/call-bind-apply-helpers/functionApply.js b/scraper-service/node_modules/call-bind-apply-helpers/functionApply.js new file mode 100644 index 0000000..c71df9c --- /dev/null +++ b/scraper-service/node_modules/call-bind-apply-helpers/functionApply.js @@ -0,0 +1,4 @@ +'use strict'; + +/** @type {import('./functionApply')} */ +module.exports = Function.prototype.apply; diff --git a/scraper-service/node_modules/call-bind-apply-helpers/functionCall.d.ts b/scraper-service/node_modules/call-bind-apply-helpers/functionCall.d.ts new file mode 100644 index 0000000..15e93df --- /dev/null +++ b/scraper-service/node_modules/call-bind-apply-helpers/functionCall.d.ts @@ -0,0 +1 @@ +export = Function.prototype.call; \ No newline at end of file diff --git a/scraper-service/node_modules/call-bind-apply-helpers/functionCall.js b/scraper-service/node_modules/call-bind-apply-helpers/functionCall.js new file mode 100644 index 0000000..7a8d873 --- /dev/null +++ b/scraper-service/node_modules/call-bind-apply-helpers/functionCall.js @@ -0,0 +1,4 @@ +'use strict'; + +/** @type {import('./functionCall')} */ +module.exports = Function.prototype.call; diff --git a/scraper-service/node_modules/call-bind-apply-helpers/index.d.ts b/scraper-service/node_modules/call-bind-apply-helpers/index.d.ts new file mode 100644 index 0000000..541516b --- /dev/null +++ b/scraper-service/node_modules/call-bind-apply-helpers/index.d.ts @@ -0,0 +1,64 @@ +type RemoveFromTuple< + Tuple extends readonly unknown[], + RemoveCount extends number, + Index extends 1[] = [] +> = Index["length"] extends RemoveCount + ? Tuple + : Tuple extends [infer First, ...infer Rest] + ? RemoveFromTuple + : Tuple; + +type ConcatTuples< + Prefix extends readonly unknown[], + Suffix extends readonly unknown[] +> = [...Prefix, ...Suffix]; + +type ExtractFunctionParams = T extends (this: infer TThis, ...args: infer P extends readonly unknown[]) => infer R + ? { thisArg: TThis; params: P; returnType: R } + : never; + +type BindFunction< + T extends (this: any, ...args: any[]) => any, + TThis, + TBoundArgs extends readonly unknown[], + ReceiverBound extends boolean +> = ExtractFunctionParams extends { + thisArg: infer OrigThis; + params: infer P extends readonly unknown[]; + returnType: infer R; +} + ? ReceiverBound extends true + ? (...args: RemoveFromTuple>) => R extends [OrigThis, ...infer Rest] + ? [TThis, ...Rest] // Replace `this` with `thisArg` + : R + : >>( + thisArg: U, + ...args: RemainingArgs + ) => R extends [OrigThis, ...infer Rest] + ? [U, ...ConcatTuples] // Preserve bound args in return type + : R + : never; + +declare function callBind< + const T extends (this: any, ...args: any[]) => any, + Extracted extends ExtractFunctionParams, + const TBoundArgs extends Partial & readonly unknown[], + const TThis extends Extracted["thisArg"] +>( + args: [fn: T, thisArg: TThis, ...boundArgs: TBoundArgs] +): BindFunction; + +declare function callBind< + const T extends (this: any, ...args: any[]) => any, + Extracted extends ExtractFunctionParams, + const TBoundArgs extends Partial & readonly unknown[] +>( + args: [fn: T, ...boundArgs: TBoundArgs] +): BindFunction; + +declare function callBind( + args: [fn: Exclude, ...rest: TArgs] +): never; + +// export as namespace callBind; +export = callBind; diff --git a/scraper-service/node_modules/call-bind-apply-helpers/index.js b/scraper-service/node_modules/call-bind-apply-helpers/index.js new file mode 100644 index 0000000..2f6dab4 --- /dev/null +++ b/scraper-service/node_modules/call-bind-apply-helpers/index.js @@ -0,0 +1,15 @@ +'use strict'; + +var bind = require('function-bind'); +var $TypeError = require('es-errors/type'); + +var $call = require('./functionCall'); +var $actualApply = require('./actualApply'); + +/** @type {(args: [Function, thisArg?: unknown, ...args: unknown[]]) => Function} TODO FIXME, find a way to use import('.') */ +module.exports = function callBindBasic(args) { + if (args.length < 1 || typeof args[0] !== 'function') { + throw new $TypeError('a function is required'); + } + return $actualApply(bind, $call, args); +}; diff --git a/scraper-service/node_modules/call-bind-apply-helpers/package.json b/scraper-service/node_modules/call-bind-apply-helpers/package.json new file mode 100644 index 0000000..923b8be --- /dev/null +++ b/scraper-service/node_modules/call-bind-apply-helpers/package.json @@ -0,0 +1,85 @@ +{ + "name": "call-bind-apply-helpers", + "version": "1.0.2", + "description": "Helper functions around Function call/apply/bind, for use in `call-bind`", + "main": "index.js", + "exports": { + ".": "./index.js", + "./actualApply": "./actualApply.js", + "./applyBind": "./applyBind.js", + "./functionApply": "./functionApply.js", + "./functionCall": "./functionCall.js", + "./reflectApply": "./reflectApply.js", + "./package.json": "./package.json" + }, + "scripts": { + "prepack": "npmignore --auto --commentLines=auto", + "prepublish": "not-in-publish || npm run prepublishOnly", + "prepublishOnly": "safe-publish-latest", + "prelint": "evalmd README.md", + "lint": "eslint --ext=.js,.mjs .", + "postlint": "tsc -p . && attw -P", + "pretest": "npm run lint", + "tests-only": "nyc tape 'test/**/*.js'", + "test": "npm run tests-only", + "posttest": "npx npm@'>=10.2' audit --production", + "version": "auto-changelog && git add CHANGELOG.md", + "postversion": "auto-changelog && git add CHANGELOG.md && git commit --no-edit --amend && git tag -f \"v$(node -e \"console.log(require('./package.json').version)\")\"" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/ljharb/call-bind-apply-helpers.git" + }, + "author": "Jordan Harband ", + "license": "MIT", + "bugs": { + "url": "https://github.com/ljharb/call-bind-apply-helpers/issues" + }, + "homepage": "https://github.com/ljharb/call-bind-apply-helpers#readme", + "dependencies": { + "es-errors": "^1.3.0", + "function-bind": "^1.1.2" + }, + "devDependencies": { + "@arethetypeswrong/cli": "^0.17.3", + "@ljharb/eslint-config": "^21.1.1", + "@ljharb/tsconfig": "^0.2.3", + "@types/for-each": "^0.3.3", + "@types/function-bind": "^1.1.10", + "@types/object-inspect": "^1.13.0", + "@types/tape": "^5.8.1", + "auto-changelog": "^2.5.0", + "encoding": "^0.1.13", + "es-value-fixtures": "^1.7.1", + "eslint": "=8.8.0", + "evalmd": "^0.0.19", + "for-each": "^0.3.5", + "has-strict-mode": "^1.1.0", + "in-publish": "^2.0.1", + "npmignore": "^0.3.1", + "nyc": "^10.3.2", + "object-inspect": "^1.13.4", + "safe-publish-latest": "^2.0.0", + "tape": "^5.9.0", + "typescript": "next" + }, + "testling": { + "files": "test/index.js" + }, + "auto-changelog": { + "output": "CHANGELOG.md", + "template": "keepachangelog", + "unreleased": false, + "commitLimit": false, + "backfillLimit": false, + "hideCredit": true + }, + "publishConfig": { + "ignore": [ + ".github/workflows" + ] + }, + "engines": { + "node": ">= 0.4" + } +} diff --git a/scraper-service/node_modules/call-bind-apply-helpers/reflectApply.d.ts b/scraper-service/node_modules/call-bind-apply-helpers/reflectApply.d.ts new file mode 100644 index 0000000..6b2ae76 --- /dev/null +++ b/scraper-service/node_modules/call-bind-apply-helpers/reflectApply.d.ts @@ -0,0 +1,3 @@ +declare const reflectApply: false | typeof Reflect.apply; + +export = reflectApply; diff --git a/scraper-service/node_modules/call-bind-apply-helpers/reflectApply.js b/scraper-service/node_modules/call-bind-apply-helpers/reflectApply.js new file mode 100644 index 0000000..3d03caa --- /dev/null +++ b/scraper-service/node_modules/call-bind-apply-helpers/reflectApply.js @@ -0,0 +1,4 @@ +'use strict'; + +/** @type {import('./reflectApply')} */ +module.exports = typeof Reflect !== 'undefined' && Reflect && Reflect.apply; diff --git a/scraper-service/node_modules/call-bind-apply-helpers/test/index.js b/scraper-service/node_modules/call-bind-apply-helpers/test/index.js new file mode 100644 index 0000000..1cdc89e --- /dev/null +++ b/scraper-service/node_modules/call-bind-apply-helpers/test/index.js @@ -0,0 +1,63 @@ +'use strict'; + +var callBind = require('../'); +var hasStrictMode = require('has-strict-mode')(); +var forEach = require('for-each'); +var inspect = require('object-inspect'); +var v = require('es-value-fixtures'); + +var test = require('tape'); + +test('callBindBasic', function (t) { + forEach(v.nonFunctions, function (nonFunction) { + t['throws']( + // @ts-expect-error + function () { callBind([nonFunction]); }, + TypeError, + inspect(nonFunction) + ' is not a function' + ); + }); + + var sentinel = { sentinel: true }; + /** @type {(this: T, a: A, b: B) => [T | undefined, A, B]} */ + var func = function (a, b) { + // eslint-disable-next-line no-invalid-this + return [!hasStrictMode && this === global ? undefined : this, a, b]; + }; + t.equal(func.length, 2, 'original function length is 2'); + + /** type {(thisArg: unknown, a: number, b: number) => [unknown, number, number]} */ + var bound = callBind([func]); + /** type {((a: number, b: number) => [typeof sentinel, typeof a, typeof b])} */ + var boundR = callBind([func, sentinel]); + /** type {((b: number) => [typeof sentinel, number, typeof b])} */ + var boundArg = callBind([func, sentinel, /** @type {const} */ (1)]); + + // @ts-expect-error + t.deepEqual(bound(), [undefined, undefined, undefined], 'bound func with no args'); + + // @ts-expect-error + t.deepEqual(func(), [undefined, undefined, undefined], 'unbound func with too few args'); + // @ts-expect-error + t.deepEqual(bound(1, 2), [hasStrictMode ? 1 : Object(1), 2, undefined], 'bound func too few args'); + // @ts-expect-error + t.deepEqual(boundR(), [sentinel, undefined, undefined], 'bound func with receiver, with too few args'); + // @ts-expect-error + t.deepEqual(boundArg(), [sentinel, 1, undefined], 'bound func with receiver and arg, with too few args'); + + t.deepEqual(func(1, 2), [undefined, 1, 2], 'unbound func with right args'); + t.deepEqual(bound(1, 2, 3), [hasStrictMode ? 1 : Object(1), 2, 3], 'bound func with right args'); + t.deepEqual(boundR(1, 2), [sentinel, 1, 2], 'bound func with receiver, with right args'); + t.deepEqual(boundArg(2), [sentinel, 1, 2], 'bound func with receiver and arg, with right arg'); + + // @ts-expect-error + t.deepEqual(func(1, 2, 3), [undefined, 1, 2], 'unbound func with too many args'); + // @ts-expect-error + t.deepEqual(bound(1, 2, 3, 4), [hasStrictMode ? 1 : Object(1), 2, 3], 'bound func with too many args'); + // @ts-expect-error + t.deepEqual(boundR(1, 2, 3), [sentinel, 1, 2], 'bound func with receiver, with too many args'); + // @ts-expect-error + t.deepEqual(boundArg(2, 3), [sentinel, 1, 2], 'bound func with receiver and arg, with too many args'); + + t.end(); +}); diff --git a/scraper-service/node_modules/call-bind-apply-helpers/tsconfig.json b/scraper-service/node_modules/call-bind-apply-helpers/tsconfig.json new file mode 100644 index 0000000..aef9993 --- /dev/null +++ b/scraper-service/node_modules/call-bind-apply-helpers/tsconfig.json @@ -0,0 +1,9 @@ +{ + "extends": "@ljharb/tsconfig", + "compilerOptions": { + "target": "es2021", + }, + "exclude": [ + "coverage", + ], +} \ No newline at end of file diff --git a/scraper-service/node_modules/call-bound/.eslintrc b/scraper-service/node_modules/call-bound/.eslintrc new file mode 100644 index 0000000..2612ed8 --- /dev/null +++ b/scraper-service/node_modules/call-bound/.eslintrc @@ -0,0 +1,13 @@ +{ + "root": true, + + "extends": "@ljharb", + + "rules": { + "new-cap": [2, { + "capIsNewExceptions": [ + "GetIntrinsic", + ], + }], + }, +} diff --git a/scraper-service/node_modules/call-bound/.github/FUNDING.yml b/scraper-service/node_modules/call-bound/.github/FUNDING.yml new file mode 100644 index 0000000..2a2a135 --- /dev/null +++ b/scraper-service/node_modules/call-bound/.github/FUNDING.yml @@ -0,0 +1,12 @@ +# These are supported funding model platforms + +github: [ljharb] +patreon: # Replace with a single Patreon username +open_collective: # Replace with a single Open Collective username +ko_fi: # Replace with a single Ko-fi username +tidelift: npm/call-bound +community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry +liberapay: # Replace with a single Liberapay username +issuehunt: # Replace with a single IssueHunt username +otechie: # Replace with a single Otechie username +custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2'] diff --git a/scraper-service/node_modules/call-bound/.nycrc b/scraper-service/node_modules/call-bound/.nycrc new file mode 100644 index 0000000..bdd626c --- /dev/null +++ b/scraper-service/node_modules/call-bound/.nycrc @@ -0,0 +1,9 @@ +{ + "all": true, + "check-coverage": false, + "reporter": ["text-summary", "text", "html", "json"], + "exclude": [ + "coverage", + "test" + ] +} diff --git a/scraper-service/node_modules/call-bound/CHANGELOG.md b/scraper-service/node_modules/call-bound/CHANGELOG.md new file mode 100644 index 0000000..8bde4e9 --- /dev/null +++ b/scraper-service/node_modules/call-bound/CHANGELOG.md @@ -0,0 +1,42 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [v1.0.4](https://github.com/ljharb/call-bound/compare/v1.0.3...v1.0.4) - 2025-03-03 + +### Commits + +- [types] improve types [`e648922`](https://github.com/ljharb/call-bound/commit/e6489222a9e54f350fbf952ceabe51fd8b6027ff) +- [Dev Deps] update `@arethetypeswrong/cli`, `@ljharb/tsconfig`, `@types/tape`, `es-value-fixtures`, `for-each`, `has-strict-mode`, `object-inspect` [`a42a5eb`](https://github.com/ljharb/call-bound/commit/a42a5ebe6c1b54fcdc7997c7dc64fdca9e936719) +- [Deps] update `call-bind-apply-helpers`, `get-intrinsic` [`f529eac`](https://github.com/ljharb/call-bound/commit/f529eac132404c17156bbc23ab2297a25d0f20b8) + +## [v1.0.3](https://github.com/ljharb/call-bound/compare/v1.0.2...v1.0.3) - 2024-12-15 + +### Commits + +- [Refactor] use `call-bind-apply-helpers` instead of `call-bind` [`5e0b134`](https://github.com/ljharb/call-bound/commit/5e0b13496df14fb7d05dae9412f088da8d3f75be) +- [Deps] update `get-intrinsic` [`41fc967`](https://github.com/ljharb/call-bound/commit/41fc96732a22c7b7e8f381f93ccc54bb6293be2e) +- [readme] fix example [`79a0137`](https://github.com/ljharb/call-bound/commit/79a0137723f7c6d09c9c05452bbf8d5efb5d6e49) +- [meta] add `sideEffects` flag [`08b07be`](https://github.com/ljharb/call-bound/commit/08b07be7f1c03f67dc6f3cdaf0906259771859f7) + +## [v1.0.2](https://github.com/ljharb/call-bound/compare/v1.0.1...v1.0.2) - 2024-12-10 + +### Commits + +- [Dev Deps] update `@arethetypeswrong/cli`, `@ljharb/tsconfig`, `gopd` [`e6a5ffe`](https://github.com/ljharb/call-bound/commit/e6a5ffe849368fe4f74dfd6cdeca1b9baa39e8d5) +- [Deps] update `call-bind`, `get-intrinsic` [`2aeb5b5`](https://github.com/ljharb/call-bound/commit/2aeb5b521dc2b2683d1345c753ea1161de2d1c14) +- [types] improve return type [`1a0c9fe`](https://github.com/ljharb/call-bound/commit/1a0c9fe3114471e7ca1f57d104e2efe713bb4871) + +## v1.0.1 - 2024-12-05 + +### Commits + +- Initial implementation, tests, readme, types [`6d94121`](https://github.com/ljharb/call-bound/commit/6d94121a9243602e506334069f7a03189fe3363d) +- Initial commit [`0eae867`](https://github.com/ljharb/call-bound/commit/0eae867334ea025c33e6e91cdecfc9df96680cf9) +- npm init [`71b2479`](https://github.com/ljharb/call-bound/commit/71b2479c6723e0b7d91a6b663613067e98b7b275) +- Only apps should have lockfiles [`c3754a9`](https://github.com/ljharb/call-bound/commit/c3754a949b7f9132b47e2d18c1729889736741eb) +- [actions] skip `npm ls` in node < 10 [`74275a5`](https://github.com/ljharb/call-bound/commit/74275a5186b8caf6309b6b97472bdcb0df4683a8) +- [Dev Deps] add missing peer dep [`1354de8`](https://github.com/ljharb/call-bound/commit/1354de8679413e4ae9c523d85f76fa7a5e032d97) diff --git a/scraper-service/node_modules/call-bound/LICENSE b/scraper-service/node_modules/call-bound/LICENSE new file mode 100644 index 0000000..f82f389 --- /dev/null +++ b/scraper-service/node_modules/call-bound/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2024 Jordan Harband + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/scraper-service/node_modules/call-bound/README.md b/scraper-service/node_modules/call-bound/README.md new file mode 100644 index 0000000..a44e43e --- /dev/null +++ b/scraper-service/node_modules/call-bound/README.md @@ -0,0 +1,53 @@ +# call-bound [![Version Badge][npm-version-svg]][package-url] + +[![github actions][actions-image]][actions-url] +[![coverage][codecov-image]][codecov-url] +[![dependency status][deps-svg]][deps-url] +[![dev dependency status][dev-deps-svg]][dev-deps-url] +[![License][license-image]][license-url] +[![Downloads][downloads-image]][downloads-url] + +[![npm badge][npm-badge-png]][package-url] + +Robust call-bound JavaScript intrinsics, using `call-bind` and `get-intrinsic`. + +## Getting started + +```sh +npm install --save call-bound +``` + +## Usage/Examples + +```js +const assert = require('assert'); +const callBound = require('call-bound'); + +const slice = callBound('Array.prototype.slice'); + +delete Function.prototype.call; +delete Function.prototype.bind; +delete Array.prototype.slice; + +assert.deepEqual(slice([1, 2, 3, 4], 1, -1), [2, 3]); +``` + +## Tests + +Clone the repo, `npm install`, and run `npm test` + +[package-url]: https://npmjs.org/package/call-bound +[npm-version-svg]: https://versionbadg.es/ljharb/call-bound.svg +[deps-svg]: https://david-dm.org/ljharb/call-bound.svg +[deps-url]: https://david-dm.org/ljharb/call-bound +[dev-deps-svg]: https://david-dm.org/ljharb/call-bound/dev-status.svg +[dev-deps-url]: https://david-dm.org/ljharb/call-bound#info=devDependencies +[npm-badge-png]: https://nodei.co/npm/call-bound.png?downloads=true&stars=true +[license-image]: https://img.shields.io/npm/l/call-bound.svg +[license-url]: LICENSE +[downloads-image]: https://img.shields.io/npm/dm/call-bound.svg +[downloads-url]: https://npm-stat.com/charts.html?package=call-bound +[codecov-image]: https://codecov.io/gh/ljharb/call-bound/branch/main/graphs/badge.svg +[codecov-url]: https://app.codecov.io/gh/ljharb/call-bound/ +[actions-image]: https://img.shields.io/endpoint?url=https://github-actions-badge-u3jn4tfpocch.runkit.sh/ljharb/call-bound +[actions-url]: https://github.com/ljharb/call-bound/actions diff --git a/scraper-service/node_modules/call-bound/index.d.ts b/scraper-service/node_modules/call-bound/index.d.ts new file mode 100644 index 0000000..5562f00 --- /dev/null +++ b/scraper-service/node_modules/call-bound/index.d.ts @@ -0,0 +1,94 @@ +type Intrinsic = typeof globalThis; + +type IntrinsicName = keyof Intrinsic | `%${keyof Intrinsic}%`; + +type IntrinsicPath = IntrinsicName | `${StripPercents}.${string}` | `%${StripPercents}.${string}%`; + +type AllowMissing = boolean; + +type StripPercents = T extends `%${infer U}%` ? U : T; + +type BindMethodPrecise = + F extends (this: infer This, ...args: infer Args) => infer R + ? (obj: This, ...args: Args) => R + : F extends { + (this: infer This1, ...args: infer Args1): infer R1; + (this: infer This2, ...args: infer Args2): infer R2 + } + ? { + (obj: This1, ...args: Args1): R1; + (obj: This2, ...args: Args2): R2 + } + : never + +// Extract method type from a prototype +type GetPrototypeMethod = + (typeof globalThis)[T] extends { prototype: any } + ? M extends keyof (typeof globalThis)[T]['prototype'] + ? (typeof globalThis)[T]['prototype'][M] + : never + : never + +// Get static property/method +type GetStaticMember = + P extends keyof (typeof globalThis)[T] ? (typeof globalThis)[T][P] : never + +// Type that maps string path to actual bound function or value with better precision +type BoundIntrinsic = + S extends `${infer Obj}.prototype.${infer Method}` + ? Obj extends keyof typeof globalThis + ? BindMethodPrecise> + : unknown + : S extends `${infer Obj}.${infer Prop}` + ? Obj extends keyof typeof globalThis + ? GetStaticMember + : unknown + : unknown + +declare function arraySlice(array: readonly T[], start?: number, end?: number): T[]; +declare function arraySlice(array: ArrayLike, start?: number, end?: number): T[]; +declare function arraySlice(array: IArguments, start?: number, end?: number): T[]; + +// Special cases for methods that need explicit typing +interface SpecialCases { + '%Object.prototype.isPrototypeOf%': (thisArg: {}, obj: unknown) => boolean; + '%String.prototype.replace%': { + (str: string, searchValue: string | RegExp, replaceValue: string): string; + (str: string, searchValue: string | RegExp, replacer: (substring: string, ...args: any[]) => string): string + }; + '%Object.prototype.toString%': (obj: {}) => string; + '%Object.prototype.hasOwnProperty%': (obj: {}, v: PropertyKey) => boolean; + '%Array.prototype.slice%': typeof arraySlice; + '%Array.prototype.map%': (array: readonly T[], callbackfn: (value: T, index: number, array: readonly T[]) => U, thisArg?: any) => U[]; + '%Array.prototype.filter%': (array: readonly T[], predicate: (value: T, index: number, array: readonly T[]) => unknown, thisArg?: any) => T[]; + '%Array.prototype.indexOf%': (array: readonly T[], searchElement: T, fromIndex?: number) => number; + '%Function.prototype.apply%': (fn: (...args: A) => R, thisArg: any, args: A) => R; + '%Function.prototype.call%': (fn: (...args: A) => R, thisArg: any, ...args: A) => R; + '%Function.prototype.bind%': (fn: (...args: A) => R, thisArg: any, ...args: A) => (...remainingArgs: A) => R; + '%Promise.prototype.then%': { + (promise: Promise, onfulfilled: (value: T) => R | PromiseLike): Promise; + (promise: Promise, onfulfilled: ((value: T) => R | PromiseLike) | undefined | null, onrejected: (reason: any) => R | PromiseLike): Promise; + }; + '%RegExp.prototype.test%': (regexp: RegExp, str: string) => boolean; + '%RegExp.prototype.exec%': (regexp: RegExp, str: string) => RegExpExecArray | null; + '%Error.prototype.toString%': (error: Error) => string; + '%TypeError.prototype.toString%': (error: TypeError) => string; + '%String.prototype.split%': ( + obj: unknown, + splitter: string | RegExp | { + [Symbol.split](string: string, limit?: number): string[]; + }, + limit?: number | undefined + ) => string[]; +} + +/** + * Returns a bound function for a prototype method, or a value for a static property. + * + * @param name - The name of the intrinsic (e.g. 'Array.prototype.slice') + * @param {AllowMissing} [allowMissing] - Whether to allow missing intrinsics (default: false) + */ +declare function callBound, S extends IntrinsicPath>(name: K, allowMissing?: AllowMissing): SpecialCases[`%${StripPercents}%`]; +declare function callBound, S extends IntrinsicPath>(name: S, allowMissing?: AllowMissing): BoundIntrinsic; + +export = callBound; diff --git a/scraper-service/node_modules/call-bound/index.js b/scraper-service/node_modules/call-bound/index.js new file mode 100644 index 0000000..e9ade74 --- /dev/null +++ b/scraper-service/node_modules/call-bound/index.js @@ -0,0 +1,19 @@ +'use strict'; + +var GetIntrinsic = require('get-intrinsic'); + +var callBindBasic = require('call-bind-apply-helpers'); + +/** @type {(thisArg: string, searchString: string, position?: number) => number} */ +var $indexOf = callBindBasic([GetIntrinsic('%String.prototype.indexOf%')]); + +/** @type {import('.')} */ +module.exports = function callBoundIntrinsic(name, allowMissing) { + /* eslint no-extra-parens: 0 */ + + var intrinsic = /** @type {(this: unknown, ...args: unknown[]) => unknown} */ (GetIntrinsic(name, !!allowMissing)); + if (typeof intrinsic === 'function' && $indexOf(name, '.prototype.') > -1) { + return callBindBasic(/** @type {const} */ ([intrinsic])); + } + return intrinsic; +}; diff --git a/scraper-service/node_modules/call-bound/package.json b/scraper-service/node_modules/call-bound/package.json new file mode 100644 index 0000000..d542db4 --- /dev/null +++ b/scraper-service/node_modules/call-bound/package.json @@ -0,0 +1,99 @@ +{ + "name": "call-bound", + "version": "1.0.4", + "description": "Robust call-bound JavaScript intrinsics, using `call-bind` and `get-intrinsic`.", + "main": "index.js", + "exports": { + ".": "./index.js", + "./package.json": "./package.json" + }, + "sideEffects": false, + "scripts": { + "prepack": "npmignore --auto --commentLines=auto", + "prepublish": "not-in-publish || npm run prepublishOnly", + "prepublishOnly": "safe-publish-latest", + "prelint": "evalmd README.md", + "lint": "eslint --ext=.js,.mjs .", + "postlint": "tsc -p . && attw -P", + "pretest": "npm run lint", + "tests-only": "nyc tape 'test/**/*.js'", + "test": "npm run tests-only", + "posttest": "npx npm@'>=10.2' audit --production", + "version": "auto-changelog && git add CHANGELOG.md", + "postversion": "auto-changelog && git add CHANGELOG.md && git commit --no-edit --amend && git tag -f \"v$(node -e \"console.log(require('./package.json').version)\")\"" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/ljharb/call-bound.git" + }, + "keywords": [ + "javascript", + "ecmascript", + "es", + "js", + "callbind", + "callbound", + "call", + "bind", + "bound", + "call-bind", + "call-bound", + "function", + "es-abstract" + ], + "author": "Jordan Harband ", + "funding": { + "url": "https://github.com/sponsors/ljharb" + }, + "license": "MIT", + "bugs": { + "url": "https://github.com/ljharb/call-bound/issues" + }, + "homepage": "https://github.com/ljharb/call-bound#readme", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "get-intrinsic": "^1.3.0" + }, + "devDependencies": { + "@arethetypeswrong/cli": "^0.17.4", + "@ljharb/eslint-config": "^21.1.1", + "@ljharb/tsconfig": "^0.3.0", + "@types/call-bind": "^1.0.5", + "@types/get-intrinsic": "^1.2.3", + "@types/tape": "^5.8.1", + "auto-changelog": "^2.5.0", + "encoding": "^0.1.13", + "es-value-fixtures": "^1.7.1", + "eslint": "=8.8.0", + "evalmd": "^0.0.19", + "for-each": "^0.3.5", + "gopd": "^1.2.0", + "has-strict-mode": "^1.1.0", + "in-publish": "^2.0.1", + "npmignore": "^0.3.1", + "nyc": "^10.3.2", + "object-inspect": "^1.13.4", + "safe-publish-latest": "^2.0.0", + "tape": "^5.9.0", + "typescript": "next" + }, + "testling": { + "files": "test/index.js" + }, + "auto-changelog": { + "output": "CHANGELOG.md", + "template": "keepachangelog", + "unreleased": false, + "commitLimit": false, + "backfillLimit": false, + "hideCredit": true + }, + "publishConfig": { + "ignore": [ + ".github/workflows" + ] + }, + "engines": { + "node": ">= 0.4" + } +} diff --git a/scraper-service/node_modules/call-bound/test/index.js b/scraper-service/node_modules/call-bound/test/index.js new file mode 100644 index 0000000..a2fc9f0 --- /dev/null +++ b/scraper-service/node_modules/call-bound/test/index.js @@ -0,0 +1,61 @@ +'use strict'; + +var test = require('tape'); + +var callBound = require('../'); + +/** @template {true} T @template U @typedef {T extends U ? T : never} AssertType */ + +test('callBound', function (t) { + // static primitive + t.equal(callBound('Array.length'), Array.length, 'Array.length yields itself'); + t.equal(callBound('%Array.length%'), Array.length, '%Array.length% yields itself'); + + // static non-function object + t.equal(callBound('Array.prototype'), Array.prototype, 'Array.prototype yields itself'); + t.equal(callBound('%Array.prototype%'), Array.prototype, '%Array.prototype% yields itself'); + t.equal(callBound('Array.constructor'), Array.constructor, 'Array.constructor yields itself'); + t.equal(callBound('%Array.constructor%'), Array.constructor, '%Array.constructor% yields itself'); + + // static function + t.equal(callBound('Date.parse'), Date.parse, 'Date.parse yields itself'); + t.equal(callBound('%Date.parse%'), Date.parse, '%Date.parse% yields itself'); + + // prototype primitive + t.equal(callBound('Error.prototype.message'), Error.prototype.message, 'Error.prototype.message yields itself'); + t.equal(callBound('%Error.prototype.message%'), Error.prototype.message, '%Error.prototype.message% yields itself'); + + var x = callBound('Object.prototype.toString'); + var y = callBound('%Object.prototype.toString%'); + + // prototype function + t.notEqual(x, Object.prototype.toString, 'Object.prototype.toString does not yield itself'); + t.notEqual(y, Object.prototype.toString, '%Object.prototype.toString% does not yield itself'); + t.equal(x(true), Object.prototype.toString.call(true), 'call-bound Object.prototype.toString calls into the original'); + t.equal(y(true), Object.prototype.toString.call(true), 'call-bound %Object.prototype.toString% calls into the original'); + + t['throws']( + // @ts-expect-error + function () { callBound('does not exist'); }, + SyntaxError, + 'nonexistent intrinsic throws' + ); + t['throws']( + // @ts-expect-error + function () { callBound('does not exist', true); }, + SyntaxError, + 'allowMissing arg still throws for unknown intrinsic' + ); + + t.test('real but absent intrinsic', { skip: typeof WeakRef !== 'undefined' }, function (st) { + st['throws']( + function () { callBound('WeakRef'); }, + TypeError, + 'real but absent intrinsic throws' + ); + st.equal(callBound('WeakRef', true), undefined, 'allowMissing arg avoids exception'); + st.end(); + }); + + t.end(); +}); diff --git a/scraper-service/node_modules/call-bound/tsconfig.json b/scraper-service/node_modules/call-bound/tsconfig.json new file mode 100644 index 0000000..8976d98 --- /dev/null +++ b/scraper-service/node_modules/call-bound/tsconfig.json @@ -0,0 +1,10 @@ +{ + "extends": "@ljharb/tsconfig", + "compilerOptions": { + "target": "ESNext", + "lib": ["es2024"], + }, + "exclude": [ + "coverage", + ], +} diff --git a/scraper-service/node_modules/content-disposition/HISTORY.md b/scraper-service/node_modules/content-disposition/HISTORY.md new file mode 100644 index 0000000..488effa --- /dev/null +++ b/scraper-service/node_modules/content-disposition/HISTORY.md @@ -0,0 +1,60 @@ +0.5.4 / 2021-12-10 +================== + + * deps: safe-buffer@5.2.1 + +0.5.3 / 2018-12-17 +================== + + * Use `safe-buffer` for improved Buffer API + +0.5.2 / 2016-12-08 +================== + + * Fix `parse` to accept any linear whitespace character + +0.5.1 / 2016-01-17 +================== + + * perf: enable strict mode + +0.5.0 / 2014-10-11 +================== + + * Add `parse` function + +0.4.0 / 2014-09-21 +================== + + * Expand non-Unicode `filename` to the full ISO-8859-1 charset + +0.3.0 / 2014-09-20 +================== + + * Add `fallback` option + * Add `type` option + +0.2.0 / 2014-09-19 +================== + + * Reduce ambiguity of file names with hex escape in buggy browsers + +0.1.2 / 2014-09-19 +================== + + * Fix periodic invalid Unicode filename header + +0.1.1 / 2014-09-19 +================== + + * Fix invalid characters appearing in `filename*` parameter + +0.1.0 / 2014-09-18 +================== + + * Make the `filename` argument optional + +0.0.0 / 2014-09-18 +================== + + * Initial release diff --git a/scraper-service/node_modules/content-disposition/LICENSE b/scraper-service/node_modules/content-disposition/LICENSE new file mode 100644 index 0000000..84441fb --- /dev/null +++ b/scraper-service/node_modules/content-disposition/LICENSE @@ -0,0 +1,22 @@ +(The MIT License) + +Copyright (c) 2014-2017 Douglas Christopher Wilson + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +'Software'), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/scraper-service/node_modules/content-disposition/README.md b/scraper-service/node_modules/content-disposition/README.md new file mode 100644 index 0000000..3a0bb05 --- /dev/null +++ b/scraper-service/node_modules/content-disposition/README.md @@ -0,0 +1,142 @@ +# content-disposition + +[![NPM Version][npm-image]][npm-url] +[![NPM Downloads][downloads-image]][downloads-url] +[![Node.js Version][node-version-image]][node-version-url] +[![Build Status][github-actions-ci-image]][github-actions-ci-url] +[![Test Coverage][coveralls-image]][coveralls-url] + +Create and parse HTTP `Content-Disposition` header + +## Installation + +```sh +$ npm install content-disposition +``` + +## API + +```js +var contentDisposition = require('content-disposition') +``` + +### contentDisposition(filename, options) + +Create an attachment `Content-Disposition` header value using the given file name, +if supplied. The `filename` is optional and if no file name is desired, but you +want to specify `options`, set `filename` to `undefined`. + +```js +res.setHeader('Content-Disposition', contentDisposition('∫ maths.pdf')) +``` + +**note** HTTP headers are of the ISO-8859-1 character set. If you are writing this +header through a means different from `setHeader` in Node.js, you'll want to specify +the `'binary'` encoding in Node.js. + +#### Options + +`contentDisposition` accepts these properties in the options object. + +##### fallback + +If the `filename` option is outside ISO-8859-1, then the file name is actually +stored in a supplemental field for clients that support Unicode file names and +a ISO-8859-1 version of the file name is automatically generated. + +This specifies the ISO-8859-1 file name to override the automatic generation or +disables the generation all together, defaults to `true`. + + - A string will specify the ISO-8859-1 file name to use in place of automatic + generation. + - `false` will disable including a ISO-8859-1 file name and only include the + Unicode version (unless the file name is already ISO-8859-1). + - `true` will enable automatic generation if the file name is outside ISO-8859-1. + +If the `filename` option is ISO-8859-1 and this option is specified and has a +different value, then the `filename` option is encoded in the extended field +and this set as the fallback field, even though they are both ISO-8859-1. + +##### type + +Specifies the disposition type, defaults to `"attachment"`. This can also be +`"inline"`, or any other value (all values except inline are treated like +`attachment`, but can convey additional information if both parties agree to +it). The type is normalized to lower-case. + +### contentDisposition.parse(string) + +```js +var disposition = contentDisposition.parse('attachment; filename="EURO rates.txt"; filename*=UTF-8\'\'%e2%82%ac%20rates.txt') +``` + +Parse a `Content-Disposition` header string. This automatically handles extended +("Unicode") parameters by decoding them and providing them under the standard +parameter name. This will return an object with the following properties (examples +are shown for the string `'attachment; filename="EURO rates.txt"; filename*=UTF-8\'\'%e2%82%ac%20rates.txt'`): + + - `type`: The disposition type (always lower case). Example: `'attachment'` + + - `parameters`: An object of the parameters in the disposition (name of parameter + always lower case and extended versions replace non-extended versions). Example: + `{filename: "€ rates.txt"}` + +## Examples + +### Send a file for download + +```js +var contentDisposition = require('content-disposition') +var destroy = require('destroy') +var fs = require('fs') +var http = require('http') +var onFinished = require('on-finished') + +var filePath = '/path/to/public/plans.pdf' + +http.createServer(function onRequest (req, res) { + // set headers + res.setHeader('Content-Type', 'application/pdf') + res.setHeader('Content-Disposition', contentDisposition(filePath)) + + // send file + var stream = fs.createReadStream(filePath) + stream.pipe(res) + onFinished(res, function () { + destroy(stream) + }) +}) +``` + +## Testing + +```sh +$ npm test +``` + +## References + +- [RFC 2616: Hypertext Transfer Protocol -- HTTP/1.1][rfc-2616] +- [RFC 5987: Character Set and Language Encoding for Hypertext Transfer Protocol (HTTP) Header Field Parameters][rfc-5987] +- [RFC 6266: Use of the Content-Disposition Header Field in the Hypertext Transfer Protocol (HTTP)][rfc-6266] +- [Test Cases for HTTP Content-Disposition header field (RFC 6266) and the Encodings defined in RFCs 2047, 2231 and 5987][tc-2231] + +[rfc-2616]: https://tools.ietf.org/html/rfc2616 +[rfc-5987]: https://tools.ietf.org/html/rfc5987 +[rfc-6266]: https://tools.ietf.org/html/rfc6266 +[tc-2231]: http://greenbytes.de/tech/tc2231/ + +## License + +[MIT](LICENSE) + +[npm-image]: https://img.shields.io/npm/v/content-disposition.svg +[npm-url]: https://npmjs.org/package/content-disposition +[node-version-image]: https://img.shields.io/node/v/content-disposition.svg +[node-version-url]: https://nodejs.org/en/download +[coveralls-image]: https://img.shields.io/coveralls/jshttp/content-disposition.svg +[coveralls-url]: https://coveralls.io/r/jshttp/content-disposition?branch=master +[downloads-image]: https://img.shields.io/npm/dm/content-disposition.svg +[downloads-url]: https://npmjs.org/package/content-disposition +[github-actions-ci-image]: https://img.shields.io/github/workflow/status/jshttp/content-disposition/ci/master?label=ci +[github-actions-ci-url]: https://github.com/jshttp/content-disposition?query=workflow%3Aci diff --git a/scraper-service/node_modules/content-disposition/index.js b/scraper-service/node_modules/content-disposition/index.js new file mode 100644 index 0000000..ecec899 --- /dev/null +++ b/scraper-service/node_modules/content-disposition/index.js @@ -0,0 +1,458 @@ +/*! + * content-disposition + * Copyright(c) 2014-2017 Douglas Christopher Wilson + * MIT Licensed + */ + +'use strict' + +/** + * Module exports. + * @public + */ + +module.exports = contentDisposition +module.exports.parse = parse + +/** + * Module dependencies. + * @private + */ + +var basename = require('path').basename +var Buffer = require('safe-buffer').Buffer + +/** + * RegExp to match non attr-char, *after* encodeURIComponent (i.e. not including "%") + * @private + */ + +var ENCODE_URL_ATTR_CHAR_REGEXP = /[\x00-\x20"'()*,/:;<=>?@[\\\]{}\x7f]/g // eslint-disable-line no-control-regex + +/** + * RegExp to match percent encoding escape. + * @private + */ + +var HEX_ESCAPE_REGEXP = /%[0-9A-Fa-f]{2}/ +var HEX_ESCAPE_REPLACE_REGEXP = /%([0-9A-Fa-f]{2})/g + +/** + * RegExp to match non-latin1 characters. + * @private + */ + +var NON_LATIN1_REGEXP = /[^\x20-\x7e\xa0-\xff]/g + +/** + * RegExp to match quoted-pair in RFC 2616 + * + * quoted-pair = "\" CHAR + * CHAR = + * @private + */ + +var QESC_REGEXP = /\\([\u0000-\u007f])/g // eslint-disable-line no-control-regex + +/** + * RegExp to match chars that must be quoted-pair in RFC 2616 + * @private + */ + +var QUOTE_REGEXP = /([\\"])/g + +/** + * RegExp for various RFC 2616 grammar + * + * parameter = token "=" ( token | quoted-string ) + * token = 1* + * separators = "(" | ")" | "<" | ">" | "@" + * | "," | ";" | ":" | "\" | <"> + * | "/" | "[" | "]" | "?" | "=" + * | "{" | "}" | SP | HT + * quoted-string = ( <"> *(qdtext | quoted-pair ) <"> ) + * qdtext = > + * quoted-pair = "\" CHAR + * CHAR = + * TEXT = + * LWS = [CRLF] 1*( SP | HT ) + * CRLF = CR LF + * CR = + * LF = + * SP = + * HT = + * CTL = + * OCTET = + * @private + */ + +var PARAM_REGEXP = /;[\x09\x20]*([!#$%&'*+.0-9A-Z^_`a-z|~-]+)[\x09\x20]*=[\x09\x20]*("(?:[\x20!\x23-\x5b\x5d-\x7e\x80-\xff]|\\[\x20-\x7e])*"|[!#$%&'*+.0-9A-Z^_`a-z|~-]+)[\x09\x20]*/g // eslint-disable-line no-control-regex +var TEXT_REGEXP = /^[\x20-\x7e\x80-\xff]+$/ +var TOKEN_REGEXP = /^[!#$%&'*+.0-9A-Z^_`a-z|~-]+$/ + +/** + * RegExp for various RFC 5987 grammar + * + * ext-value = charset "'" [ language ] "'" value-chars + * charset = "UTF-8" / "ISO-8859-1" / mime-charset + * mime-charset = 1*mime-charsetc + * mime-charsetc = ALPHA / DIGIT + * / "!" / "#" / "$" / "%" / "&" + * / "+" / "-" / "^" / "_" / "`" + * / "{" / "}" / "~" + * language = ( 2*3ALPHA [ extlang ] ) + * / 4ALPHA + * / 5*8ALPHA + * extlang = *3( "-" 3ALPHA ) + * value-chars = *( pct-encoded / attr-char ) + * pct-encoded = "%" HEXDIG HEXDIG + * attr-char = ALPHA / DIGIT + * / "!" / "#" / "$" / "&" / "+" / "-" / "." + * / "^" / "_" / "`" / "|" / "~" + * @private + */ + +var EXT_VALUE_REGEXP = /^([A-Za-z0-9!#$%&+\-^_`{}~]+)'(?:[A-Za-z]{2,3}(?:-[A-Za-z]{3}){0,3}|[A-Za-z]{4,8}|)'((?:%[0-9A-Fa-f]{2}|[A-Za-z0-9!#$&+.^_`|~-])+)$/ + +/** + * RegExp for various RFC 6266 grammar + * + * disposition-type = "inline" | "attachment" | disp-ext-type + * disp-ext-type = token + * disposition-parm = filename-parm | disp-ext-parm + * filename-parm = "filename" "=" value + * | "filename*" "=" ext-value + * disp-ext-parm = token "=" value + * | ext-token "=" ext-value + * ext-token = + * @private + */ + +var DISPOSITION_TYPE_REGEXP = /^([!#$%&'*+.0-9A-Z^_`a-z|~-]+)[\x09\x20]*(?:$|;)/ // eslint-disable-line no-control-regex + +/** + * Create an attachment Content-Disposition header. + * + * @param {string} [filename] + * @param {object} [options] + * @param {string} [options.type=attachment] + * @param {string|boolean} [options.fallback=true] + * @return {string} + * @public + */ + +function contentDisposition (filename, options) { + var opts = options || {} + + // get type + var type = opts.type || 'attachment' + + // get parameters + var params = createparams(filename, opts.fallback) + + // format into string + return format(new ContentDisposition(type, params)) +} + +/** + * Create parameters object from filename and fallback. + * + * @param {string} [filename] + * @param {string|boolean} [fallback=true] + * @return {object} + * @private + */ + +function createparams (filename, fallback) { + if (filename === undefined) { + return + } + + var params = {} + + if (typeof filename !== 'string') { + throw new TypeError('filename must be a string') + } + + // fallback defaults to true + if (fallback === undefined) { + fallback = true + } + + if (typeof fallback !== 'string' && typeof fallback !== 'boolean') { + throw new TypeError('fallback must be a string or boolean') + } + + if (typeof fallback === 'string' && NON_LATIN1_REGEXP.test(fallback)) { + throw new TypeError('fallback must be ISO-8859-1 string') + } + + // restrict to file base name + var name = basename(filename) + + // determine if name is suitable for quoted string + var isQuotedString = TEXT_REGEXP.test(name) + + // generate fallback name + var fallbackName = typeof fallback !== 'string' + ? fallback && getlatin1(name) + : basename(fallback) + var hasFallback = typeof fallbackName === 'string' && fallbackName !== name + + // set extended filename parameter + if (hasFallback || !isQuotedString || HEX_ESCAPE_REGEXP.test(name)) { + params['filename*'] = name + } + + // set filename parameter + if (isQuotedString || hasFallback) { + params.filename = hasFallback + ? fallbackName + : name + } + + return params +} + +/** + * Format object to Content-Disposition header. + * + * @param {object} obj + * @param {string} obj.type + * @param {object} [obj.parameters] + * @return {string} + * @private + */ + +function format (obj) { + var parameters = obj.parameters + var type = obj.type + + if (!type || typeof type !== 'string' || !TOKEN_REGEXP.test(type)) { + throw new TypeError('invalid type') + } + + // start with normalized type + var string = String(type).toLowerCase() + + // append parameters + if (parameters && typeof parameters === 'object') { + var param + var params = Object.keys(parameters).sort() + + for (var i = 0; i < params.length; i++) { + param = params[i] + + var val = param.substr(-1) === '*' + ? ustring(parameters[param]) + : qstring(parameters[param]) + + string += '; ' + param + '=' + val + } + } + + return string +} + +/** + * Decode a RFC 5987 field value (gracefully). + * + * @param {string} str + * @return {string} + * @private + */ + +function decodefield (str) { + var match = EXT_VALUE_REGEXP.exec(str) + + if (!match) { + throw new TypeError('invalid extended field value') + } + + var charset = match[1].toLowerCase() + var encoded = match[2] + var value + + // to binary string + var binary = encoded.replace(HEX_ESCAPE_REPLACE_REGEXP, pdecode) + + switch (charset) { + case 'iso-8859-1': + value = getlatin1(binary) + break + case 'utf-8': + value = Buffer.from(binary, 'binary').toString('utf8') + break + default: + throw new TypeError('unsupported charset in extended field') + } + + return value +} + +/** + * Get ISO-8859-1 version of string. + * + * @param {string} val + * @return {string} + * @private + */ + +function getlatin1 (val) { + // simple Unicode -> ISO-8859-1 transformation + return String(val).replace(NON_LATIN1_REGEXP, '?') +} + +/** + * Parse Content-Disposition header string. + * + * @param {string} string + * @return {object} + * @public + */ + +function parse (string) { + if (!string || typeof string !== 'string') { + throw new TypeError('argument string is required') + } + + var match = DISPOSITION_TYPE_REGEXP.exec(string) + + if (!match) { + throw new TypeError('invalid type format') + } + + // normalize type + var index = match[0].length + var type = match[1].toLowerCase() + + var key + var names = [] + var params = {} + var value + + // calculate index to start at + index = PARAM_REGEXP.lastIndex = match[0].substr(-1) === ';' + ? index - 1 + : index + + // match parameters + while ((match = PARAM_REGEXP.exec(string))) { + if (match.index !== index) { + throw new TypeError('invalid parameter format') + } + + index += match[0].length + key = match[1].toLowerCase() + value = match[2] + + if (names.indexOf(key) !== -1) { + throw new TypeError('invalid duplicate parameter') + } + + names.push(key) + + if (key.indexOf('*') + 1 === key.length) { + // decode extended value + key = key.slice(0, -1) + value = decodefield(value) + + // overwrite existing value + params[key] = value + continue + } + + if (typeof params[key] === 'string') { + continue + } + + if (value[0] === '"') { + // remove quotes and escapes + value = value + .substr(1, value.length - 2) + .replace(QESC_REGEXP, '$1') + } + + params[key] = value + } + + if (index !== -1 && index !== string.length) { + throw new TypeError('invalid parameter format') + } + + return new ContentDisposition(type, params) +} + +/** + * Percent decode a single character. + * + * @param {string} str + * @param {string} hex + * @return {string} + * @private + */ + +function pdecode (str, hex) { + return String.fromCharCode(parseInt(hex, 16)) +} + +/** + * Percent encode a single character. + * + * @param {string} char + * @return {string} + * @private + */ + +function pencode (char) { + return '%' + String(char) + .charCodeAt(0) + .toString(16) + .toUpperCase() +} + +/** + * Quote a string for HTTP. + * + * @param {string} val + * @return {string} + * @private + */ + +function qstring (val) { + var str = String(val) + + return '"' + str.replace(QUOTE_REGEXP, '\\$1') + '"' +} + +/** + * Encode a Unicode string for HTTP (RFC 5987). + * + * @param {string} val + * @return {string} + * @private + */ + +function ustring (val) { + var str = String(val) + + // percent encode as UTF-8 + var encoded = encodeURIComponent(str) + .replace(ENCODE_URL_ATTR_CHAR_REGEXP, pencode) + + return 'UTF-8\'\'' + encoded +} + +/** + * Class for parsed Content-Disposition header for v8 optimization + * + * @public + * @param {string} type + * @param {object} parameters + * @constructor + */ + +function ContentDisposition (type, parameters) { + this.type = type + this.parameters = parameters +} diff --git a/scraper-service/node_modules/content-disposition/package.json b/scraper-service/node_modules/content-disposition/package.json new file mode 100644 index 0000000..43c70ce --- /dev/null +++ b/scraper-service/node_modules/content-disposition/package.json @@ -0,0 +1,44 @@ +{ + "name": "content-disposition", + "description": "Create and parse Content-Disposition header", + "version": "0.5.4", + "author": "Douglas Christopher Wilson ", + "license": "MIT", + "keywords": [ + "content-disposition", + "http", + "rfc6266", + "res" + ], + "repository": "jshttp/content-disposition", + "dependencies": { + "safe-buffer": "5.2.1" + }, + "devDependencies": { + "deep-equal": "1.0.1", + "eslint": "7.32.0", + "eslint-config-standard": "13.0.1", + "eslint-plugin-import": "2.25.3", + "eslint-plugin-markdown": "2.2.1", + "eslint-plugin-node": "11.1.0", + "eslint-plugin-promise": "5.2.0", + "eslint-plugin-standard": "4.1.0", + "istanbul": "0.4.5", + "mocha": "9.1.3" + }, + "files": [ + "LICENSE", + "HISTORY.md", + "README.md", + "index.js" + ], + "engines": { + "node": ">= 0.6" + }, + "scripts": { + "lint": "eslint .", + "test": "mocha --reporter spec --bail --check-leaks test/", + "test-ci": "istanbul cover node_modules/mocha/bin/_mocha --report lcovonly -- --reporter spec --check-leaks test/", + "test-cov": "istanbul cover node_modules/mocha/bin/_mocha -- --reporter dot --check-leaks test/" + } +} diff --git a/scraper-service/node_modules/content-type/HISTORY.md b/scraper-service/node_modules/content-type/HISTORY.md new file mode 100644 index 0000000..4583671 --- /dev/null +++ b/scraper-service/node_modules/content-type/HISTORY.md @@ -0,0 +1,29 @@ +1.0.5 / 2023-01-29 +================== + + * perf: skip value escaping when unnecessary + +1.0.4 / 2017-09-11 +================== + + * perf: skip parameter parsing when no parameters + +1.0.3 / 2017-09-10 +================== + + * perf: remove argument reassignment + +1.0.2 / 2016-05-09 +================== + + * perf: enable strict mode + +1.0.1 / 2015-02-13 +================== + + * Improve missing `Content-Type` header error message + +1.0.0 / 2015-02-01 +================== + + * Initial implementation, derived from `media-typer@0.3.0` diff --git a/scraper-service/node_modules/content-type/LICENSE b/scraper-service/node_modules/content-type/LICENSE new file mode 100644 index 0000000..34b1a2d --- /dev/null +++ b/scraper-service/node_modules/content-type/LICENSE @@ -0,0 +1,22 @@ +(The MIT License) + +Copyright (c) 2015 Douglas Christopher Wilson + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +'Software'), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/scraper-service/node_modules/content-type/README.md b/scraper-service/node_modules/content-type/README.md new file mode 100644 index 0000000..c1a922a --- /dev/null +++ b/scraper-service/node_modules/content-type/README.md @@ -0,0 +1,94 @@ +# content-type + +[![NPM Version][npm-version-image]][npm-url] +[![NPM Downloads][npm-downloads-image]][npm-url] +[![Node.js Version][node-image]][node-url] +[![Build Status][ci-image]][ci-url] +[![Coverage Status][coveralls-image]][coveralls-url] + +Create and parse HTTP Content-Type header according to RFC 7231 + +## Installation + +```sh +$ npm install content-type +``` + +## API + +```js +var contentType = require('content-type') +``` + +### contentType.parse(string) + +```js +var obj = contentType.parse('image/svg+xml; charset=utf-8') +``` + +Parse a `Content-Type` header. This will return an object with the following +properties (examples are shown for the string `'image/svg+xml; charset=utf-8'`): + + - `type`: The media type (the type and subtype, always lower case). + Example: `'image/svg+xml'` + + - `parameters`: An object of the parameters in the media type (name of parameter + always lower case). Example: `{charset: 'utf-8'}` + +Throws a `TypeError` if the string is missing or invalid. + +### contentType.parse(req) + +```js +var obj = contentType.parse(req) +``` + +Parse the `Content-Type` header from the given `req`. Short-cut for +`contentType.parse(req.headers['content-type'])`. + +Throws a `TypeError` if the `Content-Type` header is missing or invalid. + +### contentType.parse(res) + +```js +var obj = contentType.parse(res) +``` + +Parse the `Content-Type` header set on the given `res`. Short-cut for +`contentType.parse(res.getHeader('content-type'))`. + +Throws a `TypeError` if the `Content-Type` header is missing or invalid. + +### contentType.format(obj) + +```js +var str = contentType.format({ + type: 'image/svg+xml', + parameters: { charset: 'utf-8' } +}) +``` + +Format an object into a `Content-Type` header. This will return a string of the +content type for the given object with the following properties (examples are +shown that produce the string `'image/svg+xml; charset=utf-8'`): + + - `type`: The media type (will be lower-cased). Example: `'image/svg+xml'` + + - `parameters`: An object of the parameters in the media type (name of the + parameter will be lower-cased). Example: `{charset: 'utf-8'}` + +Throws a `TypeError` if the object contains an invalid type or parameter names. + +## License + +[MIT](LICENSE) + +[ci-image]: https://badgen.net/github/checks/jshttp/content-type/master?label=ci +[ci-url]: https://github.com/jshttp/content-type/actions/workflows/ci.yml +[coveralls-image]: https://badgen.net/coveralls/c/github/jshttp/content-type/master +[coveralls-url]: https://coveralls.io/r/jshttp/content-type?branch=master +[node-image]: https://badgen.net/npm/node/content-type +[node-url]: https://nodejs.org/en/download +[npm-downloads-image]: https://badgen.net/npm/dm/content-type +[npm-url]: https://npmjs.org/package/content-type +[npm-version-image]: https://badgen.net/npm/v/content-type diff --git a/scraper-service/node_modules/content-type/index.js b/scraper-service/node_modules/content-type/index.js new file mode 100644 index 0000000..41840e7 --- /dev/null +++ b/scraper-service/node_modules/content-type/index.js @@ -0,0 +1,225 @@ +/*! + * content-type + * Copyright(c) 2015 Douglas Christopher Wilson + * MIT Licensed + */ + +'use strict' + +/** + * RegExp to match *( ";" parameter ) in RFC 7231 sec 3.1.1.1 + * + * parameter = token "=" ( token / quoted-string ) + * token = 1*tchar + * tchar = "!" / "#" / "$" / "%" / "&" / "'" / "*" + * / "+" / "-" / "." / "^" / "_" / "`" / "|" / "~" + * / DIGIT / ALPHA + * ; any VCHAR, except delimiters + * quoted-string = DQUOTE *( qdtext / quoted-pair ) DQUOTE + * qdtext = HTAB / SP / %x21 / %x23-5B / %x5D-7E / obs-text + * obs-text = %x80-FF + * quoted-pair = "\" ( HTAB / SP / VCHAR / obs-text ) + */ +var PARAM_REGEXP = /; *([!#$%&'*+.^_`|~0-9A-Za-z-]+) *= *("(?:[\u000b\u0020\u0021\u0023-\u005b\u005d-\u007e\u0080-\u00ff]|\\[\u000b\u0020-\u00ff])*"|[!#$%&'*+.^_`|~0-9A-Za-z-]+) */g // eslint-disable-line no-control-regex +var TEXT_REGEXP = /^[\u000b\u0020-\u007e\u0080-\u00ff]+$/ // eslint-disable-line no-control-regex +var TOKEN_REGEXP = /^[!#$%&'*+.^_`|~0-9A-Za-z-]+$/ + +/** + * RegExp to match quoted-pair in RFC 7230 sec 3.2.6 + * + * quoted-pair = "\" ( HTAB / SP / VCHAR / obs-text ) + * obs-text = %x80-FF + */ +var QESC_REGEXP = /\\([\u000b\u0020-\u00ff])/g // eslint-disable-line no-control-regex + +/** + * RegExp to match chars that must be quoted-pair in RFC 7230 sec 3.2.6 + */ +var QUOTE_REGEXP = /([\\"])/g + +/** + * RegExp to match type in RFC 7231 sec 3.1.1.1 + * + * media-type = type "/" subtype + * type = token + * subtype = token + */ +var TYPE_REGEXP = /^[!#$%&'*+.^_`|~0-9A-Za-z-]+\/[!#$%&'*+.^_`|~0-9A-Za-z-]+$/ + +/** + * Module exports. + * @public + */ + +exports.format = format +exports.parse = parse + +/** + * Format object to media type. + * + * @param {object} obj + * @return {string} + * @public + */ + +function format (obj) { + if (!obj || typeof obj !== 'object') { + throw new TypeError('argument obj is required') + } + + var parameters = obj.parameters + var type = obj.type + + if (!type || !TYPE_REGEXP.test(type)) { + throw new TypeError('invalid type') + } + + var string = type + + // append parameters + if (parameters && typeof parameters === 'object') { + var param + var params = Object.keys(parameters).sort() + + for (var i = 0; i < params.length; i++) { + param = params[i] + + if (!TOKEN_REGEXP.test(param)) { + throw new TypeError('invalid parameter name') + } + + string += '; ' + param + '=' + qstring(parameters[param]) + } + } + + return string +} + +/** + * Parse media type to object. + * + * @param {string|object} string + * @return {Object} + * @public + */ + +function parse (string) { + if (!string) { + throw new TypeError('argument string is required') + } + + // support req/res-like objects as argument + var header = typeof string === 'object' + ? getcontenttype(string) + : string + + if (typeof header !== 'string') { + throw new TypeError('argument string is required to be a string') + } + + var index = header.indexOf(';') + var type = index !== -1 + ? header.slice(0, index).trim() + : header.trim() + + if (!TYPE_REGEXP.test(type)) { + throw new TypeError('invalid media type') + } + + var obj = new ContentType(type.toLowerCase()) + + // parse parameters + if (index !== -1) { + var key + var match + var value + + PARAM_REGEXP.lastIndex = index + + while ((match = PARAM_REGEXP.exec(header))) { + if (match.index !== index) { + throw new TypeError('invalid parameter format') + } + + index += match[0].length + key = match[1].toLowerCase() + value = match[2] + + if (value.charCodeAt(0) === 0x22 /* " */) { + // remove quotes + value = value.slice(1, -1) + + // remove escapes + if (value.indexOf('\\') !== -1) { + value = value.replace(QESC_REGEXP, '$1') + } + } + + obj.parameters[key] = value + } + + if (index !== header.length) { + throw new TypeError('invalid parameter format') + } + } + + return obj +} + +/** + * Get content-type from req/res objects. + * + * @param {object} + * @return {Object} + * @private + */ + +function getcontenttype (obj) { + var header + + if (typeof obj.getHeader === 'function') { + // res-like + header = obj.getHeader('content-type') + } else if (typeof obj.headers === 'object') { + // req-like + header = obj.headers && obj.headers['content-type'] + } + + if (typeof header !== 'string') { + throw new TypeError('content-type header is missing from object') + } + + return header +} + +/** + * Quote a string if necessary. + * + * @param {string} val + * @return {string} + * @private + */ + +function qstring (val) { + var str = String(val) + + // no need to quote tokens + if (TOKEN_REGEXP.test(str)) { + return str + } + + if (str.length > 0 && !TEXT_REGEXP.test(str)) { + throw new TypeError('invalid parameter value') + } + + return '"' + str.replace(QUOTE_REGEXP, '\\$1') + '"' +} + +/** + * Class to represent a content type. + * @private + */ +function ContentType (type) { + this.parameters = Object.create(null) + this.type = type +} diff --git a/scraper-service/node_modules/content-type/package.json b/scraper-service/node_modules/content-type/package.json new file mode 100644 index 0000000..9db19f6 --- /dev/null +++ b/scraper-service/node_modules/content-type/package.json @@ -0,0 +1,42 @@ +{ + "name": "content-type", + "description": "Create and parse HTTP Content-Type header", + "version": "1.0.5", + "author": "Douglas Christopher Wilson ", + "license": "MIT", + "keywords": [ + "content-type", + "http", + "req", + "res", + "rfc7231" + ], + "repository": "jshttp/content-type", + "devDependencies": { + "deep-equal": "1.0.1", + "eslint": "8.32.0", + "eslint-config-standard": "15.0.1", + "eslint-plugin-import": "2.27.5", + "eslint-plugin-node": "11.1.0", + "eslint-plugin-promise": "6.1.1", + "eslint-plugin-standard": "4.1.0", + "mocha": "10.2.0", + "nyc": "15.1.0" + }, + "files": [ + "LICENSE", + "HISTORY.md", + "README.md", + "index.js" + ], + "engines": { + "node": ">= 0.6" + }, + "scripts": { + "lint": "eslint .", + "test": "mocha --reporter spec --check-leaks --bail test/", + "test-ci": "nyc --reporter=lcovonly --reporter=text npm test", + "test-cov": "nyc --reporter=html --reporter=text npm test", + "version": "node scripts/version-history.js && git add HISTORY.md" + } +} diff --git a/scraper-service/node_modules/cookie-signature/History.md b/scraper-service/node_modules/cookie-signature/History.md new file mode 100644 index 0000000..bcf8cc9 --- /dev/null +++ b/scraper-service/node_modules/cookie-signature/History.md @@ -0,0 +1,42 @@ +1.0.7 / 2023-04-12 +================== + +* backport the buffer support from the 1.2.x release branch (thanks @FadhiliNjagi!) + +1.0.6 / 2015-02-03 +================== + +* use `npm test` instead of `make test` to run tests +* clearer assertion messages when checking input + +1.0.5 / 2014-09-05 +================== + +* add license to package.json + +1.0.4 / 2014-06-25 +================== + + * corrected avoidance of timing attacks (thanks @tenbits!) + +1.0.3 / 2014-01-28 +================== + + * [incorrect] fix for timing attacks + +1.0.2 / 2014-01-28 +================== + + * fix missing repository warning + * fix typo in test + +1.0.1 / 2013-04-15 +================== + + * Revert "Changed underlying HMAC algo. to sha512." + * Revert "Fix for timing attacks on MAC verification." + +0.0.1 / 2010-01-03 +================== + + * Initial release diff --git a/scraper-service/node_modules/cookie-signature/Readme.md b/scraper-service/node_modules/cookie-signature/Readme.md new file mode 100644 index 0000000..2559e84 --- /dev/null +++ b/scraper-service/node_modules/cookie-signature/Readme.md @@ -0,0 +1,42 @@ + +# cookie-signature + + Sign and unsign cookies. + +## Example + +```js +var cookie = require('cookie-signature'); + +var val = cookie.sign('hello', 'tobiiscool'); +val.should.equal('hello.DGDUkGlIkCzPz+C0B064FNgHdEjox7ch8tOBGslZ5QI'); + +var val = cookie.sign('hello', 'tobiiscool'); +cookie.unsign(val, 'tobiiscool').should.equal('hello'); +cookie.unsign(val, 'luna').should.be.false; +``` + +## License + +(The MIT License) + +Copyright (c) 2012 LearnBoost <tj@learnboost.com> + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +'Software'), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. \ No newline at end of file diff --git a/scraper-service/node_modules/cookie-signature/index.js b/scraper-service/node_modules/cookie-signature/index.js new file mode 100644 index 0000000..336d487 --- /dev/null +++ b/scraper-service/node_modules/cookie-signature/index.js @@ -0,0 +1,51 @@ +/** + * Module dependencies. + */ + +var crypto = require('crypto'); + +/** + * Sign the given `val` with `secret`. + * + * @param {String} val + * @param {String|NodeJS.ArrayBufferView|crypto.KeyObject} secret + * @return {String} + * @api private + */ + +exports.sign = function(val, secret){ + if ('string' !== typeof val) throw new TypeError("Cookie value must be provided as a string."); + if (null == secret) throw new TypeError("Secret key must be provided."); + return val + '.' + crypto + .createHmac('sha256', secret) + .update(val) + .digest('base64') + .replace(/\=+$/, ''); +}; + +/** + * Unsign and decode the given `val` with `secret`, + * returning `false` if the signature is invalid. + * + * @param {String} val + * @param {String|NodeJS.ArrayBufferView|crypto.KeyObject} secret + * @return {String|Boolean} + * @api private + */ + +exports.unsign = function(val, secret){ + if ('string' !== typeof val) throw new TypeError("Signed cookie string must be provided."); + if (null == secret) throw new TypeError("Secret key must be provided."); + var str = val.slice(0, val.lastIndexOf('.')) + , mac = exports.sign(str, secret); + + return sha1(mac) == sha1(val) ? str : false; +}; + +/** + * Private + */ + +function sha1(str){ + return crypto.createHash('sha1').update(str).digest('hex'); +} diff --git a/scraper-service/node_modules/cookie-signature/package.json b/scraper-service/node_modules/cookie-signature/package.json new file mode 100644 index 0000000..738487b --- /dev/null +++ b/scraper-service/node_modules/cookie-signature/package.json @@ -0,0 +1,18 @@ +{ + "name": "cookie-signature", + "version": "1.0.7", + "description": "Sign and unsign cookies", + "keywords": ["cookie", "sign", "unsign"], + "author": "TJ Holowaychuk ", + "license": "MIT", + "repository": { "type": "git", "url": "https://github.com/visionmedia/node-cookie-signature.git"}, + "dependencies": {}, + "devDependencies": { + "mocha": "*", + "should": "*" + }, + "scripts": { + "test": "mocha --require should --reporter spec" + }, + "main": "index" +} \ No newline at end of file diff --git a/scraper-service/node_modules/cookie/LICENSE b/scraper-service/node_modules/cookie/LICENSE new file mode 100644 index 0000000..058b6b4 --- /dev/null +++ b/scraper-service/node_modules/cookie/LICENSE @@ -0,0 +1,24 @@ +(The MIT License) + +Copyright (c) 2012-2014 Roman Shtylman +Copyright (c) 2015 Douglas Christopher Wilson + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +'Software'), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + diff --git a/scraper-service/node_modules/cookie/README.md b/scraper-service/node_modules/cookie/README.md new file mode 100644 index 0000000..71fdac1 --- /dev/null +++ b/scraper-service/node_modules/cookie/README.md @@ -0,0 +1,317 @@ +# cookie + +[![NPM Version][npm-version-image]][npm-url] +[![NPM Downloads][npm-downloads-image]][npm-url] +[![Node.js Version][node-image]][node-url] +[![Build Status][ci-image]][ci-url] +[![Coverage Status][coveralls-image]][coveralls-url] + +Basic HTTP cookie parser and serializer for HTTP servers. + +## Installation + +This is a [Node.js](https://nodejs.org/en/) module available through the +[npm registry](https://www.npmjs.com/). Installation is done using the +[`npm install` command](https://docs.npmjs.com/getting-started/installing-npm-packages-locally): + +```sh +$ npm install cookie +``` + +## API + +```js +var cookie = require('cookie'); +``` + +### cookie.parse(str, options) + +Parse an HTTP `Cookie` header string and returning an object of all cookie name-value pairs. +The `str` argument is the string representing a `Cookie` header value and `options` is an +optional object containing additional parsing options. + +```js +var cookies = cookie.parse('foo=bar; equation=E%3Dmc%5E2'); +// { foo: 'bar', equation: 'E=mc^2' } +``` + +#### Options + +`cookie.parse` accepts these properties in the options object. + +##### decode + +Specifies a function that will be used to decode a cookie's value. Since the value of a cookie +has a limited character set (and must be a simple string), this function can be used to decode +a previously-encoded cookie value into a JavaScript string or other object. + +The default function is the global `decodeURIComponent`, which will decode any URL-encoded +sequences into their byte representations. + +**note** if an error is thrown from this function, the original, non-decoded cookie value will +be returned as the cookie's value. + +### cookie.serialize(name, value, options) + +Serialize a cookie name-value pair into a `Set-Cookie` header string. The `name` argument is the +name for the cookie, the `value` argument is the value to set the cookie to, and the `options` +argument is an optional object containing additional serialization options. + +```js +var setCookie = cookie.serialize('foo', 'bar'); +// foo=bar +``` + +#### Options + +`cookie.serialize` accepts these properties in the options object. + +##### domain + +Specifies the value for the [`Domain` `Set-Cookie` attribute][rfc-6265-5.2.3]. By default, no +domain is set, and most clients will consider the cookie to apply to only the current domain. + +##### encode + +Specifies a function that will be used to encode a cookie's value. Since value of a cookie +has a limited character set (and must be a simple string), this function can be used to encode +a value into a string suited for a cookie's value. + +The default function is the global `encodeURIComponent`, which will encode a JavaScript string +into UTF-8 byte sequences and then URL-encode any that fall outside of the cookie range. + +##### expires + +Specifies the `Date` object to be the value for the [`Expires` `Set-Cookie` attribute][rfc-6265-5.2.1]. +By default, no expiration is set, and most clients will consider this a "non-persistent cookie" and +will delete it on a condition like exiting a web browser application. + +**note** the [cookie storage model specification][rfc-6265-5.3] states that if both `expires` and +`maxAge` are set, then `maxAge` takes precedence, but it is possible not all clients by obey this, +so if both are set, they should point to the same date and time. + +##### httpOnly + +Specifies the `boolean` value for the [`HttpOnly` `Set-Cookie` attribute][rfc-6265-5.2.6]. When truthy, +the `HttpOnly` attribute is set, otherwise it is not. By default, the `HttpOnly` attribute is not set. + +**note** be careful when setting this to `true`, as compliant clients will not allow client-side +JavaScript to see the cookie in `document.cookie`. + +##### maxAge + +Specifies the `number` (in seconds) to be the value for the [`Max-Age` `Set-Cookie` attribute][rfc-6265-5.2.2]. +The given number will be converted to an integer by rounding down. By default, no maximum age is set. + +**note** the [cookie storage model specification][rfc-6265-5.3] states that if both `expires` and +`maxAge` are set, then `maxAge` takes precedence, but it is possible not all clients by obey this, +so if both are set, they should point to the same date and time. + +##### partitioned + +Specifies the `boolean` value for the [`Partitioned` `Set-Cookie`](rfc-cutler-httpbis-partitioned-cookies) +attribute. When truthy, the `Partitioned` attribute is set, otherwise it is not. By default, the +`Partitioned` attribute is not set. + +**note** This is an attribute that has not yet been fully standardized, and may change in the future. +This also means many clients may ignore this attribute until they understand it. + +More information about can be found in [the proposal](https://github.com/privacycg/CHIPS). + +##### path + +Specifies the value for the [`Path` `Set-Cookie` attribute][rfc-6265-5.2.4]. By default, the path +is considered the ["default path"][rfc-6265-5.1.4]. + +##### priority + +Specifies the `string` to be the value for the [`Priority` `Set-Cookie` attribute][rfc-west-cookie-priority-00-4.1]. + + - `'low'` will set the `Priority` attribute to `Low`. + - `'medium'` will set the `Priority` attribute to `Medium`, the default priority when not set. + - `'high'` will set the `Priority` attribute to `High`. + +More information about the different priority levels can be found in +[the specification][rfc-west-cookie-priority-00-4.1]. + +**note** This is an attribute that has not yet been fully standardized, and may change in the future. +This also means many clients may ignore this attribute until they understand it. + +##### sameSite + +Specifies the `boolean` or `string` to be the value for the [`SameSite` `Set-Cookie` attribute][rfc-6265bis-09-5.4.7]. + + - `true` will set the `SameSite` attribute to `Strict` for strict same site enforcement. + - `false` will not set the `SameSite` attribute. + - `'lax'` will set the `SameSite` attribute to `Lax` for lax same site enforcement. + - `'none'` will set the `SameSite` attribute to `None` for an explicit cross-site cookie. + - `'strict'` will set the `SameSite` attribute to `Strict` for strict same site enforcement. + +More information about the different enforcement levels can be found in +[the specification][rfc-6265bis-09-5.4.7]. + +**note** This is an attribute that has not yet been fully standardized, and may change in the future. +This also means many clients may ignore this attribute until they understand it. + +##### secure + +Specifies the `boolean` value for the [`Secure` `Set-Cookie` attribute][rfc-6265-5.2.5]. When truthy, +the `Secure` attribute is set, otherwise it is not. By default, the `Secure` attribute is not set. + +**note** be careful when setting this to `true`, as compliant clients will not send the cookie back to +the server in the future if the browser does not have an HTTPS connection. + +## Example + +The following example uses this module in conjunction with the Node.js core HTTP server +to prompt a user for their name and display it back on future visits. + +```js +var cookie = require('cookie'); +var escapeHtml = require('escape-html'); +var http = require('http'); +var url = require('url'); + +function onRequest(req, res) { + // Parse the query string + var query = url.parse(req.url, true, true).query; + + if (query && query.name) { + // Set a new cookie with the name + res.setHeader('Set-Cookie', cookie.serialize('name', String(query.name), { + httpOnly: true, + maxAge: 60 * 60 * 24 * 7 // 1 week + })); + + // Redirect back after setting cookie + res.statusCode = 302; + res.setHeader('Location', req.headers.referer || '/'); + res.end(); + return; + } + + // Parse the cookies on the request + var cookies = cookie.parse(req.headers.cookie || ''); + + // Get the visitor name set in the cookie + var name = cookies.name; + + res.setHeader('Content-Type', 'text/html; charset=UTF-8'); + + if (name) { + res.write('

Welcome back, ' + escapeHtml(name) + '!

'); + } else { + res.write('

Hello, new visitor!

'); + } + + res.write('
'); + res.write(' '); + res.end('
'); +} + +http.createServer(onRequest).listen(3000); +``` + +## Testing + +```sh +$ npm test +``` + +## Benchmark + +``` +$ npm run bench + +> cookie@0.5.0 bench +> node benchmark/index.js + + node@18.18.2 + acorn@8.10.0 + ada@2.6.0 + ares@1.19.1 + brotli@1.0.9 + cldr@43.1 + icu@73.2 + llhttp@6.0.11 + modules@108 + napi@9 + nghttp2@1.57.0 + nghttp3@0.7.0 + ngtcp2@0.8.1 + openssl@3.0.10+quic + simdutf@3.2.14 + tz@2023c + undici@5.26.3 + unicode@15.0 + uv@1.44.2 + uvwasi@0.0.18 + v8@10.2.154.26-node.26 + zlib@1.2.13.1-motley + +> node benchmark/parse-top.js + + cookie.parse - top sites + + 14 tests completed. + + parse accounts.google.com x 2,588,913 ops/sec ±0.74% (186 runs sampled) + parse apple.com x 2,370,002 ops/sec ±0.69% (186 runs sampled) + parse cloudflare.com x 2,213,102 ops/sec ±0.88% (188 runs sampled) + parse docs.google.com x 2,194,157 ops/sec ±1.03% (184 runs sampled) + parse drive.google.com x 2,265,084 ops/sec ±0.79% (187 runs sampled) + parse en.wikipedia.org x 457,099 ops/sec ±0.81% (186 runs sampled) + parse linkedin.com x 504,407 ops/sec ±0.89% (186 runs sampled) + parse maps.google.com x 1,230,959 ops/sec ±0.98% (186 runs sampled) + parse microsoft.com x 926,294 ops/sec ±0.88% (184 runs sampled) + parse play.google.com x 2,311,338 ops/sec ±0.83% (185 runs sampled) + parse support.google.com x 1,508,850 ops/sec ±0.86% (186 runs sampled) + parse www.google.com x 1,022,582 ops/sec ±1.32% (182 runs sampled) + parse youtu.be x 332,136 ops/sec ±1.02% (185 runs sampled) + parse youtube.com x 323,833 ops/sec ±0.77% (183 runs sampled) + +> node benchmark/parse.js + + cookie.parse - generic + + 6 tests completed. + + simple x 3,214,032 ops/sec ±1.61% (183 runs sampled) + decode x 587,237 ops/sec ±1.16% (187 runs sampled) + unquote x 2,954,618 ops/sec ±1.35% (183 runs sampled) + duplicates x 857,008 ops/sec ±0.89% (187 runs sampled) + 10 cookies x 292,133 ops/sec ±0.89% (187 runs sampled) + 100 cookies x 22,610 ops/sec ±0.68% (187 runs sampled) +``` + +## References + +- [RFC 6265: HTTP State Management Mechanism][rfc-6265] +- [Same-site Cookies][rfc-6265bis-09-5.4.7] + +[rfc-cutler-httpbis-partitioned-cookies]: https://tools.ietf.org/html/draft-cutler-httpbis-partitioned-cookies/ +[rfc-west-cookie-priority-00-4.1]: https://tools.ietf.org/html/draft-west-cookie-priority-00#section-4.1 +[rfc-6265bis-09-5.4.7]: https://tools.ietf.org/html/draft-ietf-httpbis-rfc6265bis-09#section-5.4.7 +[rfc-6265]: https://tools.ietf.org/html/rfc6265 +[rfc-6265-5.1.4]: https://tools.ietf.org/html/rfc6265#section-5.1.4 +[rfc-6265-5.2.1]: https://tools.ietf.org/html/rfc6265#section-5.2.1 +[rfc-6265-5.2.2]: https://tools.ietf.org/html/rfc6265#section-5.2.2 +[rfc-6265-5.2.3]: https://tools.ietf.org/html/rfc6265#section-5.2.3 +[rfc-6265-5.2.4]: https://tools.ietf.org/html/rfc6265#section-5.2.4 +[rfc-6265-5.2.5]: https://tools.ietf.org/html/rfc6265#section-5.2.5 +[rfc-6265-5.2.6]: https://tools.ietf.org/html/rfc6265#section-5.2.6 +[rfc-6265-5.3]: https://tools.ietf.org/html/rfc6265#section-5.3 + +## License + +[MIT](LICENSE) + +[ci-image]: https://badgen.net/github/checks/jshttp/cookie/master?label=ci +[ci-url]: https://github.com/jshttp/cookie/actions/workflows/ci.yml +[coveralls-image]: https://badgen.net/coveralls/c/github/jshttp/cookie/master +[coveralls-url]: https://coveralls.io/r/jshttp/cookie?branch=master +[node-image]: https://badgen.net/npm/node/cookie +[node-url]: https://nodejs.org/en/download +[npm-downloads-image]: https://badgen.net/npm/dm/cookie +[npm-url]: https://npmjs.org/package/cookie +[npm-version-image]: https://badgen.net/npm/v/cookie diff --git a/scraper-service/node_modules/cookie/SECURITY.md b/scraper-service/node_modules/cookie/SECURITY.md new file mode 100644 index 0000000..fd4a6c5 --- /dev/null +++ b/scraper-service/node_modules/cookie/SECURITY.md @@ -0,0 +1,25 @@ +# Security Policies and Procedures + +## Reporting a Bug + +The `cookie` team and community take all security bugs seriously. Thank +you for improving the security of the project. We appreciate your efforts and +responsible disclosure and will make every effort to acknowledge your +contributions. + +Report security bugs by emailing the current owner(s) of `cookie`. This +information can be found in the npm registry using the command +`npm owner ls cookie`. +If unsure or unable to get the information from the above, open an issue +in the [project issue tracker](https://github.com/jshttp/cookie/issues) +asking for the current contact information. + +To ensure the timely response to your report, please ensure that the entirety +of the report is contained within the email body and not solely behind a web +link or an attachment. + +At least one owner will acknowledge your email within 48 hours, and will send a +more detailed response within 48 hours indicating the next steps in handling +your report. After the initial reply to your report, the owners will +endeavor to keep you informed of the progress towards a fix and full +announcement, and may ask for additional information or guidance. diff --git a/scraper-service/node_modules/cookie/index.js b/scraper-service/node_modules/cookie/index.js new file mode 100644 index 0000000..acd5acd --- /dev/null +++ b/scraper-service/node_modules/cookie/index.js @@ -0,0 +1,335 @@ +/*! + * cookie + * Copyright(c) 2012-2014 Roman Shtylman + * Copyright(c) 2015 Douglas Christopher Wilson + * MIT Licensed + */ + +'use strict'; + +/** + * Module exports. + * @public + */ + +exports.parse = parse; +exports.serialize = serialize; + +/** + * Module variables. + * @private + */ + +var __toString = Object.prototype.toString +var __hasOwnProperty = Object.prototype.hasOwnProperty + +/** + * RegExp to match cookie-name in RFC 6265 sec 4.1.1 + * This refers out to the obsoleted definition of token in RFC 2616 sec 2.2 + * which has been replaced by the token definition in RFC 7230 appendix B. + * + * cookie-name = token + * token = 1*tchar + * tchar = "!" / "#" / "$" / "%" / "&" / "'" / + * "*" / "+" / "-" / "." / "^" / "_" / + * "`" / "|" / "~" / DIGIT / ALPHA + */ + +var cookieNameRegExp = /^[!#$%&'*+\-.^_`|~0-9A-Za-z]+$/; + +/** + * RegExp to match cookie-value in RFC 6265 sec 4.1.1 + * + * cookie-value = *cookie-octet / ( DQUOTE *cookie-octet DQUOTE ) + * cookie-octet = %x21 / %x23-2B / %x2D-3A / %x3C-5B / %x5D-7E + * ; US-ASCII characters excluding CTLs, + * ; whitespace DQUOTE, comma, semicolon, + * ; and backslash + */ + +var cookieValueRegExp = /^("?)[\u0021\u0023-\u002B\u002D-\u003A\u003C-\u005B\u005D-\u007E]*\1$/; + +/** + * RegExp to match domain-value in RFC 6265 sec 4.1.1 + * + * domain-value = + * ; defined in [RFC1034], Section 3.5, as + * ; enhanced by [RFC1123], Section 2.1 + * =