webcrawl 1.2.3 → 1.2.5

package/dist/core/extract/business.js

@@ -0,0 +1,344 @@
+import { extractDomainHost } from "./currency.js";
+import { normalizeIdentity, resultDedupeKey } from "./dedupe-key.js";
+import { resolveAbsoluteUrl } from "./real-estate.js";
+const BUSINESS_SCHEMA_TYPES = new Set([
+    "localbusiness",
+    "organization",
+    "corporation",
+    "restaurant",
+    "foodestablishment",
+    "cafeorcoffeeshop",
+    "bakery",
+    "barorpub",
+    "store",
+    "autodealer",
+    "autorepair",
+    "medicalbusiness",
+    "dentist",
+    "physician",
+    "pharmacy",
+    "hospital",
+    "healthandbeautybusiness",
+    "dayspa",
+    "hairsalon",
+    "legalservice",
+    "attorney",
+    "accountingservice",
+    "financialservice",
+    "insuranceagency",
+    "realestateagent",
+    "homeandconstructionbusiness",
+    "plumber",
+    "electrician",
+    "hvacbusiness",
+    "generalcontractor",
+    "locksmith",
+    "movingcompany",
+    "professionalservice",
+    "employmentagency",
+    "travelagency",
+    "sportsactivitylocation",
+    "exerciseGym",
+]);
+function isBusinessSchemaType(rawType) {
+    const clean = rawType.toLowerCase().trim();
+    if (BUSINESS_SCHEMA_TYPES.has(clean)) {
+        return true;
+    }
+    return (clean.endsWith("business") ||
+        clean.endsWith("store") ||
+        clean.endsWith("service") ||
+        clean.endsWith("shop") ||
+        clean.endsWith("clinic") ||
+        clean.endsWith("agency") ||
+        clean.endsWith("contractor"));
+}
+function parseAddressObject(addr) {
+    if (!addr) {
+        return {};
+    }
+    if (typeof addr === "string") {
+        return { address: addr.trim() };
+    }
+    if (typeof addr === "object") {
+        const a = addr;
+        const street = typeof a.streetAddress === "string"
+            ? a.streetAddress.trim()
+            : typeof a.address1 === "string"
+                ? a.address1.trim()
+                : "";
+        const city = typeof a.addressLocality === "string"
+            ? a.addressLocality.trim()
+            : typeof a.city === "string"
+                ? a.city.trim()
+                : "";
+        const state = typeof a.addressRegion === "string"
+            ? a.addressRegion.trim()
+            : typeof a.state === "string"
+                ? a.state.trim()
+                : "";
+        const postcode = typeof a.postalCode === "string" || typeof a.postalCode === "number"
+            ? String(a.postalCode).trim()
+            : typeof a.zipCode === "string"
+                ? a.zipCode.trim()
+                : "";
+        const country = typeof a.addressCountry === "string"
+            ? a.addressCountry.trim()
+            : typeof a.addressCountry
+                ?.name === "string"
+                ? String(a.addressCountry.name).trim()
+                : undefined;
+        const full = [street, city, state, postcode].filter(Boolean).join(", ");
+        return {
+            address: full ||
+                (typeof a.formattedAddress === "string"
+                    ? a.formattedAddress
+                    : undefined),
+            city: city || undefined,
+            state: state || undefined,
+            postcode: postcode || undefined,
+            country,
+        };
+    }
+    return {};
+}
+function walkBusinessJsonNode(node, pageUrl, portal, countryCode, extractedVia, out, depth = 0) {
+    if (depth > 12 || out.length >= 45 || !node || typeof node !== "object") {
+        return;
+    }
+    if (Array.isArray(node)) {
+        for (const item of node) {
+            walkBusinessJsonNode(item, pageUrl, portal, countryCode, extractedVia, out, depth + 1);
+        }
+        return;
+    }
+    const obj = node;
+    const rawTypeVal = Array.isArray(obj["@type"])
+        ? String(obj["@type"][0] || "")
+        : String(obj["@type"] || obj.type || "");
+    const name = typeof obj.name === "string"
+        ? obj.name.trim()
+        : typeof obj.businessName === "string"
+            ? obj.businessName.trim()
+            : typeof obj.companyName === "string"
+                ? obj.companyName.trim()
+                : "";
+    const phone = typeof obj.telephone === "string"
+        ? obj.telephone.trim()
+        : typeof obj.phone === "string"
+            ? obj.phone.trim()
+            : typeof obj.displayPhone === "string"
+                ? obj.displayPhone.trim()
+                : undefined;
+    const email = typeof obj.email === "string"
+        ? obj.email.replace(/^mailto:/i, "").trim()
+        : undefined;
+    const addrInfo = parseAddressObject(obj.address || obj.location);
+    const aggRating = obj.aggregateRating;
+    const ratingRaw = aggRating?.ratingValue ?? obj.rating ?? obj.averageRating;
+    const ratingNum = typeof ratingRaw === "number" || typeof ratingRaw === "string"
+        ? Number.parseFloat(String(ratingRaw))
+        : undefined;
+    const reviewCountRaw = aggRating?.reviewCount ??
+        aggRating?.ratingCount ??
+        obj.reviewCount ??
+        obj.numberOfReviews;
+    const reviewCountNum = typeof reviewCountRaw === "number" || typeof reviewCountRaw === "string"
+        ? Number.parseInt(String(reviewCountRaw).replace(/,/g, ""), 10)
+        : undefined;
+    const isGenericOrgOnly = /^(organization|corporation|brand|newsmediaorganization)$/i.test(rawTypeVal);
+    const hasBusinessSignals = (isBusinessSchemaType(rawTypeVal) &&
+        (!isGenericOrgOnly ||
+            Boolean(addrInfo.address || phone || ratingNum !== undefined))) ||
+        Boolean(name.length >= 2 &&
+            name.length <= 120 &&
+            (phone ||
+                (addrInfo.address && (ratingNum !== undefined || obj.priceRange)) ||
+                (obj.businessId !== undefined && addrInfo.address)));
+    // Skip generic site-level WebSite / SearchAction / BreadcrumbList nodes
+    if (hasBusinessSignals &&
+        name &&
+        !/^(website|webpage|breadcrumblist|searchaction|itemlist|faqpage)$/i.test(rawTypeVal)) {
+        const geoObj = obj.geo;
+        const lat = typeof geoObj?.latitude === "number"
+            ? geoObj.latitude
+            : typeof obj.latitude === "number"
+                ? obj.latitude
+                : undefined;
+        const lng = typeof geoObj?.longitude === "number"
+            ? geoObj.longitude
+            : typeof obj.longitude === "number"
+                ? obj.longitude
+                : undefined;
+        let category = typeof obj.category === "string"
+            ? obj.category
+            : Array.isArray(obj.categories) && typeof obj.categories[0] === "string"
+                ? obj.categories[0]
+                : typeof obj.servesCuisine === "string"
+                    ? obj.servesCuisine
+                    : rawTypeVal && !/^(organization|localbusiness)$/i.test(rawTypeVal)
+                        ? rawTypeVal
+                        : undefined;
+        if (Array.isArray(obj.categories) &&
+            obj.categories[0] &&
+            typeof obj.categories[0] === "object") {
+            const firstCat = obj.categories[0];
+            if (typeof firstCat.title === "string") {
+                category = firstCat.title;
+            }
+            else if (typeof firstCat.name === "string") {
+                category = firstCat.name;
+            }
+        }
+        let openingHours;
+        if (typeof obj.openingHours === "string") {
+            openingHours = obj.openingHours;
+        }
+        else if (Array.isArray(obj.openingHours)) {
+            openingHours = obj.openingHours
+                .filter((h) => typeof h === "string")
+                .slice(0, 7);
+        }
+        const foundingDate = typeof obj.foundingDate === "string" ||
+            typeof obj.foundingDate === "number"
+            ? String(obj.foundingDate)
+            : undefined;
+        const foundedMatch = foundingDate?.match(/\b(18\d{2}|19\d{2}|20\d{2})\b/);
+        const itemUrl = typeof obj.url === "string"
+            ? resolveAbsoluteUrl(obj.url, pageUrl)
+            : pageUrl;
+        if (addrInfo.address ||
+            phone ||
+            email ||
+            ratingNum !== undefined ||
+            typeof obj.description === "string") {
+            out.push({
+                name,
+                category,
+                description: typeof obj.description === "string"
+                    ? obj.description.trim().slice(0, 280)
+                    : undefined,
+                address: addrInfo.address,
+                city: addrInfo.city,
+                state: addrInfo.state,
+                postcode: addrInfo.postcode,
+                country: addrInfo.country || countryCode,
+                phone,
+                email,
+                website: itemUrl,
+                rating: ratingNum && ratingNum > 0 && ratingNum <= 10
+                    ? Number(ratingNum.toFixed(1))
+                    : undefined,
+                reviewCount: reviewCountNum &&
+                    Number.isFinite(reviewCountNum) &&
+                    reviewCountNum > 0
+                    ? reviewCountNum
+                    : undefined,
+                priceRange: typeof obj.priceRange === "string"
+                    ? obj.priceRange.trim()
+                    : undefined,
+                openingHours,
+                foundedYear: foundedMatch
+                    ? Number.parseInt(foundedMatch[1], 10)
+                    : undefined,
+                latitude: lat,
+                longitude: lng,
+                url: itemUrl,
+                sourceDomain: extractDomainHost(itemUrl) || portal,
+                extractedVia,
+            });
+        }
+    }
+    for (const val of Object.values(obj)) {
+        if (val && typeof val === "object") {
+            walkBusinessJsonNode(val, pageUrl, portal, countryCode, extractedVia, out, depth + 1);
+        }
+    }
+}
+export function extractBusinessListingsFromHtml(html, pageUrl, countryCode) {
+    const out = [];
+    const portal = extractDomainHost(pageUrl) || "unknown";
+    // Tier 1: Schema.org JSON-LD
+    const jsonLdRegex = /<script[^>]+type=["']application\/ld\+json["'][^>]*>([\s\S]*?)<\/script>/gi;
+    let match;
+    while ((match = jsonLdRegex.exec(html)) !== null) {
+        try {
+            const parsed = JSON.parse(match[1].trim());
+            walkBusinessJsonNode(parsed, pageUrl, portal, countryCode, "json-ld", out);
+        }
+        catch {
+            // Ignore malformed JSON-LD
+        }
+    }
+    // Tier 2: Embedded JSON state (__NEXT_DATA__, application/json)
+    if (out.length < 3) {
+        const stateRegex = /<script[^>]+(?:id=["']__NEXT_DATA__["']|type=["']application\/json["'])[^>]*>([\s\S]*?)<\/script>/gi;
+        while ((match = stateRegex.exec(html)) !== null) {
+            try {
+                const raw = match[1].trim();
+                if (raw.length > 20 && raw.length < 2_000_000) {
+                    const parsed = JSON.parse(raw);
+                    walkBusinessJsonNode(parsed, pageUrl, portal, countryCode, "json-state", out);
+                }
+            }
+            catch {
+                // Ignore
+            }
+        }
+    }
+    return dedupeBusinessListings(out);
+}
+export function extractBusinessListingFromSerpResult(title, description, url, countryCode) {
+    const combined = `${title} ${description}`;
+    const cleanName = title.replace(/\s*[|\-–\u2014•:]\s*.*$/, "").trim();
+    if (!cleanName ||
+        cleanName.length < 2 ||
+        /\b(the\s+\d+\s+best|best\s+\d+|top\s+\d+|best\s+.*\s+in\b|places?\s+to\b|where\s+to\b|guide\s+to\b|directory\b|yellow\s+pages|yelp|tripadvisor|timeout|broadsheet|urban\s+list)\b/i.test(cleanName)) {
+        return null;
+    }
+    const phoneMatch = combined.match(/(?:\+?\d{1,3}[\s.-]?)?\(?\d{2,4}\)?[\s.-]\d{3,4}[\s.-]\d{3,4}\b/);
+    const ratingMatch = combined.match(/\b([1-5]\.\d)\s*(?:\/\s*5|stars?|★)/i) ||
+        combined.match(/\brating[:\s]+([1-5]\.\d)\b/i);
+    const reviewMatch = combined.match(/\b(\d[\d,]{0,6})\s+reviews?\b/i);
+    const priceRangeMatch = combined.match(/(?:^|\s)(\${1,4})(?:\s|[·•,]|$)/);
+    const addrMatch = description.match(/\b(\d{1,5}\s+[A-Z][a-zA-Z0-9\s.]{2,30}\s+(?:St|Street|Rd|Road|Ave|Avenue|Blvd|Boulevard|Dr|Drive|Ln|Lane|Way|Pl|Place|Pde|Parade|Hwy|Highway)(?:,\s*[A-Za-z\s]+)?)/);
+    // Require at least one concrete business attribute (address, phone, or rating) in the SERP snippet
+    if (!addrMatch && !phoneMatch && !ratingMatch) {
+        return null;
+    }
+    const sourceDomain = extractDomainHost(url) || "unknown";
+    return {
+        name: cleanName,
+        description: description.trim().slice(0, 240) || undefined,
+        address: addrMatch ? addrMatch[1].trim() : undefined,
+        country: countryCode,
+        phone: phoneMatch ? phoneMatch[0].trim() : undefined,
+        website: url,
+        rating: ratingMatch ? Number.parseFloat(ratingMatch[1]) : undefined,
+        reviewCount: reviewMatch
+            ? Number.parseInt(reviewMatch[1].replace(/,/g, ""), 10)
+            : undefined,
+        priceRange: priceRangeMatch ? priceRangeMatch[1] : undefined,
+        url,
+        sourceDomain,
+        extractedVia: "serp-snippet",
+    };
+}
+export function dedupeBusinessListings(businesses, maxItems = 30) {
+    const seen = new Set();
+    const out = [];
+    for (const b of businesses) {
+        const normName = normalizeIdentity(b.name);
+        // Same name on different directory pages / URLs is kept; only exact repeats dropped.
+        const key = resultDedupeKey(b.url || b.website, normName);
+        if (!normName || normName.length < 2 || seen.has(key)) {
+            continue;
+        }
+        seen.add(key);
+        out.push(b);
+        if (out.length >= maxItems) {
+            break;
+        }
+    }
+    return out;
+}

package/dist/core/extract/currency.d.ts

@@ -0,0 +1,15 @@
+/**
+ * Derive ISO 4217 currency code from a raw price string and/or 2-letter ISO country code.
+ * Uses standard currency symbols and ISO country conventions without hardcoded retailer lists.
+ */
+export declare function resolveCurrencyCode(rawPriceOrCurrency?: string, countryCode?: string): string;
+/**
+ * Parse a numeric price and currency from a human price string such as
+ * "$360.41", "A$399.00", "£173.49", "1.299,00 €", "₹1,19,900", "₹1.45 Crores", "S$429", "7.490.000 ₫".
+ */
+export declare function parsePriceString(raw: string | number | undefined, countryCode?: string, explicitCurrency?: string, maxAllowedPrice?: number): {
+    price: number;
+    rawPrice: string;
+    currency: string;
+} | null;
+export declare function extractDomainHost(url: string): string;

package/dist/core/extract/currency.js

@@ -0,0 +1,259 @@
+const VALID_ISO_CURRENCIES = new Set([
+    "USD",
+    "AUD",
+    "GBP",
+    "EUR",
+    "CAD",
+    "NZD",
+    "SGD",
+    "HKD",
+    "JPY",
+    "CNY",
+    "INR",
+    "CHF",
+    "SEK",
+    "NOK",
+    "DKK",
+    "ZAR",
+    "BRL",
+    "MXN",
+    "AED",
+    "SAR",
+    "THB",
+    "MYR",
+    "IDR",
+    "PHP",
+    "VND",
+    "KRW",
+    "PLN",
+    "TRY",
+    "TWD",
+]);
+/**
+ * Derive ISO 4217 currency code from a raw price string and/or 2-letter ISO country code.
+ * Uses standard currency symbols and ISO country conventions without hardcoded retailer lists.
+ */
+export function resolveCurrencyCode(rawPriceOrCurrency, countryCode) {
+    const s = (rawPriceOrCurrency || "").trim();
+    const isoMatches = s.match(/\b([A-Z]{3})\b/g);
+    if (isoMatches) {
+        for (const code of isoMatches) {
+            if (VALID_ISO_CURRENCIES.has(code)) {
+                return code;
+            }
+        }
+    }
+    if (/A\$|AU\$/i.test(s)) {
+        return "AUD";
+    }
+    if (/NZ\$/i.test(s)) {
+        return "NZD";
+    }
+    if (/CA\$|C\$/i.test(s)) {
+        return "CAD";
+    }
+    if (/S\$|SG\$/i.test(s)) {
+        return "SGD";
+    }
+    if (/HK\$/i.test(s)) {
+        return "HKD";
+    }
+    if (/NT\$/i.test(s)) {
+        return "TWD";
+    }
+    if (s.includes("£")) {
+        return "GBP";
+    }
+    if (s.includes("€")) {
+        return "EUR";
+    }
+    if (s.includes("¥")) {
+        return (countryCode || "").toUpperCase() === "CN" ? "CNY" : "JPY";
+    }
+    if (s.includes("₹") ||
+        /\bRs\.?(?:\s|\d)/i.test(s) ||
+        /\b(?:crores?|cr|lakhs?|lacs?)\b/i.test(s)) {
+        return "INR";
+    }
+    if (s.includes("₫") || /\b(?:đồng|vnd|triệu|tỷ)\b/i.test(s)) {
+        return "VND";
+    }
+    if (s.includes("฿")) {
+        return "THB";
+    }
+    if (s.includes("₱")) {
+        return "PHP";
+    }
+    if (s.includes("₩")) {
+        return "KRW";
+    }
+    if (/\bzł\b/i.test(s)) {
+        return "PLN";
+    }
+    if (/\bRM\s*\d/i.test(s)) {
+        return "MYR";
+    }
+    if (/\bRp\.?\s*\d/i.test(s)) {
+        return "IDR";
+    }
+    const cc = (countryCode || "").trim().toUpperCase();
+    // Bare "$" in non-dollar countries (e.g. EU, GB, IN, VN, JP) indicates USD pricing
+    if (s.includes("$") &&
+        !["US", "AU", "CA", "NZ", "SG", "HK", "TW", "MX", ""].includes(cc)) {
+        return "USD";
+    }
+    const COUNTRY_TO_CURRENCY = {
+        AU: "AUD",
+        GB: "GBP",
+        UK: "GBP",
+        CA: "CAD",
+        NZ: "NZD",
+        SG: "SGD",
+        IN: "INR",
+        VN: "VND",
+        JP: "JPY",
+        CN: "CNY",
+        HK: "HKD",
+        TW: "TWD",
+        KR: "KRW",
+        TH: "THB",
+        MY: "MYR",
+        ID: "IDR",
+        PH: "PHP",
+        CH: "CHF",
+        SE: "SEK",
+        NO: "NOK",
+        DK: "DKK",
+        PL: "PLN",
+        CZ: "CZK",
+        HU: "HUF",
+        RO: "RON",
+        TR: "TRY",
+        AE: "AED",
+        SA: "SAR",
+        ZA: "ZAR",
+        BR: "BRL",
+        MX: "MXN",
+    };
+    if (COUNTRY_TO_CURRENCY[cc]) {
+        return COUNTRY_TO_CURRENCY[cc];
+    }
+    if ([
+        "DE",
+        "FR",
+        "IT",
+        "ES",
+        "NL",
+        "IE",
+        "AT",
+        "BE",
+        "PT",
+        "FI",
+        "GR",
+        "LU",
+        "SK",
+        "SI",
+        "EE",
+        "LV",
+        "LT",
+        "HR",
+        "CY",
+        "MT",
+        "EU",
+    ].includes(cc)) {
+        return "EUR";
+    }
+    return "USD";
+}
+/**
+ * Parse a numeric price and currency from a human price string such as
+ * "$360.41", "A$399.00", "£173.49", "1.299,00 €", "₹1,19,900", "₹1.45 Crores", "S$429", "7.490.000 ₫".
+ */
+export function parsePriceString(raw, countryCode, explicitCurrency, maxAllowedPrice = 1_000_000) {
+    if (typeof raw === "number") {
+        if (!Number.isFinite(raw) || raw <= 0) {
+            return null;
+        }
+        const numCurrency = resolveCurrencyCode(explicitCurrency, countryCode);
+        return {
+            price: Number(raw.toFixed(2)),
+            rawPrice: `${numCurrency} ${raw.toFixed(2)}`,
+            currency: numCurrency,
+        };
+    }
+    if (!raw || typeof raw !== "string") {
+        return null;
+    }
+    const cleaned = raw.replace(/\s+/g, " ").trim();
+    if (!cleaned) {
+        return null;
+    }
+    const currency = resolveCurrencyCode(explicitCurrency || cleaned, countryCode);
+    // High-denomination currencies (VND, IDR, KRW, JPY, INR) allow much higher nominal numeric caps
+    const highDenomMultiplier = currency === "VND" || currency === "IDR"
+        ? 25_000
+        : currency === "KRW" || currency === "JPY" || currency === "INR"
+            ? 500
+            : 1;
+    const effectiveMaxPrice = maxAllowedPrice * highDenomMultiplier;
+    // Handle Indian Crore / Lakh and Vietnamese tỷ / triệu multipliers
+    const unitMatch = cleaned.match(/(\d[\d,.]*)\s*(crores?|cr|lakhs?|lacs?|tỷ|triệu)\b/i);
+    if (unitMatch) {
+        const baseNum = Number.parseFloat(unitMatch[1].replace(/,/g, ""));
+        const unit = unitMatch[2].toLowerCase();
+        const mult = unit.startsWith("cr")
+            ? 10_000_000
+            : unit.startsWith("la")
+                ? 100_000
+                : unit === "tỷ"
+                    ? 1_000_000_000
+                    : 1_000_000;
+        const total = Math.round(baseNum * mult);
+        if (Number.isFinite(total) && total > 0 && total <= effectiveMaxPrice) {
+            return {
+                price: total,
+                rawPrice: cleaned,
+                currency,
+            };
+        }
+    }
+    const numMatch = cleaned.match(/(\d[\d,.]*)/);
+    if (!numMatch) {
+        return null;
+    }
+    let numStr = numMatch[1].replace(/[.,]$/, "");
+    // Handle multi-dot thousands (e.g. "7.490.000" in VND/EUR/IDR) and European decimal comma ("194,99" or "1.299,00")
+    if (/^\d{1,3}(?:\.\d{3}){2,}$/.test(numStr)) {
+        numStr = numStr.replace(/\./g, "");
+    }
+    else if (/^\d{1,3}\.\d{3}$/.test(numStr) &&
+        (currency === "VND" || currency === "IDR" || currency === "EUR")) {
+        numStr = numStr.replace(/\./g, "");
+    }
+    else if (/\.\d{3},\d{1,2}$/.test(numStr)) {
+        numStr = numStr.replace(/\./g, "").replace(",", ".");
+    }
+    else if (/^\d+,\d{1,2}$/.test(numStr)) {
+        numStr = numStr.replace(",", ".");
+    }
+    else {
+        numStr = numStr.replace(/,/g, "");
+    }
+    const value = Number.parseFloat(numStr);
+    if (!Number.isFinite(value) || value <= 0 || value > effectiveMaxPrice) {
+        return null;
+    }
+    return {
+        price: Number(value.toFixed(2)),
+        rawPrice: cleaned,
+        currency,
+    };
+}
+export function extractDomainHost(url) {
+    try {
+        return new URL(url).hostname.replace(/^www\./i, "").toLowerCase();
+    }
+    catch {
+        return "";
+    }
+}

package/dist/core/extract/cve-cache.d.ts

@@ -0,0 +1,62 @@
+import { type CveRecord } from "./cve.js";
+/** Only CVEs published (or numbered) in this year or later are stored. */
+export declare const CVE_STORE_SINCE = 2020;
+/** A cached keyword result is trusted for this long before NVD is asked again. */
+export declare const QUERY_TTL_MS: number;
+export interface CachedCves {
+    records: CveRecord[];
+    /** Ids served from cache that are due a background refresh. */
+    stale: string[];
+}
+export interface CveCacheStore {
+    get(ids: string[]): Promise<CachedCves>;
+    getQuery(keyword: string): Promise<string[] | null>;
+    putQuery(keyword: string, ids: string[]): Promise<void>;
+    search(keyword: string, limit: number): Promise<CveRecord[]>;
+    upsert(records: CveRecord[]): Promise<number>;
+}
+/** True when a record falls inside the stored window (published year, else id year). */
+export declare function isCacheableCve(r: CveRecord): boolean;
+export declare function isStale(fetchedAt: Date, published: string | undefined, now?: number): boolean;
+export declare function cveQueryKey(keyword: string): string;
+/** One row of the batched upsert; exported so the SQL shape is unit-testable. */
+export declare function cveRow(r: CveRecord): unknown[];
+/**
+ * INSERT … ON CONFLICT for `n` rows. An existing row is only replaced when the
+ * incoming record is at least as new (`modified`) and actually differs, so cache
+ * hits written back unchanged cost no row churn.
+ */
+export declare function buildUpsertSql(n: number): string;
+interface PgPoolLike {
+    query(sql: string, params?: unknown[]): Promise<{
+        rows: Array<Record<string, any>>;
+        rowCount?: number | null;
+    }>;
+}
+/**
+ * Registers an external Postgres pool & schema (e.g., Supabase Postgres in webcrawl.app cloud)
+ * when the engine is consumed as an installed npm package (`webcrawl/extract/cve-cache`).
+ */
+export declare function registerExternalCveDb(mods: {
+    getPgPool?: () => Promise<PgPoolLike | null>;
+    CVE_SCHEMA_SQL?: string;
+} | null): void;
+/** Swap the backing store (tests). Pass null to restore Postgres. */
+export declare function setCveCacheStoreForTests(store: CveCacheStore | null): void;
+/** Cached records for exact ids (CVE or alias such as GHSA). Never throws. */
+export declare function getCachedCves(ids: string[]): Promise<CachedCves>;
+/** Ids NVD returned for this keyword within QUERY_TTL_MS, or null. Never throws. */
+export declare function getCachedQuery(keyword: string): Promise<string[] | null>;
+export declare function rememberQuery(keyword: string, ids: string[]): void;
+/** Full-text search over cached rows; the fallback when NVD is down or rate limited. */
+export declare function searchCachedCves(keyword: string, limit: number): Promise<CveRecord[]>;
+/** Store records in the window; resolves to rows written (0 on any failure). */
+export declare function upsertCves(records: CveRecord[]): Promise<number>;
+/** Background write that outlives the response on Vercel (no-op elsewhere). */
+export declare function runInBackground(p: Promise<unknown>): void;
+/**
+ * Background refresh of stale ids: fetch live, merge over the cached copy (keeps
+ * OSV fixed versions the live NVD record lacks), write back.
+ */
+export declare function refreshStaleCves(stale: string[], cached: CveRecord[], fetchLive: (ids: string[]) => Promise<CveRecord[]>): void;
+export {};