webcrawl 1.2.3 → 1.2.5

package/dist/core/search/providers.d.ts

@@ -0,0 +1,62 @@
+import { type CveRecord, type ProductOffer } from "../extract/index.js";
+import type { RawCandidateDocument, UserLocationContext } from "../ranker/index.js";
+import { type SearchRequest } from "./catalog.js";
+/**
+ * Wikipedia only helps for entity / definition queries ("retrieval augmented
+ * generation", "what is rag"). For intent queries ("best X under 500",
+ * "A vs B", "how to ...") it injects off-topic pages (songs, unrelated topics).
+ */
+export declare function isEntityQuery(q: string): boolean;
+/**
+ * Stage 1: Hedged Multi-Engine Native Federation with Early-Exit Quorum.
+ * Fires all native zero-key search engines concurrently and exits as soon as a fast
+ * quorum (at least 2 engines + >= limit results + 80ms grace window) finishes,
+ * preventing any single slow engine from inflating tail latency.
+ */
+export declare function fetchHedgedNativeFederation(rawQuery: string, localizedQuery: string, perEngineLimit: number, req: SearchRequest, userLocation: UserLocationContext, quorumTimeoutMs: number, targetSites: string[], onOffer: (offer: ProductOffer) => void, onCve?: (cve: CveRecord) => void): Promise<RawCandidateDocument[]>;
+/** Below this many unique URLs from the free engines, the Jina fallback runs. */
+export declare const JINA_FALLBACK_MIN_URLS = 3;
+export declare function fetchJinaSearch(query: string, limit: number, req?: {
+    country?: string;
+    lang?: string;
+}): Promise<RawCandidateDocument[]>;
+/**
+ * 0A. Bing Shopping Native Provider (`bing-shopping`) via Impit Chrome TLS impersonation.
+ * Parses embedded `CustomData` JSON + `<li class="br-item">` seller/rating cards with zero API keys and zero headless browsers (~400ms).
+ * When `targetSites` is set (e.g. `["jbhifi.com.au", "amazon.com.au"]`), queries per-site in parallel so even bot-gated retailers return live prices and direct product URLs.
+ */
+export declare function fetchBingShoppingNative(rawQuery: string, limit: number, userLocation?: UserLocationContext, targetSites?: string[], onOffer?: (offer: ProductOffer) => void): Promise<RawCandidateDocument[]>;
+/**
+ * 0B. Google Shopping Provider (`google-shopping`).
+ * Tier 1: Serper `/shopping` endpoint when `SERPER_API_KEY` is configured.
+ * Tier 2 (Zero-key fallback): Shopping-intent Brave query ("<query> buy price") with snippet price extraction.
+ */
+export declare function fetchGoogleShoppingProvider(rawQuery: string, localizedQuery: string, limit: number, userLocation?: UserLocationContext, targetSites?: string[], onOffer?: (offer: ProductOffer) => void): Promise<RawCandidateDocument[]>;
+/**
+ * 1. Brave Search Native HTML parser via Impit Chrome TLS impersonation (`search.brave.com/search`).
+ * Independent web index with zero API keys required.
+ */
+export declare function fetchBraveNativeSerp(query: string, limit: number, userLocation?: UserLocationContext): Promise<RawCandidateDocument[]>;
+/**
+ * 3. Google News Search RSS (`news.google.com/rss/search`) with dynamic edition localization (`gl`, `hl`, `ceid`).
+ */
+export declare function fetchGoogleNewsSearchRss(query: string, limit: number, userLocation?: UserLocationContext, lang?: string, timeoutMs?: number): Promise<RawCandidateDocument[]>;
+/**
+ * 6. ArXiv API (`export.arxiv.org/api/query`) + OpenAlex Scholarly Works API
+ * for scientific, ML/AI, biomedical, and peer-reviewed research paper search.
+ */
+export declare function buildArxivSearchQuery(rawQuery: string): string;
+export declare function fetchArxivApi(query: string, limit: number): Promise<RawCandidateDocument[]>;
+export declare function nvdKeywordFromQuery(query: string): string;
+/**
+ * Exact CVE ids, cache first: hits come from Postgres (stale ones refresh in the
+ * background), and only misses go to NVD, one request each, capped at `cap`.
+ * Used for query ids and page-mentioned ids. Write-back happens in the pipeline
+ * once NVD and OSV records are merged.
+ */
+export declare function fetchNvdByIds(ids: string[], cap?: number): Promise<CveRecord[]>;
+/** "lodash npm vulnerabilities" -> { name: "lodash", ecosystem: "npm" } */
+export declare function osvPackageFromQuery(query: string): {
+    name: string;
+    ecosystem: string;
+} | null;