webcrawl 1.2.3 → 1.2.5

package/dist/core/engine/crawler.d.ts

@@ -0,0 +1,151 @@
+export interface MapWebsiteOptions {
+    /** Only crawl the exact root/target domain (default: true when includeSubdomains is false and selectedDomains is empty) */
+    onlyRootDomain?: boolean;
+    /** Also crawl subdomains of the allowed domain(s), e.g. docs.example.com (default: false) */
+    includeSubdomains?: boolean;
+    /** Explicit allowlist of domains/subdomains (supports exact hosts like "docs.example.com" or wildcards like "*.example.com") */
+    selectedDomains?: string[];
+    /** Maximum number of discovered URLs to return (default: 5000) */
+    limit?: number;
+    /** Maximum number of HTML pages to fetch during multi-hop BFS crawl when sitemap is missing (default: 250) */
+    maxCrawlPages?: number;
+    /** Alias for maxCrawlPages */
+    maxCrawl?: number;
+    /** Maximum BFS hop depth from the entry URL (default: 8) */
+    maxDepth?: number;
+    /** Only keep URLs whose path matches one of these (prefix, `*` wildcards, trailing `$`). */
+    includePaths?: string[];
+    /** Drop URLs whose path matches any of these. Wins over includePaths. */
+    excludePaths?: string[];
+    /** Stay under the start URL's path when no includePaths are given (default: true). */
+    restrictToSubpath?: boolean;
+    /** Sitemaps to read first (e.g. a site whose sitemap is not at /sitemap.xml). */
+    sitemapUrls?: string[];
+    /** Overall map timeout in milliseconds (default: 25000) */
+    timeoutMs?: number;
+    /** Bypass 24-hour map cache (default: false) */
+    bypassCache?: boolean;
+    /** When a bot check blocks plain requests, find pages through a real browser (default: true) */
+    stealth?: boolean;
+    /** Time budget for the stealth browser session (default: 75000) */
+    stealthBudgetMs?: number;
+    /** Allow the paid remote stealth provider (default true). Off for unbilled runs. */
+    remoteStealth?: boolean;
+}
+export declare function mapWebsiteUrls(targetUrl: string, options?: MapWebsiteOptions): Promise<{
+    success: boolean;
+    links: string[];
+    latencyMs: number;
+    source: string;
+    cached?: boolean;
+    cacheLayer?: "memory" | "disk" | "postgres" | "miss";
+    pagesCrawled?: number;
+    /** The site answered plain requests with a bot check. */
+    blocked?: boolean;
+    /** Pages were found through the stealth browser. */
+    stealth?: boolean;
+    /** What a remote stealth provider charged for discovery, in USD. */
+    stealthCostUsd?: number;
+    scope?: {
+        onlyRootDomain: boolean;
+        includeSubdomains: boolean;
+        selectedDomains: string[];
+        maxCrawlPages: number;
+        maxDepth: number;
+        includePaths: string[];
+        excludePaths: string[];
+    };
+}>;
+/**
+ * Ultra-fast HTTP status batch check for `websiteDeletedPages.ts`.
+ * Checks if URLs return 404/410 without downloading/rendering full markdown!
+ */
+export declare function checkUrlsStatusBatch(urls: string[], timeoutMs?: number): Promise<Array<{
+    url: string;
+    statusCode: number;
+    ok: boolean;
+}>>;
+/**
+ * Extracts structured `{ app_name, product_features }` (or custom JSON schema) from a URL.
+ * Replaces `firecrawl.asyncExtract` in `website-extract.ts`.
+ */
+export declare function extractWebsiteData(url: string, options?: {
+    prompt?: string;
+    schema?: Record<string, unknown>;
+    openaiApiKey?: string;
+    bypassCache?: boolean;
+    /** Already-extracted content (e.g. a parsed PDF); skips the HTML fetch and cache. */
+    markdown?: string;
+}): Promise<{
+    app_name: string;
+    product_features: string[];
+    cached?: boolean;
+    cacheLayer?: "memory" | "disk" | "postgres" | "miss";
+}>;
+/**
+ * Full-website multi-page crawler (`POST /v1/crawl`) backed by the shared 3-tier
+ * L1 Memory -> L2 Disk -> L3 Postgres URL store (`crawled_pages`).
+ * Pages already stored in Postgres for this website are served directly without re-crawling!
+ */
+export declare function crawlWebsitePages(targetUrl: string, options?: {
+    maxPages?: number;
+    includeSubdomains?: boolean;
+    onlyRootDomain?: boolean;
+    bypassCache?: boolean;
+    extractStructured?: boolean;
+    timeoutMs?: number;
+}): Promise<{
+    success: boolean;
+    rootUrl: string;
+    pagesCount: number;
+    cachedPagesCount: number;
+    postgresHits: number;
+    latencyMs: number;
+    pages: CrawledPage[];
+}>;
+export interface CrawledPage {
+    url: string;
+    title: string;
+    markdown: string;
+    linksCount: number;
+    cached: boolean;
+    cacheLayer: "memory" | "disk" | "postgres" | "miss";
+    structuredData?: Record<string, unknown> | null;
+    statusCode?: number;
+    error?: string;
+    /** The site served a bot check instead of this page. */
+    blocked?: boolean;
+    /** Read through the stealth browser. */
+    stealth?: boolean;
+    /** Remote provider that hosted the stealth browser (billed with a surcharge). */
+    stealthProvider?: string;
+}
+/** True for markdown converted from a bot-check page (old cache entries). */
+export declare function isChallengeMarkdown(title: string, markdown: string): boolean;
+/**
+ * Re-reads blocked pages through one stealth browser session. Pages that come
+ * back clean are cached and returned as content; pages still blocked keep
+ * `blocked: true` and an honest error.
+ */
+export declare function crawlPagesStealth(entryUrl: string, urls: string[], options?: {
+    extractStructured?: boolean;
+    budgetMs?: number;
+    remote?: boolean;
+}): Promise<{
+    pages: CrawledPage[];
+    solved: boolean;
+    durationMs: number;
+    error?: string;
+    provider?: string;
+    providerCostUsd?: number;
+}>;
+/**
+ * Fetches one page for a crawl: cache first (unless bypassed), then a
+ * TLS-impersonated fetch. Never throws; failures come back with `error`.
+ * Used by `crawlWebsitePages` and by the durable crawl workflow (one call per page).
+ */
+export declare function crawlPage(pageUrl: string, options?: {
+    bypassCache?: boolean;
+    extractStructured?: boolean;
+    timeoutMs?: number;
+}): Promise<CrawledPage>;