recker 1.0.102 → 1.0.103-next.8501b56

package/dist/browser/core/errors.d.ts

@@ -1,6 +1,6 @@
 import { ReckerRequest, ReckerResponse } from '../types/index.js';
-export type CanonicalErrorCategory = 'http' | 'network' | 'timeout' | 'protocol' | 'validation' | 'state' | 'filesystem' | 'resource' | 'policy' | 'queue' | 'unknown';
-export type ErrorSource = 'client' | 'transport' | 'server' | 'upstream';
+export type CanonicalErrorCategory = 'http' | 'network' | 'timeout' | 'protocol' | 'validation' | 'state' | 'filesystem' | 'resource' | 'policy' | 'queue' | 'scrape' | 'unknown';
+export type ErrorSource = 'client' | 'transport' | 'server' | 'upstream' | 'spider';
 export type CanonicalErrorSeverity = 'low' | 'medium' | 'high';
 export interface CanonicalErrorMetadata {
     category: CanonicalErrorCategory;

package/dist/browser/scrape/crawl-queue.d.ts

@@ -3,6 +3,8 @@ export interface CrawlQueueItem {
     depth: number;
     priority?: number;
     discoveredFrom?: string;
+    sourceUrl?: string;
+    rewriteReason?: string;
 }
 export interface CrawlQueueAdapter {
     push(item: CrawlQueueItem): Promise<void>;

package/dist/browser/scrape/domain-stats.d.ts

@@ -0,0 +1,25 @@
+export interface DomainTransportStats {
+    hostname: string;
+    curlSuccess: number;
+    curlFail: number;
+    undiciSuccess: number;
+    undiciFail: number;
+    lastSuccessTransport?: 'curl' | 'undici';
+    preferredTransport?: 'curl' | 'undici';
+    lastSeenTs: number;
+}
+export interface DomainStatsAdapter {
+    load(hostname: string): Promise<DomainTransportStats | null>;
+    loadAll(): Promise<Map<string, DomainTransportStats>>;
+    record(hostname: string, transport: 'curl' | 'undici', success: boolean): Promise<void>;
+    clear?(): Promise<void>;
+    close?(): Promise<void>;
+}
+export declare class InMemoryDomainStats implements DomainStatsAdapter {
+    private map;
+    load(hostname: string): Promise<DomainTransportStats | null>;
+    loadAll(): Promise<Map<string, DomainTransportStats>>;
+    record(hostname: string, transport: 'curl' | 'undici', success: boolean): Promise<void>;
+    clear(): Promise<void>;
+    close(): Promise<void>;
+}

package/dist/browser/scrape/domain-stats.js

@@ -0,0 +1,47 @@
+export class InMemoryDomainStats {
+    map = new Map();
+    async load(hostname) {
+        const s = this.map.get(hostname);
+        return s ? { ...s } : null;
+    }
+    async loadAll() {
+        const copy = new Map();
+        for (const [k, v] of this.map) {
+            copy.set(k, { ...v });
+        }
+        return copy;
+    }
+    async record(hostname, transport, success) {
+        let stats = this.map.get(hostname);
+        if (!stats) {
+            stats = {
+                hostname,
+                curlSuccess: 0,
+                curlFail: 0,
+                undiciSuccess: 0,
+                undiciFail: 0,
+                lastSeenTs: 0,
+            };
+            this.map.set(hostname, stats);
+        }
+        if (success) {
+            if (transport === 'curl')
+                stats.curlSuccess += 1;
+            else
+                stats.undiciSuccess += 1;
+            stats.lastSuccessTransport = transport;
+        }
+        else {
+            if (transport === 'curl')
+                stats.curlFail += 1;
+            else
+                stats.undiciFail += 1;
+        }
+        stats.lastSeenTs = Date.now();
+    }
+    async clear() {
+        this.map.clear();
+    }
+    async close() {
+    }
+}

package/dist/browser/scrape/errors.d.ts

@@ -0,0 +1,82 @@
+import { ReckerError } from '../core/errors.js';
+import type { BlockDetectionResult } from '../utils/block-detector.js';
+import type { CaptchaDetectionResult } from '../utils/block-detector.js';
+export type SpiderTransportLabel = 'auto' | 'undici' | 'curl';
+export type CaptchaProviderLabel = CaptchaDetectionResult['provider'];
+export declare class SpiderBlockError extends ReckerError {
+    url: string;
+    domain: string;
+    reason: BlockDetectionResult['reason'];
+    confidence: number;
+    transport: SpiderTransportLabel;
+    preferredTransport?: SpiderTransportLabel;
+    constructor(opts: {
+        url: string;
+        domain: string;
+        reason: BlockDetectionResult['reason'];
+        confidence: number;
+        transport: SpiderTransportLabel;
+        preferredTransport?: SpiderTransportLabel;
+        statusCode?: number;
+        message?: string;
+    });
+}
+export declare class SpiderChallengeError extends ReckerError {
+    url: string;
+    domain: string;
+    provider?: CaptchaProviderLabel;
+    confidence: number;
+    cooldownMs: number;
+    transport: SpiderTransportLabel;
+    constructor(opts: {
+        url: string;
+        domain: string;
+        provider?: CaptchaProviderLabel;
+        confidence: number;
+        cooldownMs: number;
+        transport: SpiderTransportLabel;
+        statusCode?: number;
+    });
+}
+export declare class SpiderRobotsDisallowedError extends ReckerError {
+    url: string;
+    path: string;
+    userAgent: string;
+    robotsUrl: string;
+    constructor(opts: {
+        url: string;
+        path: string;
+        userAgent: string;
+        robotsUrl: string;
+    });
+}
+export declare class SpiderDepthLimitError extends ReckerError {
+    url: string;
+    depth: number;
+    maxDepth: number;
+    constructor(opts: {
+        url: string;
+        depth: number;
+        maxDepth: number;
+    });
+}
+export declare class SpiderDomainOutOfScopeError extends ReckerError {
+    url: string;
+    allowedDomains?: string[];
+    constructor(opts: {
+        url: string;
+        allowedDomains?: string[];
+    });
+}
+export declare class SpiderUnsupportedContentError extends ReckerError {
+    url: string;
+    contentType: string;
+    reason: 'binary' | 'pdf' | 'doc' | 'media' | 'archive';
+    fallbackSuggestion?: string;
+    constructor(opts: {
+        url: string;
+        contentType: string;
+        reason: 'binary' | 'pdf' | 'doc' | 'media' | 'archive';
+        fallbackSuggestion?: string;
+    });
+}

package/dist/browser/scrape/errors.js

@@ -0,0 +1,138 @@
+import { ReckerError } from '../core/errors.js';
+export class SpiderBlockError extends ReckerError {
+    url;
+    domain;
+    reason;
+    confidence;
+    transport;
+    preferredTransport;
+    constructor(opts) {
+        super(opts.message ?? `Request to ${opts.url} was blocked (${opts.reason ?? 'unknown'})`, undefined, undefined, [
+            opts.preferredTransport && opts.preferredTransport !== opts.transport
+                ? `Retry with the ${opts.preferredTransport} transport.`
+                : 'Rotate user-agent, proxy, or use curl-impersonate to bypass TLS fingerprinting.',
+            'Increase the retry delay and honor any Retry-After header.',
+        ], true, {
+            category: 'scrape',
+            source: 'spider',
+            severity: 'medium',
+            canRetry: true,
+            reason: `Blocked: ${opts.reason ?? 'unknown'}`,
+            statusCode: opts.statusCode,
+        });
+        this.name = 'SpiderBlockError';
+        this.url = opts.url;
+        this.domain = opts.domain;
+        this.reason = opts.reason;
+        this.confidence = opts.confidence;
+        this.transport = opts.transport;
+        this.preferredTransport = opts.preferredTransport;
+    }
+}
+export class SpiderChallengeError extends ReckerError {
+    url;
+    domain;
+    provider;
+    confidence;
+    cooldownMs;
+    transport;
+    constructor(opts) {
+        super(`CAPTCHA challenge detected on ${opts.url}${opts.provider ? ` (${opts.provider})` : ''}`, undefined, undefined, [
+            `Cool down the host for at least ${opts.cooldownMs}ms before retrying.`,
+            'Consider solving the challenge via an external CAPTCHA-solving service.',
+            'Rotate proxy/IP before retrying.',
+        ], true, {
+            category: 'scrape',
+            source: 'spider',
+            severity: 'high',
+            canRetry: true,
+            reason: opts.provider ? `Challenge: ${opts.provider}` : 'CAPTCHA challenge',
+            statusCode: opts.statusCode,
+            retryAfterMs: opts.cooldownMs,
+        });
+        this.name = 'SpiderChallengeError';
+        this.url = opts.url;
+        this.domain = opts.domain;
+        this.provider = opts.provider;
+        this.confidence = opts.confidence;
+        this.cooldownMs = opts.cooldownMs;
+        this.transport = opts.transport;
+    }
+}
+export class SpiderRobotsDisallowedError extends ReckerError {
+    url;
+    path;
+    userAgent;
+    robotsUrl;
+    constructor(opts) {
+        super(`URL ${opts.url} is disallowed by robots.txt for ${opts.userAgent}`, undefined, undefined, [
+            'Do not crawl this URL — the site operator has explicitly excluded it.',
+            `Check ${opts.robotsUrl} for the full rules.`,
+        ], false, {
+            category: 'scrape',
+            source: 'spider',
+            severity: 'low',
+            canRetry: false,
+            reason: 'Disallowed by robots.txt',
+        });
+        this.name = 'SpiderRobotsDisallowedError';
+        this.url = opts.url;
+        this.path = opts.path;
+        this.userAgent = opts.userAgent;
+        this.robotsUrl = opts.robotsUrl;
+    }
+}
+export class SpiderDepthLimitError extends ReckerError {
+    url;
+    depth;
+    maxDepth;
+    constructor(opts) {
+        super(`URL ${opts.url} exceeds max crawl depth (${opts.depth} > ${opts.maxDepth})`, undefined, undefined, ['Increase maxDepth if deeper crawls are expected.'], false, {
+            category: 'scrape',
+            source: 'spider',
+            severity: 'low',
+            canRetry: false,
+            reason: 'Depth limit exceeded',
+        });
+        this.name = 'SpiderDepthLimitError';
+        this.url = opts.url;
+        this.depth = opts.depth;
+        this.maxDepth = opts.maxDepth;
+    }
+}
+export class SpiderDomainOutOfScopeError extends ReckerError {
+    url;
+    allowedDomains;
+    constructor(opts) {
+        super(`URL ${opts.url} is outside the crawl scope`, undefined, undefined, ['Add the host to allowedDomains or disable sameDomain filtering.'], false, {
+            category: 'scrape',
+            source: 'spider',
+            severity: 'low',
+            canRetry: false,
+            reason: 'Domain out of scope',
+        });
+        this.name = 'SpiderDomainOutOfScopeError';
+        this.url = opts.url;
+        this.allowedDomains = opts.allowedDomains;
+    }
+}
+export class SpiderUnsupportedContentError extends ReckerError {
+    url;
+    contentType;
+    reason;
+    fallbackSuggestion;
+    constructor(opts) {
+        super(`Unsupported content type on ${opts.url}: ${opts.contentType}`, undefined, undefined, opts.fallbackSuggestion ? [opts.fallbackSuggestion] : ['Handle this URL with a specialty fetcher or skip.'], false, {
+            category: 'scrape',
+            source: 'spider',
+            severity: 'low',
+            canRetry: false,
+            reason: `Unsupported content: ${opts.contentType}`,
+        });
+        this.name = 'SpiderUnsupportedContentError';
+        this.url = opts.url;
+        this.contentType = opts.contentType;
+        this.reason = opts.reason;
+        this.fallbackSuggestion = opts.fallbackSuggestion;
+    }
+}

package/dist/browser/scrape/index.d.ts

@@ -4,12 +4,17 @@ export { ScrapeDocument } from './document.js';
 export { ScrapeElement } from './element.js';
 export { Spider, spider } from './spider.js';
 export type { SpiderOptions, SpiderPageResult, SpiderPageEvent, SpiderProgress, SpiderResult, } from './spider.js';
+export { SpiderBlockError, SpiderChallengeError, SpiderRobotsDisallowedError, SpiderDepthLimitError, SpiderDomainOutOfScopeError, SpiderUnsupportedContentError, } from './errors.js';
+export { rewriteUrl } from './rewrite-url.js';
+export type { UrlRewriteResult } from './rewrite-url.js';
 export { InMemoryCrawlQueue } from './crawl-queue.js';
 export { SqliteCrawlQueue } from './sqlite-crawl-queue.js';
 export type { CrawlQueueAdapter, CrawlQueueItem } from './crawl-queue.js';
 export { InMemoryCrawlStorage } from './crawl-storage.js';
 export { SqliteCrawlStorage } from './sqlite-crawl-storage.js';
 export type { CrawlStorageAdapter } from './crawl-storage.js';
+export { InMemoryDomainStats } from './domain-stats.js';
+export type { DomainStatsAdapter, DomainTransportStats } from './domain-stats.js';
 export { ListProxyAdapter } from './proxy-adapter.js';
 export type { ProxyAdapter } from './proxy-adapter.js';
 export { extractLinks, extractImages, extractMeta, extractOpenGraph, extractTwitterCard, extractJsonLd, extractForms, extractTables, extractScripts, extractStyles, } from './extractors.js';

package/dist/browser/scrape/index.js

@@ -2,9 +2,12 @@ export { parse as parseHtmlSync, HTMLElement, TextNode, CommentNode, Node, NodeT
 export { ScrapeDocument } from './document.js';
 export { ScrapeElement } from './element.js';
 export { Spider, spider } from './spider.js';
+export { SpiderBlockError, SpiderChallengeError, SpiderRobotsDisallowedError, SpiderDepthLimitError, SpiderDomainOutOfScopeError, SpiderUnsupportedContentError, } from './errors.js';
+export { rewriteUrl } from './rewrite-url.js';
 export { InMemoryCrawlQueue } from './crawl-queue.js';
 export { SqliteCrawlQueue } from './sqlite-crawl-queue.js';
 export { InMemoryCrawlStorage } from './crawl-storage.js';
 export { SqliteCrawlStorage } from './sqlite-crawl-storage.js';
+export { InMemoryDomainStats } from './domain-stats.js';
 export { ListProxyAdapter } from './proxy-adapter.js';
 export { extractLinks, extractImages, extractMeta, extractOpenGraph, extractTwitterCard, extractJsonLd, extractForms, extractTables, extractScripts, extractStyles, } from './extractors.js';

package/dist/browser/scrape/rewrite-url.d.ts

@@ -0,0 +1,6 @@
+export interface UrlRewriteResult {
+    url: string;
+    rewritten: boolean;
+    reason?: string;
+}
+export declare function rewriteUrl(input: string): UrlRewriteResult;

package/dist/browser/scrape/rewrite-url.js

@@ -0,0 +1,77 @@
+const GOOGLE_DOC_PREFIXES = [
+    'https://docs.google.com/document/d/',
+    'http://docs.google.com/document/d/',
+];
+const GOOGLE_PRESENTATION_PREFIXES = [
+    'https://docs.google.com/presentation/d/',
+    'http://docs.google.com/presentation/d/',
+];
+const GOOGLE_SPREADSHEET_PREFIXES = [
+    'https://docs.google.com/spreadsheets/d/',
+    'http://docs.google.com/spreadsheets/d/',
+];
+const GOOGLE_DRIVE_FILE_PREFIXES = [
+    'https://drive.google.com/file/d/',
+    'http://drive.google.com/file/d/',
+];
+function startsWithAny(url, prefixes) {
+    for (const p of prefixes) {
+        if (url.startsWith(p))
+            return true;
+    }
+    return false;
+}
+export function rewriteUrl(input) {
+    if (startsWithAny(input, GOOGLE_DOC_PREFIXES)) {
+        if (input.includes('/document/d/e/')) {
+            return { url: input, rewritten: false };
+        }
+        const id = input.match(/\/document\/d\/([-\w]+)/)?.[1];
+        if (id) {
+            return {
+                url: `https://docs.google.com/document/d/${id}/export?format=html`,
+                rewritten: true,
+                reason: 'google-docs-export',
+            };
+        }
+    }
+    if (startsWithAny(input, GOOGLE_PRESENTATION_PREFIXES)) {
+        if (input.includes('/presentation/d/e/')) {
+            return { url: input, rewritten: false };
+        }
+        const id = input.match(/\/presentation\/d\/([-\w]+)/)?.[1];
+        if (id) {
+            return {
+                url: `https://docs.google.com/presentation/d/${id}/export?format=html`,
+                rewritten: true,
+                reason: 'google-slides-export',
+            };
+        }
+    }
+    if (startsWithAny(input, GOOGLE_SPREADSHEET_PREFIXES)) {
+        if (input.includes('/spreadsheets/d/e/')) {
+            return { url: input, rewritten: false };
+        }
+        const id = input.match(/\/spreadsheets\/d\/([-\w]+)/)?.[1];
+        if (id) {
+            const gidMatch = input.match(/[?&#]gid=(\d+)/);
+            const gidParam = gidMatch ? `&gid=${gidMatch[1]}` : '';
+            return {
+                url: `https://docs.google.com/spreadsheets/d/${id}/gviz/tq?tqx=out:html${gidParam}`,
+                rewritten: true,
+                reason: 'google-sheets-export',
+            };
+        }
+    }
+    if (startsWithAny(input, GOOGLE_DRIVE_FILE_PREFIXES)) {
+        const id = input.match(/\/file\/d\/([-\w]+)/)?.[1];
+        if (id) {
+            return {
+                url: `https://drive.google.com/uc?export=download&id=${id}`,
+                rewritten: true,
+                reason: 'google-drive-download',
+            };
+        }
+    }
+    return { url: input, rewritten: false };
+}

package/dist/browser/scrape/spider.d.ts

@@ -5,7 +5,9 @@ import { type SitemapUrl } from '../seo/validators/sitemap.js';
 import { type CaptchaDetectionResult, type BlockDetectionResult } from '../utils/block-detector.js';
 import { type CrawlQueueAdapter } from './crawl-queue.js';
 import { type CrawlStorageAdapter } from './crawl-storage.js';
+import type { DomainStatsAdapter } from './domain-stats.js';
 import { type ProxyAdapter } from './proxy-adapter.js';
+import type { ReckerError } from '../core/errors.js';
 export type SpiderTransport = 'auto' | 'undici' | 'curl';
 type CaptchaProvider = CaptchaDetectionResult['provider'];
 export interface SpiderOptions {
@@ -28,11 +30,15 @@ export interface SpiderOptions {
     retryJitterMs?: number;
     maxDomainBlockStrikes?: number;
     respectRobotsTxt?: boolean;
+    respectRobotsCrawlDelay?: boolean;
+    maxRobotsCrawlDelayMs?: number;
     useSitemap?: boolean;
     sitemapUrl?: string;
+    rewriteUrls?: boolean;
     proxy?: string | string[] | ProxyAdapter;
     transport?: SpiderTransport;
     preferCurlFirst?: boolean;
+    domainStats?: DomainStatsAdapter;
     onPage?: (event: SpiderPageEvent) => void | Promise<void>;
     onCaptchaDetected?: (result: {
         url: string;
@@ -134,6 +140,9 @@ export interface SpiderPageResult {
     contentHash?: string;
     isDuplicate?: boolean;
     duplicateOf?: string;
+    sourceUrl?: string;
+    rewriteReason?: string;
+    typedError?: ReckerError;
 }
 export interface SpiderPageEvent {
     result: SpiderPageResult;
@@ -191,6 +200,7 @@ export declare class Spider {
     private pool;
     private crawlQueue;
     private crawlStorage;
+    private domainStats;
     private proxyAdapter;
     private proxyClients;
     private _visitedCount;
@@ -209,11 +219,13 @@ export declare class Spider {
     private curlTransport;
     private curlAvailable;
     private domainStates;
+    private domainStatsWrites;
     private sitemapUrls;
     private sitemapUrlSet;
     private robotsData;
     private sitemapValidation;
     private robotsValidation;
+    private baseRobotsCrawlDelayMs;
     private waitForDomainRateLimit;
     private toHeaderRecord;
     constructor(options?: SpiderOptions);
@@ -227,6 +239,7 @@ export declare class Spider {
     private crawlPage;
     private getOrCreateDomainState;
     private recordTransportResult;
+    private persistTransportStat;
     private isRetryableStatus;
     private buildRequestHeaders;
     private shouldUseCurlForHost;