recker 1.0.2-0

package/dist/utils/concurrency.js

@@ -0,0 +1,116 @@
+const DEFAULT_INTERVAL = 1000;
+export function normalizeConcurrency(options) {
+    let config;
+    if (typeof options.concurrency === 'number') {
+        config = { max: options.concurrency };
+    }
+    else if (options.concurrency && typeof options.concurrency === 'object') {
+        config = options.concurrency;
+    }
+    else {
+        config = {};
+    }
+    const max = config.max ?? Infinity;
+    const requestsPerInterval = config.requestsPerInterval ?? Infinity;
+    const interval = config.interval ?? DEFAULT_INTERVAL;
+    const http2Options = parseHTTP2Options(options.http2);
+    const http2Enabled = http2Options.enabled ?? false;
+    const agentConfig = normalizeAgentConfig(config.agent, max, http2Enabled, http2Options.maxConcurrentStreams);
+    const http2Streams = normalizeHTTP2Streams(config.http2?.maxConcurrentStreams, max, http2Enabled);
+    let runnerConcurrency;
+    if (config.runner?.concurrency !== undefined) {
+        runnerConcurrency = config.runner.concurrency;
+    }
+    else if (max !== Infinity) {
+        runnerConcurrency = max;
+    }
+    else {
+        runnerConcurrency = Infinity;
+    }
+    return {
+        max,
+        requestsPerInterval,
+        interval,
+        runner: {
+            concurrency: runnerConcurrency,
+            retries: config.runner?.retries,
+            retryDelay: config.runner?.retryDelay,
+        },
+        agent: {
+            ...agentConfig,
+            connections: agentConfig.connections,
+        },
+        http2: {
+            maxConcurrentStreams: http2Streams,
+        },
+    };
+}
+function parseHTTP2Options(http2) {
+    if (!http2)
+        return {};
+    if (typeof http2 === 'boolean') {
+        return { enabled: http2 };
+    }
+    return http2;
+}
+function normalizeAgentConfig(agentConfig, maxConcurrent, http2Enabled, http2MaxStreams) {
+    const defaults = {
+        pipelining: 1,
+        keepAlive: true,
+        keepAliveTimeout: 4000,
+        keepAliveMaxTimeout: 600000,
+        connectTimeout: 10000,
+        perDomainPooling: true,
+    };
+    const merged = { ...defaults, ...agentConfig };
+    let connections;
+    if (agentConfig?.connections === 'auto' || agentConfig?.connections === undefined) {
+        connections = calculateOptimalConnections(maxConcurrent, http2Enabled, http2MaxStreams, merged.pipelining ?? 1);
+    }
+    else {
+        connections = agentConfig.connections;
+    }
+    return {
+        ...merged,
+        connections,
+    };
+}
+export function calculateOptimalConnections(maxConcurrent, http2Enabled, http2MaxStreams, pipelining = 1) {
+    if (http2Enabled) {
+        const streams = http2MaxStreams ?? 100;
+        return Math.max(1, Math.ceil(maxConcurrent / streams));
+    }
+    else {
+        const requestsPerConnection = pipelining > 1 ? pipelining : 2;
+        return Math.max(1, Math.min(Math.ceil(maxConcurrent / requestsPerConnection), 50));
+    }
+}
+function normalizeHTTP2Streams(configuredStreams, maxConcurrent, http2Enabled) {
+    if (!http2Enabled) {
+        return 100;
+    }
+    if (configuredStreams === 'auto' || configuredStreams === undefined) {
+        return Math.min(Math.max(maxConcurrent, 100), 200);
+    }
+    return configuredStreams;
+}
+export function createBatchConfig(baseConcurrency, batchConcurrency, requestCount) {
+    const http2Enabled = baseConcurrency.http2.maxConcurrentStreams > 0;
+    const connections = calculateOptimalConnections(batchConcurrency, http2Enabled, baseConcurrency.http2.maxConcurrentStreams, baseConcurrency.agent.pipelining);
+    const smartPipelining = requestCount > 20 && batchConcurrency > 5
+        ? 2
+        : baseConcurrency.agent.pipelining ?? 1;
+    return {
+        ...baseConcurrency,
+        max: batchConcurrency,
+        runner: {
+            ...baseConcurrency.runner,
+            concurrency: batchConcurrency,
+        },
+        agent: {
+            ...baseConcurrency.agent,
+            connections,
+            pipelining: smartPipelining,
+        },
+    };
+}

package/dist/utils/dns.d.ts

@@ -0,0 +1,7 @@
+import { DNSOptions } from '../types/index.js';
+export declare function createLookupFunction(options: DNSOptions): (hostname: string, opts: any, callback: (err: NodeJS.ErrnoException | null, address: string, family: number) => void) => void;
+export declare function customDNSLookup(hostname: string, options?: DNSOptions): Promise<{
+    address: string;
+    family: number;
+}>;
+//# sourceMappingURL=dns.d.ts.map

package/dist/utils/dns.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"dns.d.ts","sourceRoot":"","sources":["../../src/utils/dns.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAQ/C,wBAAgB,oBAAoB,CAAC,OAAO,EAAE,UAAU,IAC9C,UAAU,MAAM,EAAE,MAAM,GAAG,EAAE,UAAU,CAAC,GAAG,EAAE,MAAM,CAAC,cAAc,GAAG,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,KAAK,IAAI,UAiB5H;AAOD,wBAAsB,eAAe,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,GAAE,UAAe,GAAG,OAAO,CAAC;IAAE,OAAO,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,CAAC,CAc9H"}

package/dist/utils/dns.js

@@ -0,0 +1,26 @@
+import { lookup } from 'node:dns';
+import { promisify } from 'node:util';
+const lookupAsync = promisify(lookup);
+export function createLookupFunction(options) {
+    return (hostname, opts, callback) => {
+        if (options.override && options.override[hostname]) {
+            const ip = options.override[hostname];
+            const family = ip.includes(':') ? 6 : 4;
+            return callback(null, ip, family);
+        }
+        lookup(hostname, opts, callback);
+    };
+}
+export async function customDNSLookup(hostname, options = {}) {
+    if (options.override && options.override[hostname]) {
+        const ip = options.override[hostname];
+        const family = ip.includes(':') ? 6 : 4;
+        return { address: ip, family };
+    }
+    const lookupOptions = {};
+    if (options.preferIPv4) {
+        lookupOptions.family = 4;
+    }
+    const result = await lookupAsync(hostname, lookupOptions);
+    return { address: result.address, family: result.family };
+}

package/dist/utils/doh.d.ts

@@ -0,0 +1,3 @@
+export type DoHProvider = 'cloudflare' | 'google' | 'quad9' | string;
+export declare function createDoHLookup(provider?: DoHProvider): (hostname: string, opts: any, callback: (err: Error | null, address: string, family: number) => void) => Promise<void>;
+//# sourceMappingURL=doh.d.ts.map

package/dist/utils/doh.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"doh.d.ts","sourceRoot":"","sources":["../../src/utils/doh.ts"],"names":[],"mappings":"AAEA,MAAM,MAAM,WAAW,GAAG,YAAY,GAAG,QAAQ,GAAG,OAAO,GAAG,MAAM,CAAC;AAsBrE,wBAAgB,eAAe,CAAC,QAAQ,GAAE,WAA0B,IAGpD,UAAU,MAAM,EAAE,MAAM,GAAG,EAAE,UAAU,CAAC,GAAG,EAAE,KAAK,GAAG,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,KAAK,IAAI,mBAqClH"}

package/dist/utils/doh.js

@@ -0,0 +1,35 @@
+import { request } from 'undici';
+const PROVIDERS = {
+    cloudflare: 'https://1.1.1.1/dns-query',
+    google: 'https://8.8.8.8/resolve',
+    quad9: 'https://9.9.9.9:5053/dns-query',
+};
+export function createDoHLookup(provider = 'cloudflare') {
+    const endpoint = PROVIDERS[provider] || provider;
+    return async (hostname, opts, callback) => {
+        try {
+            const url = new URL(endpoint);
+            url.searchParams.set('name', hostname);
+            url.searchParams.set('type', 'A');
+            const { body, statusCode } = await request(url, {
+                method: 'GET',
+                headers: { 'Accept': 'application/dns-json' }
+            });
+            if (statusCode !== 200) {
+                throw new Error(`DoH request failed with status ${statusCode}`);
+            }
+            const data = await body.json();
+            if (data.Status !== 0 || !data.Answer || data.Answer.length === 0) {
+                return callback(new Error(`DNS lookup failed for ${hostname}`), '', 4);
+            }
+            const record = data.Answer.find(r => r.type === 1);
+            if (record) {
+                return callback(null, record.data, 4);
+            }
+            callback(new Error(`No A record found for ${hostname}`), '', 4);
+        }
+        catch (err) {
+            callback(err, '', 0);
+        }
+    };
+}

package/dist/utils/header-parser.d.ts

@@ -0,0 +1,81 @@
+export interface CacheInfo {
+    hit: boolean;
+    status?: 'HIT' | 'MISS' | 'EXPIRED' | 'STALE' | 'BYPASS' | 'REVALIDATED';
+    age?: number;
+    maxAge?: number;
+    provider?: 'cloudflare' | 'fastly' | 'akamai' | 'cloudfront' | 'nginx' | 'varnish' | 'unknown';
+}
+export interface PlatformInfo {
+    provider?: string;
+    category?: 'cdn' | 'cloud' | 'hosting' | 'server' | 'proxy' | 'unknown';
+    region?: string;
+    server?: string;
+    metadata?: {
+        ray?: string;
+        requestId?: string;
+        pop?: string;
+        cacheId?: string;
+        traceId?: string;
+        [key: string]: string | number | boolean | undefined;
+    };
+}
+export interface RateLimitInfo {
+    limited: boolean;
+    limit?: number;
+    remaining?: number;
+    reset?: Date;
+    retryAfter?: number;
+    policy?: string;
+}
+export declare function parseCacheInfo(headers: Headers): CacheInfo;
+export declare function parsePlatformInfo(headers: Headers): PlatformInfo;
+export declare function parseRateLimitInfo(headers: Headers, status: number): RateLimitInfo;
+export interface CompressionInfo {
+    encoding?: 'gzip' | 'br' | 'deflate' | 'compress' | 'identity' | string;
+    originalSize?: number;
+    compressedSize?: number;
+    ratio?: number;
+}
+export declare function parseCompressionInfo(headers: Headers): CompressionInfo;
+export interface CSPInfo {
+    policy?: string;
+    directives: Record<string, string[]>;
+    reportOnly: boolean;
+}
+export declare function parseCSPInfo(headers: Headers): CSPInfo;
+export interface ContentTypeInfo {
+    mediaType?: string;
+    charset?: string;
+    boundary?: string;
+    type?: string;
+    subtype?: string;
+}
+export declare function parseContentType(headers: Headers): ContentTypeInfo;
+export interface AcceptInfo {
+    types: Array<{
+        mediaType: string;
+        q: number;
+        type?: string;
+        subtype?: string;
+    }>;
+    encodings: Array<{
+        encoding: string;
+        q: number;
+    }>;
+    languages: Array<{
+        language: string;
+        q: number;
+    }>;
+}
+export declare function parseAcceptInfo(headers: Headers): AcceptInfo;
+export interface HeaderInfo {
+    cache: CacheInfo;
+    platform: PlatformInfo;
+    rateLimit: RateLimitInfo;
+    compression: CompressionInfo;
+    csp: CSPInfo;
+    contentType: ContentTypeInfo;
+    accept: AcceptInfo;
+}
+export declare function parseHeaders(headers: Headers, status: number): HeaderInfo;
+//# sourceMappingURL=header-parser.d.ts.map

package/dist/utils/header-parser.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"header-parser.d.ts","sourceRoot":"","sources":["../../src/utils/header-parser.ts"],"names":[],"mappings":"AAKA,MAAM,WAAW,SAAS;IACxB,GAAG,EAAE,OAAO,CAAC;IACb,MAAM,CAAC,EAAE,KAAK,GAAG,MAAM,GAAG,SAAS,GAAG,OAAO,GAAG,QAAQ,GAAG,aAAa,CAAC;IACzE,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,YAAY,GAAG,QAAQ,GAAG,QAAQ,GAAG,YAAY,GAAG,OAAO,GAAG,SAAS,GAAG,SAAS,CAAC;CAChG;AAED,MAAM,WAAW,YAAY;IAE3B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAGlB,QAAQ,CAAC,EAAE,KAAK,GAAG,OAAO,GAAG,SAAS,GAAG,QAAQ,GAAG,OAAO,GAAG,SAAS,CAAC;IAGxE,MAAM,CAAC,EAAE,MAAM,CAAC;IAGhB,MAAM,CAAC,EAAE,MAAM,CAAC;IAGhB,QAAQ,CAAC,EAAE;QAET,GAAG,CAAC,EAAE,MAAM,CAAC;QAGb,SAAS,CAAC,EAAE,MAAM,CAAC;QAGnB,GAAG,CAAC,EAAE,MAAM,CAAC;QAGb,OAAO,CAAC,EAAE,MAAM,CAAC;QAGjB,OAAO,CAAC,EAAE,MAAM,CAAC;QAGjB,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,GAAG,SAAS,CAAC;KACtD,CAAC;CACH;AAED,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE,OAAO,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,KAAK,CAAC,EAAE,IAAI,CAAC;IACb,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAKD,wBAAgB,cAAc,CAAC,OAAO,EAAE,OAAO,GAAG,SAAS,CAuD1D;AAuKD,wBAAgB,iBAAiB,CAAC,OAAO,EAAE,OAAO,GAAG,YAAY,CAsFhE;AAKD,wBAAgB,kBAAkB,CAAC,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,GAAG,aAAa,CAgDlF;AAKD,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,GAAG,UAAU,GAAG,UAAU,GAAG,MAAM,CAAC;IACxE,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAKD,wBAAgB,oBAAoB,CAAC,OAAO,EAAE,OAAO,GAAG,eAAe,CA2BtE;AAKD,MAAM,WAAW,OAAO;IACtB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;IACrC,UAAU,EAAE,OAAO,CAAC;CACrB;AAKD,wBAAgB,YAAY,CAAC,OAAO,EAAE,OAAO,GAAG,OAAO,CA+BtD;AAKD,MAAM,WAAW,eAAe;IAC9B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAKD,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,OAAO,GAAG,eAAe,CA+BlE;AAKD,MAAM,WAAW,UAAU;IACzB,KAAK,EAAE,KAAK,CAAC;QACX,SAAS,EAAE,MAAM,CAAC;QAClB,CAAC,EAAE,MAAM,CAAC;QACV,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,OAAO,CAAC,EAAE,MAAM,CAAC;KAClB,CAAC,CAAC;IACH,SAAS,EAAE,KAAK,CAAC;QACf,QAAQ,EAAE,MAAM,CAAC;QACjB,CAAC,EAAE,MAAM,CAAC;KACX,CAAC,CAAC;IACH,SAAS,EAAE,KAAK,CAAC;QACf,QAAQ,EAAE,MAAM,CAAC;QACjB,CAAC,EAAE,MAAM,CAAC;KACX,CAAC,CAAC;CACJ;AAKD,wBAAgB,eAAe,CAAC,OAAO,EAAE,OAAO,GAAG,UAAU,CAgF5D;AAKD,MAAM,WAAW,UAAU;IACzB,KAAK,EAAE,SAAS,CAAC;IACjB,QAAQ,EAAE,YAAY,CAAC;IACvB,SAAS,EAAE,aAAa,CAAC;IACzB,WAAW,EAAE,eAAe,CAAC;IAC7B,GAAG,EAAE,OAAO,CAAC;IACb,WAAW,EAAE,eAAe,CAAC;IAC7B,MAAM,EAAE,UAAU,CAAC;CACpB;AAED,wBAAgB,YAAY,CAAC,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,GAAG,UAAU,CAUzE"}