recker 1.0.26 → 1.0.27

package/dist/browser/utils/link-header.d.ts

@@ -0,0 +1,69 @@
+export interface Link {
+    uri: string;
+    rel?: string;
+    anchor?: string;
+    type?: string;
+    media?: string;
+    title?: string;
+    hreflang?: string;
+    [key: string]: string | undefined;
+}
+export interface LinkCollection {
+    all: Link[];
+    [rel: string]: Link | Link[] | undefined;
+}
+export declare function parseLink(headerValue: string): LinkCollection;
+export declare const LinkRel: {
+    readonly NEXT: "next";
+    readonly PREV: "prev";
+    readonly PREVIOUS: "previous";
+    readonly FIRST: "first";
+    readonly LAST: "last";
+    readonly ALTERNATE: "alternate";
+    readonly CANONICAL: "canonical";
+    readonly AUTHOR: "author";
+    readonly LICENSE: "license";
+    readonly STYLESHEET: "stylesheet";
+    readonly ICON: "icon";
+    readonly PRELOAD: "preload";
+    readonly PREFETCH: "prefetch";
+    readonly PRECONNECT: "preconnect";
+    readonly DNS_PREFETCH: "dns-prefetch";
+    readonly PRERENDER: "prerender";
+    readonly SELF: "self";
+    readonly EDIT: "edit";
+    readonly COLLECTION: "collection";
+    readonly ITEM: "item";
+    readonly HELP: "help";
+    readonly DESCRIBEDBY: "describedby";
+    readonly UP: "up";
+    readonly RELATED: "related";
+    readonly REPLIES: "replies";
+    readonly VIA: "via";
+};
+export declare class LinkHeaderParser {
+    private links;
+    constructor(headerValue: string);
+    getAll(): Link[];
+    getRel(rel: string): Link | Link[] | undefined;
+    getFirst(rel: string): Link | undefined;
+    has(rel: string): boolean;
+    getPagination(): {
+        next?: string;
+        prev?: string;
+        first?: string;
+        last?: string;
+    };
+    hasNext(): boolean;
+    hasPrev(): boolean;
+    getCanonical(): string | undefined;
+    getAlternates(): Link[];
+    getResourceHints(): {
+        preload: Link[];
+        prefetch: Link[];
+        preconnect: Link[];
+        dnsPrefetch: Link[];
+    };
+    toJSON(): LinkCollection;
+}
+export declare function parseLinkHeader(headers: Headers): LinkHeaderParser | null;

package/dist/browser/utils/link-header.js

@@ -0,0 +1,190 @@
+export function parseLink(headerValue) {
+    if (!headerValue || typeof headerValue !== 'string') {
+        return { all: [] };
+    }
+    const links = [];
+    const collection = { all: links };
+    const linkStrings = splitLinkHeader(headerValue);
+    for (const linkString of linkStrings) {
+        const link = parseSingleLink(linkString.trim());
+        if (link) {
+            links.push(link);
+            if (link.rel) {
+                const relTypes = link.rel.split(/\s+/);
+                for (const rel of relTypes) {
+                    const existing = collection[rel];
+                    if (existing) {
+                        if (Array.isArray(existing)) {
+                            existing.push(link);
+                        }
+                        else {
+                            collection[rel] = [existing, link];
+                        }
+                    }
+                    else {
+                        collection[rel] = link;
+                    }
+                }
+            }
+        }
+    }
+    return collection;
+}
+function splitLinkHeader(header) {
+    const result = [];
+    let current = '';
+    let inBrackets = false;
+    let inQuotes = false;
+    for (let i = 0; i < header.length; i++) {
+        const char = header[i];
+        const prevChar = i > 0 ? header[i - 1] : '';
+        if (char === '<' && !inQuotes) {
+            inBrackets = true;
+        }
+        else if (char === '>' && !inQuotes) {
+            inBrackets = false;
+        }
+        else if (char === '"' && prevChar !== '\\') {
+            inQuotes = !inQuotes;
+        }
+        else if (char === ',' && !inBrackets && !inQuotes) {
+            result.push(current.trim());
+            current = '';
+            continue;
+        }
+        current += char;
+    }
+    if (current.trim()) {
+        result.push(current.trim());
+    }
+    return result;
+}
+function parseSingleLink(linkString) {
+    const uriMatch = linkString.match(/^<([^>]+)>/);
+    if (!uriMatch) {
+        return null;
+    }
+    const uri = uriMatch[1];
+    const link = { uri };
+    const paramsString = linkString.slice(uriMatch[0].length);
+    const params = parseParameters(paramsString);
+    for (const [key, value] of Object.entries(params)) {
+        link[key] = value;
+    }
+    return link;
+}
+function parseParameters(paramsString) {
+    const params = {};
+    const paramStrings = paramsString.split(/;/).filter(s => s.trim());
+    for (const paramString of paramStrings) {
+        const equalIndex = paramString.indexOf('=');
+        if (equalIndex === -1)
+            continue;
+        const key = paramString.slice(0, equalIndex).trim();
+        let value = paramString.slice(equalIndex + 1).trim();
+        if (value.startsWith('"') && value.endsWith('"')) {
+            value = value.slice(1, -1);
+        }
+        else if (value.startsWith("'") && value.endsWith("'")) {
+            value = value.slice(1, -1);
+        }
+        value = value.replace(/\\(.)/g, '$1');
+        params[key] = value;
+    }
+    return params;
+}
+export const LinkRel = {
+    NEXT: 'next',
+    PREV: 'prev',
+    PREVIOUS: 'previous',
+    FIRST: 'first',
+    LAST: 'last',
+    ALTERNATE: 'alternate',
+    CANONICAL: 'canonical',
+    AUTHOR: 'author',
+    LICENSE: 'license',
+    STYLESHEET: 'stylesheet',
+    ICON: 'icon',
+    PRELOAD: 'preload',
+    PREFETCH: 'prefetch',
+    PRECONNECT: 'preconnect',
+    DNS_PREFETCH: 'dns-prefetch',
+    PRERENDER: 'prerender',
+    SELF: 'self',
+    EDIT: 'edit',
+    COLLECTION: 'collection',
+    ITEM: 'item',
+    HELP: 'help',
+    DESCRIBEDBY: 'describedby',
+    UP: 'up',
+    RELATED: 'related',
+    REPLIES: 'replies',
+    VIA: 'via',
+};
+export class LinkHeaderParser {
+    links;
+    constructor(headerValue) {
+        this.links = parseLink(headerValue);
+    }
+    getAll() {
+        return this.links.all;
+    }
+    getRel(rel) {
+        return this.links[rel];
+    }
+    getFirst(rel) {
+        const links = this.links[rel];
+        if (!links)
+            return undefined;
+        return Array.isArray(links) ? links[0] : links;
+    }
+    has(rel) {
+        return rel in this.links;
+    }
+    getPagination() {
+        return {
+            next: this.getFirst(LinkRel.NEXT)?.uri,
+            prev: this.getFirst(LinkRel.PREV)?.uri || this.getFirst(LinkRel.PREVIOUS)?.uri,
+            first: this.getFirst(LinkRel.FIRST)?.uri,
+            last: this.getFirst(LinkRel.LAST)?.uri,
+        };
+    }
+    hasNext() {
+        return this.has(LinkRel.NEXT);
+    }
+    hasPrev() {
+        return this.has(LinkRel.PREV) || this.has(LinkRel.PREVIOUS);
+    }
+    getCanonical() {
+        return this.getFirst(LinkRel.CANONICAL)?.uri;
+    }
+    getAlternates() {
+        const alts = this.getRel(LinkRel.ALTERNATE);
+        if (!alts)
+            return [];
+        return Array.isArray(alts) ? alts : [alts];
+    }
+    getResourceHints() {
+        const getLinks = (rel) => {
+            const links = this.getRel(rel);
+            if (!links)
+                return [];
+            return Array.isArray(links) ? links : [links];
+        };
+        return {
+            preload: getLinks(LinkRel.PRELOAD),
+            prefetch: getLinks(LinkRel.PREFETCH),
+            preconnect: getLinks(LinkRel.PRECONNECT),
+            dnsPrefetch: getLinks(LinkRel.DNS_PREFETCH),
+        };
+    }
+    toJSON() {
+        return this.links;
+    }
+}
+export function parseLinkHeader(headers) {
+    const linkHeader = headers.get('Link');
+    if (!linkHeader)
+        return null;
+    return new LinkHeaderParser(linkHeader);
+}

package/dist/browser/utils/optional-require.d.ts

@@ -0,0 +1,19 @@
+interface DependencyInfo {
+    package: string;
+    submodule: string;
+    version?: string;
+    feature?: string;
+}
+export declare const OPTIONAL_DEPENDENCIES: Record<string, DependencyInfo>;
+export declare class MissingDependencyError extends Error {
+    readonly packageName: string;
+    readonly submodule: string;
+    readonly installCommand: string;
+    constructor(packageName: string, submodule: string, installCommand: string);
+}
+export declare function clearModuleCache(): void;
+export declare function requireOptional<T = unknown>(packageName: string, submodule?: string): Promise<T>;
+export declare function isPackageAvailable(packageName: string): boolean;
+export declare function getInstallCommand(packages: string[]): string;
+export declare function requireOptionalMany<T extends Record<string, unknown>>(packages: string[], submodule: string): Promise<T>;
+export {};

package/dist/browser/utils/optional-require.js

@@ -0,0 +1,105 @@
+export const OPTIONAL_DEPENDENCIES = {
+    'cardinal': {
+        package: 'cardinal',
+        submodule: 'recker/cli',
+        version: '^2.0.0',
+        feature: 'syntax highlighting',
+    },
+    'ssh2-sftp-client': {
+        package: 'ssh2-sftp-client',
+        submodule: 'recker/protocols/sftp',
+        version: '^11.0.0',
+        feature: 'SFTP client',
+    },
+    'cheerio': {
+        package: 'cheerio',
+        submodule: 'recker/scrape',
+        version: '^1.0.0',
+        feature: 'HTML parsing and scraping',
+    },
+    'ioredis': {
+        package: 'ioredis',
+        submodule: 'recker/cache',
+        version: '^5.0.0',
+        feature: 'Redis cache storage',
+    },
+};
+export class MissingDependencyError extends Error {
+    packageName;
+    submodule;
+    installCommand;
+    constructor(packageName, submodule, installCommand) {
+        super(`Missing optional dependency: ${packageName}\n\n` +
+            `This dependency is required for ${submodule}.\n` +
+            `Install it with:\n\n  ${installCommand}\n`);
+        this.packageName = packageName;
+        this.submodule = submodule;
+        this.installCommand = installCommand;
+        this.name = 'MissingDependencyError';
+    }
+}
+const moduleCache = new Map();
+export function clearModuleCache() {
+    moduleCache.clear();
+}
+export async function requireOptional(packageName, submodule) {
+    if (moduleCache.has(packageName)) {
+        return moduleCache.get(packageName);
+    }
+    try {
+        const mod = await import(packageName);
+        moduleCache.set(packageName, mod);
+        return mod;
+    }
+    catch (error) {
+        const err = error;
+        const isModuleNotFound = err.code === 'ERR_MODULE_NOT_FOUND' ||
+            err.code === 'MODULE_NOT_FOUND' ||
+            (err.message?.includes('Cannot find module'));
+        if (isModuleNotFound) {
+            const info = OPTIONAL_DEPENDENCIES[packageName];
+            const sub = submodule || info?.submodule || 'this feature';
+            const version = info?.version || '';
+            const pkg = version ? `${packageName}@${version.replace('^', '')}` : packageName;
+            throw new MissingDependencyError(packageName, sub, `pnpm add ${pkg}`);
+        }
+        throw error;
+    }
+}
+export function isPackageAvailable(packageName) {
+    try {
+        require.resolve(packageName);
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+export function getInstallCommand(packages) {
+    const pkgsWithVersions = packages.map((pkg) => {
+        const info = OPTIONAL_DEPENDENCIES[pkg];
+        return info?.version ? `${pkg}@${info.version.replace('^', '')}` : pkg;
+    });
+    return `pnpm add ${pkgsWithVersions.join(' ')}`;
+}
+export async function requireOptionalMany(packages, submodule) {
+    const missing = [];
+    const results = {};
+    for (const pkg of packages) {
+        try {
+            results[pkg] = await requireOptional(pkg, submodule);
+        }
+        catch (error) {
+            if (error instanceof MissingDependencyError) {
+                missing.push(pkg);
+            }
+            else {
+                throw error;
+            }
+        }
+    }
+    if (missing.length > 0) {
+        throw new MissingDependencyError(missing.join(', '), submodule, getInstallCommand(missing));
+    }
+    return results;
+}

package/dist/browser/utils/progress.d.ts

@@ -0,0 +1,8 @@
+import { ProgressEvent, ProgressCallback } from '../types/index.js';
+export interface ProgressStreamOptions {
+    total?: number;
+    direction?: 'upload' | 'download';
+    throttleMs?: number;
+}
+export declare function createProgressStream(stream: ReadableStream<Uint8Array>, onProgress: ProgressCallback, optionsOrTotal?: number | ProgressStreamOptions): ReadableStream<Uint8Array>;
+export declare function calculateProgress(loaded: number, total?: number, direction?: 'upload' | 'download'): ProgressEvent;

package/dist/browser/utils/progress.js

@@ -0,0 +1,82 @@
+export function createProgressStream(stream, onProgress, optionsOrTotal) {
+    const options = typeof optionsOrTotal === 'number'
+        ? { total: optionsOrTotal }
+        : optionsOrTotal || {};
+    const { total, direction, throttleMs = 100 } = options;
+    let loaded = 0;
+    let startTime = Date.now();
+    let lastUpdate = 0;
+    let lastLoaded = 0;
+    let lastRateUpdate = startTime;
+    let smoothedRate = 0;
+    const rateSmoothingFactor = 0.3;
+    const emitProgress = (now, isFinal) => {
+        const elapsed = (now - startTime) / 1000;
+        const intervalMs = now - lastRateUpdate;
+        const bytesInInterval = loaded - lastLoaded;
+        if (intervalMs > 0) {
+            const instantRate = (bytesInInterval / intervalMs) * 1000;
+            smoothedRate = smoothedRate === 0
+                ? instantRate
+                : smoothedRate * (1 - rateSmoothingFactor) + instantRate * rateSmoothingFactor;
+        }
+        lastLoaded = loaded;
+        lastRateUpdate = now;
+        const rate = smoothedRate;
+        let percent;
+        if (total) {
+            percent = isFinal ? 100 : Math.min((loaded / total) * 100, 99.9);
+        }
+        const progress = {
+            loaded,
+            transferred: loaded,
+            total,
+            percent,
+            rate,
+            estimated: total && rate > 0 ? ((total - loaded) / rate) * 1000 : undefined,
+            direction,
+        };
+        onProgress(progress);
+        lastUpdate = now;
+    };
+    return new ReadableStream({
+        async start(controller) {
+            const reader = stream.getReader();
+            try {
+                emitProgress(Date.now(), false);
+                while (true) {
+                    const { done, value } = await reader.read();
+                    if (done) {
+                        emitProgress(Date.now(), true);
+                        controller.close();
+                        break;
+                    }
+                    loaded += value.byteLength;
+                    const now = Date.now();
+                    const shouldUpdate = now - lastUpdate >= throttleMs ||
+                        (total && loaded === total);
+                    if (shouldUpdate) {
+                        emitProgress(now, false);
+                    }
+                    controller.enqueue(value);
+                }
+            }
+            catch (error) {
+                controller.error(error);
+                throw error;
+            }
+        },
+    });
+}
+export function calculateProgress(loaded, total, direction) {
+    const progress = {
+        loaded,
+        transferred: loaded,
+        total,
+        direction,
+    };
+    if (total && total > 0) {
+        progress.percent = (loaded / total) * 100;
+    }
+    return progress;
+}

package/dist/browser/utils/request-pool.d.ts

@@ -0,0 +1,22 @@
+import type { Middleware } from '../types/index.js';
+export interface RequestPoolOptions {
+    concurrency?: number;
+    requestsPerInterval?: number;
+    interval?: number;
+}
+export declare class RequestPool {
+    private readonly concurrency;
+    private readonly requestsPerInterval?;
+    private readonly interval?;
+    private queue;
+    private active;
+    private windowStart;
+    private startedInWindow;
+    private waitingTimer?;
+    constructor(options?: RequestPoolOptions);
+    run<T>(fn: () => Promise<T>, signal?: AbortSignal): Promise<T>;
+    private _removeFromQueue;
+    private _canStart;
+    private _schedule;
+    asMiddleware(): Middleware;
+}

package/dist/browser/utils/request-pool.js

@@ -0,0 +1,101 @@
+import { ReckerError } from '../core/errors.js';
+export class RequestPool {
+    concurrency;
+    requestsPerInterval;
+    interval;
+    queue = [];
+    active = 0;
+    windowStart = 0;
+    startedInWindow = 0;
+    waitingTimer;
+    constructor(options = {}) {
+        this.concurrency = options.concurrency ?? Number.POSITIVE_INFINITY;
+        this.requestsPerInterval = options.requestsPerInterval;
+        this.interval = options.interval;
+    }
+    run(fn, signal) {
+        if (signal?.aborted) {
+            return Promise.reject(signal.reason ?? new ReckerError('Request aborted before enqueue', undefined, undefined, ['Ensure the AbortSignal is not already aborted when calling the request.', 'Remove or reset the signal before enqueueing.']));
+        }
+        return new Promise((resolve, reject) => {
+            const request = { fn, resolve, reject, signal };
+            if (signal) {
+                const onAbort = () => {
+                    this._removeFromQueue(request);
+                    reject(signal.reason ?? new ReckerError('Request aborted while queued', undefined, undefined, ['Avoid aborting immediately after queuing.', 'Increase timeouts or adjust rate limits if cancellations are unintended.']));
+                    this._schedule();
+                };
+                signal.addEventListener('abort', onAbort, { once: true });
+                request.abortCleanup = () => signal.removeEventListener('abort', onAbort);
+            }
+            this.queue.push(request);
+            this._schedule();
+        });
+    }
+    _removeFromQueue(request) {
+        const index = this.queue.indexOf(request);
+        if (index >= 0) {
+            this.queue.splice(index, 1);
+        }
+    }
+    _canStart(now) {
+        if (this.active >= this.concurrency) {
+            return false;
+        }
+        if (this.requestsPerInterval == null || this.interval == null) {
+            return true;
+        }
+        if (now - this.windowStart >= this.interval) {
+            this.windowStart = now;
+            this.startedInWindow = 0;
+        }
+        if (this.startedInWindow < this.requestsPerInterval) {
+            return true;
+        }
+        return false;
+    }
+    _schedule() {
+        if (this.waitingTimer) {
+            return;
+        }
+        const now = Date.now();
+        while (this.queue.length > 0 && this._canStart(Date.now())) {
+            const request = this.queue.shift();
+            if (request.signal?.aborted) {
+                request.abortCleanup?.();
+                request.reject(request.signal.reason ?? new ReckerError('Request aborted while queued', undefined, undefined, ['Avoid aborting immediately after queuing.', 'Increase timeouts or adjust rate limits if cancellations are unintended.']));
+                continue;
+            }
+            this.active++;
+            this.startedInWindow++;
+            const clearAbort = request.abortCleanup;
+            if (clearAbort) {
+                clearAbort();
+                request.abortCleanup = undefined;
+            }
+            Promise.resolve()
+                .then(() => request.fn())
+                .then((result) => request.resolve(result))
+                .catch((error) => request.reject(error))
+                .finally(() => {
+                this.active--;
+                this._schedule();
+            });
+        }
+        if (this.queue.length > 0 &&
+            this.requestsPerInterval != null &&
+            this.interval != null &&
+            !this._canStart(Date.now())) {
+            const wait = Math.max(0, this.windowStart + this.interval - Date.now());
+            this.waitingTimer = setTimeout(() => {
+                this.waitingTimer = undefined;
+                this._schedule();
+            }, wait);
+        }
+    }
+    asMiddleware() {
+        return async (req, next) => {
+            return this.run(() => next(req), req.signal);
+        };
+    }
+}

package/dist/browser/utils/sse.d.ts

@@ -0,0 +1,7 @@
+export interface SSEEvent {
+    id?: string;
+    event?: string;
+    data: string;
+    retry?: number;
+}
+export declare function parseSSE(response: Response): AsyncGenerator<SSEEvent>;

package/dist/browser/utils/sse.js

@@ -0,0 +1,67 @@
+import { StreamError } from '../core/errors.js';
+export async function* parseSSE(response) {
+    if (!response.body) {
+        throw new StreamError('Response body is empty; cannot parse SSE stream.', {
+            streamType: 'sse',
+            retriable: true,
+        });
+    }
+    const reader = response.body.getReader();
+    const decoder = new TextDecoder();
+    let buffer = '';
+    let currentEvent = { data: '' };
+    try {
+        while (true) {
+            const { done, value } = await reader.read();
+            if (done)
+                break;
+            buffer += decoder.decode(value, { stream: true });
+            const lines = buffer.split(/\r\n|\r|\n/);
+            buffer = lines.pop() || '';
+            for (const line of lines) {
+                if (line.trim() === '') {
+                    if (currentEvent.data || currentEvent.event || currentEvent.id) {
+                        if (currentEvent.data.endsWith('\n')) {
+                            currentEvent.data = currentEvent.data.slice(0, -1);
+                        }
+                        yield currentEvent;
+                        currentEvent = { data: '' };
+                    }
+                    continue;
+                }
+                const colonIndex = line.indexOf(':');
+                let field = line;
+                let value = '';
+                if (colonIndex !== -1) {
+                    field = line.slice(0, colonIndex);
+                    value = line.slice(colonIndex + 1);
+                    if (value.startsWith(' ')) {
+                        value = value.slice(1);
+                    }
+                }
+                switch (field) {
+                    case 'data':
+                        currentEvent.data += value + '\n';
+                        break;
+                    case 'event':
+                        currentEvent.event = value;
+                        break;
+                    case 'id':
+                        currentEvent.id = value;
+                        break;
+                    case 'retry':
+                        const retry = parseInt(value, 10);
+                        if (!isNaN(retry)) {
+                            currentEvent.retry = retry;
+                        }
+                        break;
+                }
+            }
+        }
+        if (buffer.trim() !== '') {
+        }
+    }
+    finally {
+        reader.releaseLock();
+    }
+}

package/dist/browser/utils/streaming.d.ts

@@ -0,0 +1,17 @@
+import { Readable } from 'node:stream';
+export declare function webToNodeStream(webStream: ReadableStream): Readable;
+export declare function nodeToWebStream(nodeStream: Readable): ReadableStream<Uint8Array>;
+export interface StreamProgressOptions {
+    onProgress?: (progress: {
+        loaded: number;
+        total?: number;
+        percent?: number;
+    }) => void;
+    total?: number;
+}
+export declare function trackStreamProgress(stream: Readable, options?: StreamProgressOptions): Readable;
+export declare function pipeStream(source: Readable, destination: NodeJS.WritableStream, options?: StreamProgressOptions): Promise<void>;
+export declare function createUploadStream(source: Readable): {
+    stream: ReadableStream<Uint8Array<ArrayBufferLike>>;
+    promise: Promise<void>;
+};