recker 1.0.26 → 1.0.27

package/dist/browser/core/client.js

@@ -0,0 +1,667 @@
+import { consoleLogger } from '../types/index.js';
+import { HttpRequest } from './request.js';
+import { UndiciTransport } from '../transport/undici.js';
+import { RequestPromise } from './request-promise.js';
+import { HttpError, MaxSizeExceededError, ConfigurationError, ValidationError, TimeoutError } from '../core/errors.js';
+import { processBody, createFormData, isPlainObject } from '../utils/body.js';
+import { AgentManager } from '../utils/agent-manager.js';
+import { RequestPool } from '../utils/request-pool.js';
+import { normalizeConcurrency } from '../utils/concurrency.js';
+import { getDefaultUserAgent } from '../utils/user-agent.js';
+import { paginate, streamPages } from '../plugins/pagination.js';
+import { retryPlugin } from '../plugins/retry.js';
+import { cachePlugin } from '../plugins/cache.js';
+import { dedupPlugin } from '../plugins/dedup.js';
+import { createXSRFMiddleware } from '../plugins/xsrf.js';
+import { createCompressionMiddleware } from '../plugins/compression.js';
+import { serializeXML } from '../plugins/xml.js';
+import { MemoryStorage } from '../cache/memory-storage.js';
+import { FileStorage } from '../cache/basic-file-storage.js';
+import { RequestRunner } from '../runner/request-runner.js';
+import { ReckerWebSocket } from '../websocket/client.js';
+import { whois as performWhois, isDomainAvailable } from '../utils/whois.js';
+import { MemoryCookieJar } from '../cookies/memory-cookie-jar.js';
+import { scrape as scrapeHelper } from '../plugins/scrape.js';
+import { HlsPromise } from '../plugins/hls.js';
+export class Client {
+    baseUrl;
+    middlewares;
+    hooks;
+    transport;
+    defaultHeaders;
+    defaultParams;
+    paginationConfig;
+    handler;
+    logger;
+    debugEnabled;
+    agentManager;
+    concurrencyConfig;
+    requestPool;
+    maxResponseSize;
+    cookieJar;
+    cookieIgnoreInvalid = false;
+    defaultTimeout;
+    constructor(options = {}) {
+        this.baseUrl = options.baseUrl || '';
+        this.middlewares = options.middlewares || [];
+        this.defaultTimeout = options.timeout;
+        this.hooks = {
+            beforeRequest: options.hooks?.beforeRequest || [],
+            afterResponse: options.hooks?.afterResponse || [],
+            onError: options.hooks?.onError || [],
+            onRetry: options.hooks?.onRetry || [],
+            onUrlResolved: options.hooks?.onUrlResolved || [],
+        };
+        this.defaultHeaders = {
+            'User-Agent': getDefaultUserAgent(),
+            ...(options.headers || {})
+        };
+        this.defaultParams = options.defaults?.params || {};
+        this.paginationConfig = options.pagination;
+        this.maxResponseSize = options.maxResponseSize;
+        this.debugEnabled = options.debug === true;
+        if (this.debugEnabled) {
+            this.logger = options.logger ?? consoleLogger;
+        }
+        else if (options.logger) {
+            this.logger = options.logger;
+        }
+        this.concurrencyConfig = normalizeConcurrency({
+            concurrency: options.concurrency,
+            http2: options.http2
+        });
+        if (options.transport) {
+            this.transport = options.transport;
+        }
+        else {
+            let http2Options;
+            if (options.http2) {
+                if (typeof options.http2 === 'boolean') {
+                    http2Options = { enabled: options.http2 };
+                }
+                else {
+                    http2Options = options.http2;
+                }
+            }
+            this.agentManager = new AgentManager(this.concurrencyConfig.agent);
+            this.transport = new UndiciTransport(this.baseUrl || undefined, {
+                proxy: options.proxy,
+                http2: http2Options,
+                dns: options.dns,
+                agent: this.agentManager,
+                socketPath: options.socketPath,
+                tls: options.tls,
+                observability: options.observability
+            });
+        }
+        if (options.retry) {
+            retryPlugin(options.retry)(this);
+        }
+        if (this.concurrencyConfig.max < Infinity || this.concurrencyConfig.requestsPerInterval < Infinity) {
+            this.requestPool = new RequestPool({
+                concurrency: this.concurrencyConfig.max,
+                requestsPerInterval: this.concurrencyConfig.requestsPerInterval,
+                interval: this.concurrencyConfig.interval
+            });
+            this.middlewares.unshift(this.requestPool.asMiddleware());
+            if (this.debugEnabled && this.logger) {
+                this.logger.debug(`Global concurrency limit: ${this.concurrencyConfig.max} concurrent requests`);
+            }
+        }
+        else {
+            if (this.debugEnabled && this.logger) {
+                this.logger.debug('No global concurrency limit (allows unlimited parallel batches)');
+            }
+        }
+        if (options.dedup) {
+            dedupPlugin(options.dedup)(this);
+        }
+        if (options.cache) {
+            let storage;
+            if (options.cache.storage) {
+                storage = options.cache.storage;
+            }
+            else if (options.cache.driver === 'file') {
+                storage = new FileStorage(options.cache.fileStoragePath);
+            }
+            else {
+                storage = new MemoryStorage();
+            }
+            cachePlugin({
+                ...options.cache,
+                storage
+            })(this);
+        }
+        if (options.plugins) {
+            options.plugins.forEach((plugin) => plugin(this));
+        }
+        if (options.compression) {
+            const compressionMiddleware = createCompressionMiddleware(options.compression);
+            if (compressionMiddleware) {
+                this.middlewares.push(compressionMiddleware);
+            }
+        }
+        if (options.xsrf) {
+            const xsrfMiddleware = createXSRFMiddleware(options.xsrf);
+            if (xsrfMiddleware) {
+                this.middlewares.push(xsrfMiddleware);
+            }
+        }
+        if (options.cookies) {
+            this.setupCookieJar(options.cookies);
+        }
+        if (this.maxResponseSize !== undefined) {
+            this.middlewares.push(this.createMaxSizeMiddleware(this.maxResponseSize));
+        }
+        if (this.debugEnabled && this.logger) {
+            this.middlewares.unshift(this.createLoggingMiddleware(this.logger));
+        }
+        this.middlewares.push(this.httpErrorMiddleware);
+        this.handler = this.composeMiddlewares();
+    }
+    createLoggingMiddleware(logger) {
+        return async (req, next) => {
+            const startTime = Date.now();
+            logger.debug({ type: 'request', method: req.method, url: req.url }, `→ ${req.method} ${req.url}`);
+            try {
+                const response = await next(req);
+                const duration = Date.now() - startTime;
+                logger.debug({
+                    type: 'response',
+                    method: req.method,
+                    url: req.url,
+                    status: response.status,
+                    duration,
+                    timings: response.timings,
+                }, `← ${response.status} ${req.method} ${req.url} (${duration}ms)`);
+                return response;
+            }
+            catch (error) {
+                const duration = Date.now() - startTime;
+                const err = error;
+                logger.error({
+                    type: 'error',
+                    method: req.method,
+                    url: req.url,
+                    error: err.message,
+                    errorName: err.name,
+                    duration,
+                }, `✖ ${req.method} ${req.url} - ${err.message}`);
+                throw error;
+            }
+        };
+    }
+    createMaxSizeMiddleware(globalMaxSize) {
+        return async (req, next) => {
+            const response = await next(req);
+            const limit = req.maxResponseSize ?? globalMaxSize;
+            if (limit === undefined)
+                return response;
+            const contentLength = response.headers.get('Content-Length');
+            if (contentLength) {
+                const size = parseInt(contentLength, 10);
+                if (!isNaN(size) && size > limit) {
+                    throw new MaxSizeExceededError(limit, size, req);
+                }
+            }
+            return response;
+        };
+    }
+    setupCookieJar(options) {
+        if (options === true) {
+            this.cookieJar = new MemoryCookieJar();
+        }
+        else if (typeof options === 'object') {
+            if (options.jar === true) {
+                this.cookieJar = new MemoryCookieJar();
+            }
+            else if (options.jar && typeof options.jar === 'object') {
+                this.cookieJar = options.jar;
+            }
+            this.cookieIgnoreInvalid = options.ignoreInvalid ?? false;
+        }
+        if (this.cookieJar) {
+            this.middlewares.push(this.createCookieMiddleware());
+        }
+    }
+    createCookieMiddleware() {
+        return async (req, next) => {
+            const jar = this.cookieJar;
+            try {
+                const cookieString = await jar.getCookieString(req.url);
+                if (cookieString) {
+                    const existingCookie = req.headers.get('cookie');
+                    const newCookie = existingCookie
+                        ? `${existingCookie}; ${cookieString}`
+                        : cookieString;
+                    req.headers.set('cookie', newCookie);
+                }
+            }
+            catch (error) {
+                if (!this.cookieIgnoreInvalid) {
+                    throw error;
+                }
+            }
+            const response = await next(req);
+            const setCookieHeader = response.headers.get('set-cookie');
+            if (setCookieHeader) {
+                const cookies = this.splitSetCookieHeader(setCookieHeader);
+                for (const cookie of cookies) {
+                    try {
+                        await jar.setCookie(cookie, req.url);
+                    }
+                    catch (error) {
+                        if (!this.cookieIgnoreInvalid) {
+                            throw error;
+                        }
+                    }
+                }
+            }
+            return response;
+        };
+    }
+    splitSetCookieHeader(header) {
+        return header.split(/,(?=\s*[a-zA-Z0-9_-]+=)/g).map(s => s.trim());
+    }
+    composeMiddlewares() {
+        const chain = [...this.middlewares];
+        const transportDispatch = this.transport.dispatch.bind(this.transport);
+        if (this.hooks.beforeRequest?.length || this.hooks.afterResponse?.length) {
+            chain.unshift(this.hooksMiddleware);
+        }
+        if (chain.length === 0) {
+            return transportDispatch;
+        }
+        return chain.reduceRight((next, middleware) => {
+            return (req) => middleware(req, next);
+        }, transportDispatch);
+    }
+    hooksMiddleware = async (req, next) => {
+        let modifiedReq = req;
+        if (this.hooks.beforeRequest && this.hooks.beforeRequest.length > 0) {
+            for (const hook of this.hooks.beforeRequest) {
+                const result = await hook(modifiedReq);
+                if (result) {
+                    modifiedReq = result;
+                }
+            }
+        }
+        try {
+            let response = await next(modifiedReq);
+            if (this.hooks.afterResponse && this.hooks.afterResponse.length > 0) {
+                for (const hook of this.hooks.afterResponse) {
+                    const result = await hook(modifiedReq, response);
+                    if (result) {
+                        response = result;
+                    }
+                }
+            }
+            return response;
+        }
+        catch (error) {
+            if (this.hooks.onError && this.hooks.onError.length > 0) {
+                for (const hook of this.hooks.onError) {
+                    const result = await hook(error, modifiedReq);
+                    if (result) {
+                        return result;
+                    }
+                }
+            }
+            throw error;
+        }
+    };
+    httpErrorMiddleware = async (req, next) => {
+        const response = await next(req);
+        if (req.throwHttpErrors !== false && !response.ok && response.status !== 304) {
+            throw new HttpError(response, req);
+        }
+        return response;
+    };
+    use(middleware) {
+        this.middlewares.push(middleware);
+        this.handler = this.composeMiddlewares();
+        return this;
+    }
+    beforeRequest(hook) {
+        if (!this.hooks.beforeRequest) {
+            this.hooks.beforeRequest = [];
+        }
+        this.hooks.beforeRequest.push(hook);
+        this.handler = this.composeMiddlewares();
+        return this;
+    }
+    afterResponse(hook) {
+        if (!this.hooks.afterResponse) {
+            this.hooks.afterResponse = [];
+        }
+        this.hooks.afterResponse.push(hook);
+        this.handler = this.composeMiddlewares();
+        return this;
+    }
+    onError(hook) {
+        if (!this.hooks.onError) {
+            this.hooks.onError = [];
+        }
+        this.hooks.onError.push(hook);
+        this.handler = this.composeMiddlewares();
+        return this;
+    }
+    buildUrl(path, requestParams) {
+        const hasRequestParams = requestParams && Object.keys(requestParams).length > 0;
+        const hasDefaultParams = Object.keys(this.defaultParams).length > 0;
+        if (!hasRequestParams && !hasDefaultParams) {
+            if (path.startsWith('http://') || path.startsWith('https://')) {
+                return path;
+            }
+            if (this.baseUrl) {
+                const base = this.baseUrl.endsWith('/') ? this.baseUrl.slice(0, -1) : this.baseUrl;
+                const p = path.startsWith('/') ? path : '/' + path;
+                return base + p;
+            }
+            return path;
+        }
+        let finalPath = path;
+        const mergedParams = { ...this.defaultParams, ...requestParams };
+        const usedParams = new Set();
+        if (finalPath.includes(':')) {
+            finalPath = finalPath.replace(/:([a-zA-Z0-9_]+)/g, (match, paramName) => {
+                if (mergedParams && paramName in mergedParams) {
+                    usedParams.add(paramName);
+                    return encodeURIComponent(String(mergedParams[paramName]));
+                }
+                throw new ValidationError(`Missing required path parameter: ${paramName}`, {
+                    field: paramName,
+                    value: undefined,
+                });
+            });
+        }
+        let finalUrl;
+        if (finalPath.startsWith('http://') || finalPath.startsWith('https://')) {
+            finalUrl = finalPath;
+        }
+        else if (this.baseUrl) {
+            const base = this.baseUrl.endsWith('/') ? this.baseUrl.slice(0, -1) : this.baseUrl;
+            const p = finalPath.startsWith('/') ? finalPath : '/' + finalPath;
+            finalUrl = base + p;
+        }
+        else {
+            throw new ConfigurationError('Relative path provided without a baseUrl or explicit transport.', {
+                configKey: 'baseUrl',
+            });
+        }
+        const remainingKeys = Object.keys(mergedParams).filter((k) => !usedParams.has(k));
+        if (remainingKeys.length > 0) {
+            const queryParts = remainingKeys.map(key => `${encodeURIComponent(key)}=${encodeURIComponent(String(mergedParams[key]))}`);
+            const separator = finalUrl.includes('?') ? '&' : '?';
+            return finalUrl + separator + queryParts.join('&');
+        }
+        return finalUrl;
+    }
+    request(path, options = {}) {
+        const url = this.buildUrl(path, options.params);
+        let mergedHeaders;
+        if (options.headers) {
+            mergedHeaders = this.defaultHeaders instanceof Headers
+                ? new Headers(this.defaultHeaders)
+                : new Headers(this.defaultHeaders);
+            const optHeaders = options.headers instanceof Headers
+                ? options.headers
+                : new Headers(options.headers);
+            optHeaders.forEach((value, key) => mergedHeaders.append(key, value));
+        }
+        else {
+            mergedHeaders = this.defaultHeaders instanceof Headers
+                ? this.defaultHeaders
+                : new Headers(this.defaultHeaders);
+        }
+        const needsController = options.timeout || options.signal || this.defaultTimeout;
+        let controller;
+        let signal = options.signal;
+        let timeoutId;
+        let externalAbortCleanup;
+        if (needsController) {
+            controller = new AbortController();
+            signal = controller.signal;
+            if (options.signal) {
+                const externalSignal = options.signal;
+                const abortHandler = () => controller.abort(externalSignal.reason);
+                if (externalSignal.aborted) {
+                    abortHandler();
+                }
+                else {
+                    externalSignal.addEventListener('abort', abortHandler, { once: true });
+                    externalAbortCleanup = () => externalSignal.removeEventListener('abort', abortHandler);
+                }
+            }
+            const timeout = options.timeout ?? this.defaultTimeout;
+            if (timeout) {
+                const totalTimeout = typeof timeout === 'number' ? timeout : timeout.request;
+                if (totalTimeout) {
+                    timeoutId = setTimeout(() => controller.abort(new TimeoutError(req, {
+                        phase: 'request',
+                        timeout: totalTimeout,
+                    })), totalTimeout);
+                }
+            }
+        }
+        const req = new HttpRequest(url, {
+            ...options,
+            headers: mergedHeaders,
+            signal,
+            maxResponseSize: options.maxResponseSize ?? this.maxResponseSize
+        });
+        const responsePromise = this.handler(req);
+        if (timeoutId || externalAbortCleanup) {
+            responsePromise.finally(() => {
+                if (timeoutId)
+                    clearTimeout(timeoutId);
+                externalAbortCleanup?.();
+            }).catch(() => {
+            });
+        }
+        return new RequestPromise(responsePromise, controller);
+    }
+    get(path, options = {}) {
+        return this.request(path, { ...options, method: 'GET' });
+    }
+    async batch(requests, options = {}) {
+        const mapResponse = options.mapResponse ?? ((res) => res);
+        const batchConcurrency = options.concurrency ?? this.concurrencyConfig.runner.concurrency;
+        const runner = new RequestRunner({
+            concurrency: batchConcurrency,
+            retries: this.concurrencyConfig.runner.retries,
+            retryDelay: this.concurrencyConfig.runner.retryDelay
+        });
+        const runnerResult = await runner.run(requests, async (item) => {
+            const res = await this.request(item.path, item.options);
+            return mapResponse(res);
+        });
+        return runnerResult;
+    }
+    multi(requests, options = {}) {
+        return this.batch(requests, options);
+    }
+    requestWithBody(method, path, bodyOrOptions, options) {
+        let actualBody = bodyOrOptions;
+        let actualOptions = options;
+        const isOptionsEmpty = actualOptions === undefined ||
+            (typeof actualOptions === 'object' && actualOptions !== null && Object.keys(actualOptions).length === 0);
+        if (isOptionsEmpty && isPlainObject(bodyOrOptions)) {
+            const potentialOptions = bodyOrOptions;
+            if (potentialOptions.json !== undefined ||
+                potentialOptions.form !== undefined ||
+                potentialOptions.xml !== undefined ||
+                potentialOptions.body !== undefined ||
+                potentialOptions.headers !== undefined ||
+                potentialOptions.timeout !== undefined ||
+                potentialOptions.retry !== undefined ||
+                potentialOptions.hooks !== undefined ||
+                potentialOptions.searchParams !== undefined ||
+                potentialOptions.params !== undefined) {
+                actualOptions = bodyOrOptions;
+                actualBody = undefined;
+            }
+        }
+        actualOptions = actualOptions || {};
+        const { json, form, xml, ...restOptions } = actualOptions;
+        let finalBody = actualBody;
+        let explicitContentType;
+        if (form !== undefined) {
+            finalBody = createFormData(form);
+            explicitContentType = undefined;
+        }
+        else if (json !== undefined) {
+            finalBody = JSON.stringify(json);
+            explicitContentType = 'application/json';
+        }
+        else if (xml !== undefined) {
+            finalBody = '<?xml version="1.0" encoding="UTF-8"?>\n' + serializeXML(xml);
+            explicitContentType = 'application/xml';
+        }
+        else if (restOptions.body !== undefined) {
+            finalBody = restOptions.body;
+        }
+        const { body: processedBody, contentType } = processBody(finalBody);
+        const headers = new Headers(restOptions.headers);
+        const finalContentType = explicitContentType ?? contentType;
+        if (finalContentType && !headers.has('Content-Type')) {
+            headers.set('Content-Type', finalContentType);
+        }
+        return this.request(path, { ...restOptions, method, body: processedBody, headers });
+    }
+    post(path, body, options = {}) {
+        return this.requestWithBody('POST', path, body, options);
+    }
+    put(path, body, options = {}) {
+        return this.requestWithBody('PUT', path, body, options);
+    }
+    patch(path, body, options = {}) {
+        return this.requestWithBody('PATCH', path, body, options);
+    }
+    delete(path, options = {}) {
+        return this.request(path, { ...options, method: 'DELETE' });
+    }
+    head(path, options = {}) {
+        return this.request(path, { ...options, method: 'HEAD' });
+    }
+    options(path, options = {}) {
+        return this.request(path, { ...options, method: 'OPTIONS' });
+    }
+    trace(path, options = {}) {
+        return this.request(path, { ...options, method: 'TRACE' });
+    }
+    connect(path, options = {}) {
+        return this.request(path, { ...options, method: 'CONNECT' });
+    }
+    purge(path, options = {}) {
+        return this.request(path, { ...options, method: 'PURGE' });
+    }
+    propfind(path, body, options = {}) {
+        return this.requestWithBody('PROPFIND', path, body, options);
+    }
+    proppatch(path, body, options = {}) {
+        return this.requestWithBody('PROPPATCH', path, body, options);
+    }
+    mkcol(path, options = {}) {
+        return this.request(path, { ...options, method: 'MKCOL' });
+    }
+    copy(path, options = {}) {
+        return this.request(path, { ...options, method: 'COPY' });
+    }
+    move(path, options = {}) {
+        return this.request(path, { ...options, method: 'MOVE' });
+    }
+    lock(path, body, options = {}) {
+        return this.requestWithBody('LOCK', path, body, options);
+    }
+    unlock(path, options = {}) {
+        return this.request(path, { ...options, method: 'UNLOCK' });
+    }
+    link(path, body, options = {}) {
+        return this.requestWithBody('LINK', path, body, options);
+    }
+    unlink(path, body, options = {}) {
+        return this.requestWithBody('UNLINK', path, body, options);
+    }
+    scrape(path, options = {}) {
+        const method = options.method || 'GET';
+        const requestPromise = this.request(path, { ...options, method });
+        return scrapeHelper(requestPromise);
+    }
+    paginate(path, options = {}) {
+        const { getItems, getNextUrl, maxPages, pageParam, limitParam, resultsPath, nextCursorPath, ...reqOptions } = options;
+        const paginationOpts = {
+            getItems,
+            getNextUrl,
+            maxPages,
+            pageParam: pageParam || this.paginationConfig?.pageParam,
+            limitParam: limitParam || this.paginationConfig?.limitParam,
+            resultsPath: resultsPath || this.paginationConfig?.resultsPath,
+            nextCursorPath: nextCursorPath || this.paginationConfig?.nextCursorPath,
+        };
+        return paginate(this, path, reqOptions, paginationOpts);
+    }
+    pages(path, options = {}) {
+        const { getNextUrl, maxPages, pageParam, limitParam, resultsPath, nextCursorPath, ...reqOptions } = options;
+        const paginationOpts = {
+            getNextUrl,
+            maxPages,
+            pageParam: pageParam || this.paginationConfig?.pageParam,
+            limitParam: limitParam || this.paginationConfig?.limitParam,
+            nextCursorPath: nextCursorPath || this.paginationConfig?.nextCursorPath,
+        };
+        return streamPages(this, path, reqOptions, paginationOpts);
+    }
+    page(path, pageNumber, options = {}) {
+        const pageParam = options.pageParam || this.paginationConfig?.pageParam || 'page';
+        const url = new URL(path.startsWith('http') ? path : `http://base${path}`);
+        const params = { ...options.params, [pageParam]: pageNumber };
+        return this.request(path, { ...options, params });
+    }
+    async getAll(path, options = {}) {
+        const items = [];
+        for await (const item of this.paginate(path, options)) {
+            items.push(item);
+        }
+        return items;
+    }
+    websocket(path, options = {}) {
+        let wsUrl;
+        if (path.startsWith('ws://') || path.startsWith('wss://')) {
+            wsUrl = path;
+        }
+        else if (this.baseUrl) {
+            const base = this.baseUrl.replace(/^http/, 'ws');
+            wsUrl = new URL(path, base).toString();
+        }
+        else {
+            throw new ConfigurationError('WebSocket requires either a full ws:// URL or a baseUrl', {
+                configKey: 'baseUrl',
+            });
+        }
+        const headersObj = {};
+        if (this.defaultHeaders) {
+            const headers = new Headers(this.defaultHeaders);
+            headers.forEach((value, key) => {
+                headersObj[key] = value;
+            });
+        }
+        const finalHeaders = { ...headersObj, ...options.headers };
+        return new ReckerWebSocket(wsUrl, { ...options, headers: finalHeaders });
+    }
+    ws(path, options = {}) {
+        return this.websocket(path, options);
+    }
+    async whois(query, options) {
+        return performWhois(query, options);
+    }
+    async isDomainAvailable(domain, options) {
+        return isDomainAvailable(domain, options);
+    }
+    hls(manifestUrl, options = {}) {
+        return new HlsPromise(this, manifestUrl, options);
+    }
+}
+export function createClient(options = {}) {
+    return new Client(options);
+}