recker 1.0.26 → 1.0.27

package/dist/browser/scrape/extractors.js

@@ -0,0 +1,356 @@
+function resolveUrl(url, baseUrl) {
+    if (!url)
+        return '';
+    if (!baseUrl)
+        return url;
+    try {
+        if (url.startsWith('http://') || url.startsWith('https://') || url.startsWith('//')) {
+            return url;
+        }
+        return new URL(url, baseUrl).href;
+    }
+    catch {
+        return url;
+    }
+}
+function classifyLinkType(href, baseUrl) {
+    if (!href)
+        return undefined;
+    if (href.startsWith('mailto:'))
+        return 'mailto';
+    if (href.startsWith('tel:'))
+        return 'tel';
+    if (href.startsWith('#'))
+        return 'anchor';
+    if (baseUrl) {
+        try {
+            const base = new URL(baseUrl);
+            const link = new URL(href, baseUrl);
+            return link.hostname === base.hostname ? 'internal' : 'external';
+        }
+        catch {
+            return undefined;
+        }
+    }
+    if (href.startsWith('http://') || href.startsWith('https://') || href.startsWith('//')) {
+        return 'external';
+    }
+    return 'internal';
+}
+export function extractLinks($, options) {
+    const selector = options?.selector || 'a[href]';
+    const links = [];
+    $(selector).each((_, element) => {
+        const $el = $(element);
+        let href = $el.attr('href') || '';
+        if (options?.absolute && options?.baseUrl) {
+            href = resolveUrl(href, options.baseUrl);
+        }
+        links.push({
+            href,
+            text: $el.text().trim(),
+            rel: $el.attr('rel'),
+            target: $el.attr('target'),
+            title: $el.attr('title'),
+            type: classifyLinkType(href, options?.baseUrl),
+        });
+    });
+    return links;
+}
+export function extractImages($, options) {
+    const selector = options?.selector || 'img[src]';
+    const images = [];
+    $(selector).each((_, element) => {
+        const $el = $(element);
+        let src = $el.attr('src') || '';
+        if (options?.absolute && options?.baseUrl) {
+            src = resolveUrl(src, options.baseUrl);
+        }
+        const width = $el.attr('width');
+        const height = $el.attr('height');
+        images.push({
+            src,
+            alt: $el.attr('alt'),
+            title: $el.attr('title'),
+            width: width ? parseInt(width, 10) : undefined,
+            height: height ? parseInt(height, 10) : undefined,
+            srcset: $el.attr('srcset'),
+            loading: $el.attr('loading'),
+        });
+    });
+    return images;
+}
+export function extractMeta($) {
+    const meta = {};
+    meta.title = $('title').first().text().trim() || undefined;
+    const charsetMeta = $('meta[charset]').attr('charset');
+    if (charsetMeta) {
+        meta.charset = charsetMeta;
+    }
+    else {
+        const contentType = $('meta[http-equiv="Content-Type"]').attr('content');
+        if (contentType) {
+            const charsetMatch = contentType.match(/charset=([^;]+)/i);
+            if (charsetMatch) {
+                meta.charset = charsetMatch[1].trim();
+            }
+        }
+    }
+    const canonical = $('link[rel="canonical"]').attr('href');
+    if (canonical) {
+        meta.canonical = canonical;
+    }
+    $('meta[name]').each((_, element) => {
+        const $el = $(element);
+        const name = $el.attr('name')?.toLowerCase();
+        const content = $el.attr('content');
+        if (!name || !content)
+            return;
+        switch (name) {
+            case 'description':
+                meta.description = content;
+                break;
+            case 'keywords':
+                meta.keywords = content.split(',').map((k) => k.trim()).filter(Boolean);
+                break;
+            case 'author':
+                meta.author = content;
+                break;
+            case 'robots':
+                meta.robots = content;
+                break;
+            case 'viewport':
+                meta.viewport = content;
+                break;
+            default:
+                meta[name] = content;
+        }
+    });
+    return meta;
+}
+export function extractOpenGraph($) {
+    const og = {};
+    const images = [];
+    $('meta[property^="og:"]').each((_, element) => {
+        const $el = $(element);
+        const property = $el.attr('property');
+        const content = $el.attr('content');
+        if (!property || !content)
+            return;
+        const key = property.replace('og:', '');
+        switch (key) {
+            case 'title':
+                og.title = content;
+                break;
+            case 'type':
+                og.type = content;
+                break;
+            case 'url':
+                og.url = content;
+                break;
+            case 'image':
+                images.push(content);
+                break;
+            case 'description':
+                og.description = content;
+                break;
+            case 'site_name':
+                og.siteName = content;
+                break;
+            case 'locale':
+                og.locale = content;
+                break;
+            default:
+                og[key.replace(/_([a-z])/g, (_, c) => c.toUpperCase())] = content;
+        }
+    });
+    if (images.length === 1) {
+        og.image = images[0];
+    }
+    else if (images.length > 1) {
+        og.image = images;
+    }
+    return og;
+}
+export function extractTwitterCard($) {
+    const twitter = {};
+    $('meta[name^="twitter:"]').each((_, element) => {
+        const $el = $(element);
+        const name = $el.attr('name');
+        const content = $el.attr('content');
+        if (!name || !content)
+            return;
+        const key = name.replace('twitter:', '');
+        switch (key) {
+            case 'card':
+                twitter.card = content;
+                break;
+            case 'site':
+                twitter.site = content;
+                break;
+            case 'creator':
+                twitter.creator = content;
+                break;
+            case 'title':
+                twitter.title = content;
+                break;
+            case 'description':
+                twitter.description = content;
+                break;
+            case 'image':
+                twitter.image = content;
+                break;
+            default:
+                twitter[key] = content;
+        }
+    });
+    return twitter;
+}
+export function extractJsonLd($) {
+    const results = [];
+    $('script[type="application/ld+json"]').each((_, element) => {
+        const content = $(element).html();
+        if (!content)
+            return;
+        try {
+            const data = JSON.parse(content);
+            if (data['@graph'] && Array.isArray(data['@graph'])) {
+                results.push(...data['@graph']);
+            }
+            else if (Array.isArray(data)) {
+                results.push(...data);
+            }
+            else {
+                results.push(data);
+            }
+        }
+        catch {
+        }
+    });
+    return results;
+}
+export function extractForms($, selector) {
+    const forms = [];
+    const formSelector = selector || 'form';
+    $(formSelector).each((_, formElement) => {
+        const $form = $(formElement);
+        const fields = [];
+        $form.find('input, select, textarea').each((_, fieldElement) => {
+            const $field = $(fieldElement);
+            const tagName = fieldElement.tagName?.toLowerCase();
+            const type = $field.attr('type') || (tagName === 'textarea' ? 'textarea' : tagName === 'select' ? 'select' : 'text');
+            const field = {
+                name: $field.attr('name'),
+                type,
+                value: $field.val(),
+                placeholder: $field.attr('placeholder'),
+                required: $field.attr('required') !== undefined,
+            };
+            if (tagName === 'select') {
+                field.options = [];
+                $field.find('option').each((_, optionElement) => {
+                    const $option = $(optionElement);
+                    field.options.push({
+                        value: $option.attr('value') || $option.text(),
+                        text: $option.text().trim(),
+                    });
+                });
+            }
+            fields.push(field);
+        });
+        forms.push({
+            action: $form.attr('action'),
+            method: $form.attr('method')?.toUpperCase(),
+            name: $form.attr('name'),
+            id: $form.attr('id'),
+            fields,
+        });
+    });
+    return forms;
+}
+export function extractTables($, selector) {
+    const tables = [];
+    const tableSelector = selector || 'table';
+    $(tableSelector).each((_, tableElement) => {
+        const $table = $(tableElement);
+        const headers = [];
+        const rows = [];
+        const caption = $table.find('caption').first().text().trim() || undefined;
+        const $headerCells = $table.find('thead th, thead td, tr:first-child th');
+        if ($headerCells.length > 0) {
+            $headerCells.each((_, th) => {
+                headers.push($(th).text().trim());
+            });
+        }
+        const $rows = $table.find('tbody tr, tr').not(':has(th):not(:first-child)');
+        $rows.each((_, tr) => {
+            const row = [];
+            $(tr).find('td').each((_, td) => {
+                row.push($(td).text().trim());
+            });
+            if (row.length > 0) {
+                rows.push(row);
+            }
+        });
+        if (headers.length === 0 && rows.length > 0) {
+            const $firstRowCells = $table.find('tr:first-child td, tr:first-child th');
+            if ($firstRowCells.length > 0) {
+                $firstRowCells.each((_, cell) => {
+                    headers.push($(cell).text().trim());
+                });
+                if ($table.find('tr:first-child th').length > 0) {
+                }
+                else {
+                    const firstRow = [];
+                    $table.find('tr:first-child td').each((_, td) => {
+                        firstRow.push($(td).text().trim());
+                    });
+                    if (firstRow.length > 0 && !rows.some(r => r.join('') === firstRow.join(''))) {
+                        rows.unshift(firstRow);
+                    }
+                }
+            }
+        }
+        tables.push({ headers, rows, caption });
+    });
+    return tables;
+}
+export function extractScripts($) {
+    const scripts = [];
+    $('script').each((_, element) => {
+        const $el = $(element);
+        const src = $el.attr('src');
+        const inline = !src ? $el.html()?.trim() : undefined;
+        if (!src && !inline)
+            return;
+        scripts.push({
+            src,
+            type: $el.attr('type'),
+            async: $el.attr('async') !== undefined,
+            defer: $el.attr('defer') !== undefined,
+            inline,
+        });
+    });
+    return scripts;
+}
+export function extractStyles($) {
+    const styles = [];
+    $('link[rel="stylesheet"]').each((_, element) => {
+        const $el = $(element);
+        styles.push({
+            href: $el.attr('href'),
+            media: $el.attr('media'),
+        });
+    });
+    $('style').each((_, element) => {
+        const $el = $(element);
+        const inline = $el.html()?.trim();
+        if (inline) {
+            styles.push({
+                media: $el.attr('media'),
+                inline,
+            });
+        }
+    });
+    return styles;
+}

package/dist/browser/scrape/types.d.ts

@@ -0,0 +1,107 @@
+export interface ExtractedLink {
+    href: string;
+    text: string;
+    rel?: string;
+    target?: string;
+    title?: string;
+    type?: 'internal' | 'external' | 'anchor' | 'mailto' | 'tel';
+}
+export interface ExtractedImage {
+    src: string;
+    alt?: string;
+    title?: string;
+    width?: number;
+    height?: number;
+    srcset?: string;
+    loading?: 'lazy' | 'eager';
+}
+export interface ExtractedMeta {
+    title?: string;
+    description?: string;
+    keywords?: string[];
+    author?: string;
+    robots?: string;
+    canonical?: string;
+    viewport?: string;
+    charset?: string;
+    [key: string]: string | string[] | undefined;
+}
+export interface OpenGraphData {
+    title?: string;
+    type?: string;
+    url?: string;
+    image?: string | string[];
+    description?: string;
+    siteName?: string;
+    locale?: string;
+    [key: string]: string | string[] | undefined;
+}
+export interface TwitterCardData {
+    card?: 'summary' | 'summary_large_image' | 'app' | 'player';
+    site?: string;
+    creator?: string;
+    title?: string;
+    description?: string;
+    image?: string;
+    [key: string]: string | undefined;
+}
+export interface JsonLdData {
+    '@context'?: string;
+    '@type'?: string;
+    [key: string]: unknown;
+}
+export interface ExtractedForm {
+    action?: string;
+    method?: string;
+    name?: string;
+    id?: string;
+    fields: ExtractedFormField[];
+}
+export interface ExtractedFormField {
+    name?: string;
+    type?: string;
+    value?: string;
+    placeholder?: string;
+    required?: boolean;
+    options?: {
+        value: string;
+        text: string;
+    }[];
+}
+export interface ExtractedTable {
+    headers: string[];
+    rows: string[][];
+    caption?: string;
+}
+export interface ExtractedScript {
+    src?: string;
+    type?: string;
+    async?: boolean;
+    defer?: boolean;
+    inline?: string;
+}
+export interface ExtractedStyle {
+    href?: string;
+    media?: string;
+    inline?: string;
+}
+export type ExtractionSchemaField = string | {
+    selector: string;
+    attribute?: string;
+    multiple?: boolean;
+    transform?: (value: string) => unknown;
+};
+export interface ExtractionSchema {
+    [key: string]: ExtractionSchemaField;
+}
+export interface ScrapeOptions {
+    baseUrl?: string;
+}
+export interface LinkExtractionOptions {
+    selector?: string;
+    absolute?: boolean;
+}
+export interface ImageExtractionOptions {
+    selector?: string;
+    absolute?: boolean;
+}

package/dist/browser/scrape/types.js

@@ -0,0 +1 @@
+export {};

package/dist/browser/transport/fetch.d.ts

@@ -0,0 +1,11 @@
+import { ReckerRequest, ReckerResponse, Transport } from '../types/index.js';
+export interface FetchTransportOptions {
+    credentials?: RequestCredentials;
+    cache?: RequestCache;
+    keepalive?: boolean;
+}
+export declare class FetchTransport implements Transport {
+    private options;
+    constructor(options?: FetchTransportOptions);
+    dispatch(req: ReckerRequest): Promise<ReckerResponse>;
+}

package/dist/browser/transport/fetch.js

@@ -0,0 +1,143 @@
+export class FetchTransport {
+    options;
+    constructor(options = {}) {
+        this.options = options;
+    }
+    async dispatch(req) {
+        const start = performance.now();
+        let timeoutId;
+        let abortController;
+        const timeoutMs = typeof req.timeout === 'number'
+            ? req.timeout
+            : req.timeout?.request;
+        let signal = req.signal;
+        if (timeoutMs && !signal) {
+            abortController = new AbortController();
+            signal = abortController.signal;
+            timeoutId = setTimeout(() => abortController.abort(), timeoutMs);
+        }
+        const requestInit = {
+            method: req.method,
+            headers: req.headers,
+            body: req.body,
+            signal,
+            credentials: this.options.credentials,
+            cache: this.options.cache,
+            keepalive: this.options.keepalive ?? true,
+            duplex: req.body ? 'half' : undefined
+        };
+        try {
+            const response = await globalThis.fetch(req.url, requestInit);
+            const totalTime = performance.now() - start;
+            const timings = {
+                total: totalTime,
+                firstByte: totalTime,
+            };
+            return new FetchResponseWrapper(response, timings);
+        }
+        catch (error) {
+            if (error.name === 'AbortError' && abortController) {
+                const timeoutError = new Error(`Request timeout after ${timeoutMs}ms`);
+                timeoutError.name = 'TimeoutError';
+                throw timeoutError;
+            }
+            throw error;
+        }
+        finally {
+            if (timeoutId)
+                clearTimeout(timeoutId);
+        }
+    }
+}
+class FetchResponseWrapper {
+    raw;
+    timings;
+    constructor(raw, timings) {
+        this.raw = raw;
+        this.timings = timings;
+    }
+    get status() { return this.raw.status; }
+    get statusText() { return this.raw.statusText; }
+    get headers() { return this.raw.headers; }
+    get ok() { return this.raw.ok; }
+    get url() { return this.raw.url; }
+    get connection() { return {}; }
+    json() { return this.raw.json(); }
+    text() { return this.raw.text(); }
+    blob() { return this.raw.blob(); }
+    async cleanText() { return (await this.text()).replace(/<[^>]*>?/gm, ''); }
+    read() { return this.raw.body; }
+    clone() { return new FetchResponseWrapper(this.raw.clone(), this.timings); }
+    async *sse() {
+        if (!this.raw.body)
+            return;
+        const stream = this.raw.body.pipeThrough(new TextDecoderStream());
+        const reader = stream.getReader();
+        let buffer = '';
+        while (true) {
+            const { done, value } = await reader.read();
+            if (done)
+                break;
+            buffer += value;
+            const events = buffer.split('\n\n');
+            buffer = events.pop() || '';
+            for (const event of events) {
+                if (!event.trim())
+                    continue;
+                let data = '';
+                let eventType;
+                let id;
+                let retry;
+                const lines = event.split('\n');
+                for (const line of lines) {
+                    if (line.startsWith('data: ')) {
+                        data = data ? data + '\n' + line.slice(6) : line.slice(6);
+                    }
+                    else if (line.startsWith('event: ')) {
+                        eventType = line.slice(7);
+                    }
+                    else if (line.startsWith('id: ')) {
+                        id = line.slice(4);
+                    }
+                    else if (line.startsWith('retry: ')) {
+                        retry = parseInt(line.slice(7), 10);
+                    }
+                }
+                if (data) {
+                    yield { data, event: eventType, id, retry };
+                }
+            }
+        }
+    }
+    async *download() {
+        if (!this.raw.body)
+            return;
+        const reader = this.raw.body.getReader();
+        let loaded = 0;
+        const total = Number(this.raw.headers.get('content-length')) || undefined;
+        while (true) {
+            const { done, value } = await reader.read();
+            if (done)
+                break;
+            loaded += value.length;
+            yield {
+                loaded,
+                transferred: loaded,
+                total,
+                percent: total ? (loaded / total) * 100 : undefined,
+                direction: 'download'
+            };
+        }
+    }
+    async *[Symbol.asyncIterator]() {
+        if (!this.raw.body)
+            return;
+        const reader = this.raw.body.getReader();
+        while (true) {
+            const { done, value } = await reader.read();
+            if (done)
+                break;
+            yield value;
+        }
+    }
+}

package/dist/browser/transport/undici.d.ts

@@ -0,0 +1,38 @@
+import { ReckerRequest, ReckerResponse, Transport, ProxyOptions, HTTP2Options, DNSOptions, TLSOptions } from '../types/index.js';
+import { AgentManager } from '../utils/agent-manager.js';
+interface UndiciTransportOptions {
+    connectTimeout?: number;
+    headersTimeout?: number;
+    bodyTimeout?: number;
+    maxRedirections?: number;
+    proxy?: ProxyOptions | string;
+    http2?: HTTP2Options;
+    dns?: DNSOptions;
+    agent?: AgentManager;
+    tls?: TLSOptions;
+    socketPath?: string;
+    connections?: number;
+    pipelining?: number;
+    keepAlive?: boolean;
+    keepAliveTimeout?: number;
+    keepAliveMaxTimeout?: number;
+    perDomainPooling?: boolean;
+    localAddress?: string;
+    observability?: boolean;
+}
+export declare class UndiciTransport implements Transport {
+    private static requestCounter;
+    private baseUrl;
+    private options;
+    private proxyAgent?;
+    private dnsAgent?;
+    private agentManager?;
+    private proxyBypassList?;
+    private tlsOptions?;
+    private socketClient?;
+    private observability;
+    constructor(baseUrl?: string, options?: UndiciTransportOptions);
+    dispatch(req: ReckerRequest): Promise<ReckerResponse>;
+    private dispatchFast;
+}
+export {};