recker 1.0.26 → 1.0.27

package/dist/browser/recker.js

@@ -0,0 +1,61 @@
+import { createClient } from '../core/client.js';
+import { FetchTransport } from '../transport/fetch.js';
+let _defaultClient = null;
+function getDefaultClient() {
+    if (!_defaultClient) {
+        _defaultClient = createClient({
+            transport: new FetchTransport(),
+        });
+    }
+    return _defaultClient;
+}
+export function get(url, options) {
+    return getDefaultClient().get(url, options);
+}
+export function post(url, options) {
+    return getDefaultClient().post(url, options);
+}
+export function put(url, options) {
+    return getDefaultClient().put(url, options);
+}
+export function patch(url, options) {
+    return getDefaultClient().patch(url, options);
+}
+export function del(url, options) {
+    return getDefaultClient().delete(url, options);
+}
+export function head(url, options) {
+    return getDefaultClient().head(url, options);
+}
+export function options(url, options) {
+    return getDefaultClient().options(url, options);
+}
+export function ws(url, protocols) {
+    return new WebSocket(url, protocols);
+}
+export const recker = {
+    get,
+    post,
+    put,
+    patch,
+    delete: del,
+    head,
+    options,
+    ws,
+    client: (opts) => createClient({ ...opts, transport: new FetchTransport() }),
+    reset: () => {
+        _defaultClient = null;
+    },
+    isBrowser: true,
+    unavailable: [
+        'whois',
+        'whoisAvailable',
+        'dns',
+        'dnsSecurity',
+        'dnsClient',
+        'whoisClient',
+        'ai',
+        'aiClient',
+    ],
+};
+export default recker;

package/dist/browser/runner/request-runner.d.ts

@@ -0,0 +1,46 @@
+import { EventEmitter } from 'events';
+export interface RunnerOptions {
+    concurrency?: number;
+    retries?: number;
+    retryDelay?: number;
+}
+export interface RequestTask<T = any> {
+    id: string;
+    fn: () => Promise<T>;
+    priority: number;
+    retries?: number;
+}
+export interface RunnerResult<T = any> {
+    results: (T | Error)[];
+    stats: {
+        total: number;
+        successful: number;
+        failed: number;
+        duration: number;
+    };
+}
+export declare class RequestRunner extends EventEmitter {
+    private concurrency;
+    private queue;
+    private activeCount;
+    private paused;
+    private results;
+    private stats;
+    private startTime;
+    constructor(options?: RunnerOptions);
+    add<T>(fn: () => Promise<T>, options?: {
+        priority?: number;
+        id?: string;
+    }): void;
+    run<T>(items: any[], processor: (item: any, index: number) => Promise<T>, options?: {
+        priority?: number;
+    }): Promise<RunnerResult<T>>;
+    private processNext;
+    getProgress(): {
+        total: number;
+        completed: number;
+        pending: number;
+        active: number;
+        percent: number;
+    };
+}

package/dist/browser/runner/request-runner.js

@@ -0,0 +1,89 @@
+import { EventEmitter } from 'events';
+export class RequestRunner extends EventEmitter {
+    concurrency;
+    queue = [];
+    activeCount = 0;
+    paused = false;
+    results = new Map();
+    stats = { total: 0, successful: 0, failed: 0 };
+    startTime = 0;
+    constructor(options = {}) {
+        super();
+        this.concurrency = options.concurrency || 5;
+    }
+    add(fn, options = {}) {
+        this.queue.push({
+            id: options.id || Math.random().toString(36).slice(2),
+            fn,
+            priority: options.priority || 0,
+        });
+        this.queue.sort((a, b) => b.priority - a.priority);
+        this.stats.total++;
+        this.processNext();
+    }
+    async run(items, processor, options = {}) {
+        this.startTime = Date.now();
+        this.stats = { total: items.length, successful: 0, failed: 0 };
+        this.results.clear();
+        const promises = items.map((item, index) => {
+            return new Promise((resolve) => {
+                this.add(async () => {
+                    try {
+                        const res = await processor(item, index);
+                        resolve(res);
+                        return res;
+                    }
+                    catch (err) {
+                        resolve(err);
+                        throw err;
+                    }
+                }, { priority: options.priority, id: String(index) });
+            });
+        });
+        const results = await Promise.all(promises);
+        return {
+            results,
+            stats: {
+                ...this.stats,
+                duration: Date.now() - this.startTime
+            }
+        };
+    }
+    async processNext() {
+        if (this.paused || this.activeCount >= this.concurrency || this.queue.length === 0) {
+            return;
+        }
+        const task = this.queue.shift();
+        if (!task)
+            return;
+        this.activeCount++;
+        this.emit('taskStart', task);
+        try {
+            const result = await task.fn();
+            this.stats.successful++;
+            this.emit('taskComplete', { task, result });
+        }
+        catch (error) {
+            this.stats.failed++;
+            this.emit('taskError', { task, error });
+        }
+        finally {
+            this.activeCount--;
+            this.emit('progress', this.getProgress());
+            if (this.activeCount === 0 && this.queue.length === 0) {
+                this.emit('drained');
+            }
+            this.processNext();
+        }
+    }
+    getProgress() {
+        const completed = this.stats.successful + this.stats.failed;
+        return {
+            total: this.stats.total,
+            completed,
+            pending: this.queue.length,
+            active: this.activeCount,
+            percent: this.stats.total > 0 ? (completed / this.stats.total) * 100 : 0
+        };
+    }
+}

package/dist/browser/scrape/document.d.ts

@@ -0,0 +1,44 @@
+import type { CheerioAPI } from 'cheerio';
+import { ScrapeElement } from './element.js';
+import type { ScrapeOptions, ExtractionSchema, ExtractedLink, ExtractedImage, ExtractedMeta, OpenGraphData, TwitterCardData, JsonLdData, ExtractedForm, ExtractedTable, ExtractedScript, ExtractedStyle, LinkExtractionOptions, ImageExtractionOptions } from './types.js';
+export declare class ScrapeDocument {
+    private $;
+    private options;
+    constructor($: CheerioAPI, options?: ScrapeOptions);
+    static create(html: string, options?: ScrapeOptions): Promise<ScrapeDocument>;
+    select(selector: string): ScrapeElement;
+    selectFirst(selector: string): ScrapeElement;
+    selectAll(selector: string): ScrapeElement[];
+    query(selector: string): ScrapeElement;
+    queryAll(selector: string): ScrapeElement[];
+    text(selector: string): string;
+    texts(selector: string): string[];
+    attr(selector: string, attribute: string): string | undefined;
+    attrs(selector: string, attribute: string): string[];
+    innerHtml(selector: string): string | null;
+    outerHtml(selector: string): string;
+    links(options?: LinkExtractionOptions): ExtractedLink[];
+    images(options?: ImageExtractionOptions): ExtractedImage[];
+    meta(): ExtractedMeta;
+    openGraph(): OpenGraphData;
+    twitterCard(): TwitterCardData;
+    jsonLd(): JsonLdData[];
+    forms(selector?: string): ExtractedForm[];
+    tables(selector?: string): ExtractedTable[];
+    scripts(): ExtractedScript[];
+    styles(): ExtractedStyle[];
+    extract<T extends Record<string, unknown>>(schema: ExtractionSchema): T;
+    private extractField;
+    title(): string | undefined;
+    body(): ScrapeElement;
+    head(): ScrapeElement;
+    html(): string;
+    root(): ScrapeElement;
+    exists(selector: string): boolean;
+    count(selector: string): number;
+    findByText(text: string, selector?: string): ScrapeElement[];
+    findByExactText(text: string, selector?: string): ScrapeElement[];
+    findByData(name: string, value?: string): ScrapeElement[];
+    get raw(): CheerioAPI;
+    get baseUrl(): string | undefined;
+}

package/dist/browser/scrape/document.js

@@ -0,0 +1,210 @@
+import { requireOptional } from '../utils/optional-require.js';
+import { ScrapeElement } from './element.js';
+import { extractLinks, extractImages, extractMeta, extractOpenGraph, extractTwitterCard, extractJsonLd, extractForms, extractTables, extractScripts, extractStyles, } from './extractors.js';
+let cheerioModule = null;
+async function loadCheerio() {
+    if (cheerioModule) {
+        return cheerioModule;
+    }
+    cheerioModule = await requireOptional('cheerio', 'recker/scrape');
+    return cheerioModule;
+}
+export class ScrapeDocument {
+    $;
+    options;
+    constructor($, options) {
+        this.$ = $;
+        this.options = options || {};
+    }
+    static async create(html, options) {
+        const { load } = await loadCheerio();
+        return new ScrapeDocument(load(html), options);
+    }
+    select(selector) {
+        return new ScrapeElement(this.$(selector), this.$);
+    }
+    selectFirst(selector) {
+        return new ScrapeElement(this.$(selector).first(), this.$);
+    }
+    selectAll(selector) {
+        const elements = [];
+        this.$(selector).each((_, element) => {
+            elements.push(new ScrapeElement(this.$(element), this.$));
+        });
+        return elements;
+    }
+    query(selector) {
+        return this.select(selector);
+    }
+    queryAll(selector) {
+        return this.selectAll(selector);
+    }
+    text(selector) {
+        return this.$(selector).first().text().trim();
+    }
+    texts(selector) {
+        const texts = [];
+        this.$(selector).each((_, element) => {
+            const text = this.$(element).text().trim();
+            if (text) {
+                texts.push(text);
+            }
+        });
+        return texts;
+    }
+    attr(selector, attribute) {
+        return this.$(selector).first().attr(attribute);
+    }
+    attrs(selector, attribute) {
+        const attrs = [];
+        this.$(selector).each((_, element) => {
+            const value = this.$(element).attr(attribute);
+            if (value !== undefined) {
+                attrs.push(value);
+            }
+        });
+        return attrs;
+    }
+    innerHtml(selector) {
+        return this.$(selector).first().html();
+    }
+    outerHtml(selector) {
+        const el = this.$(selector).first();
+        return this.$.html(el) || '';
+    }
+    links(options) {
+        return extractLinks(this.$, {
+            ...options,
+            baseUrl: this.options.baseUrl,
+        });
+    }
+    images(options) {
+        return extractImages(this.$, {
+            ...options,
+            baseUrl: this.options.baseUrl,
+        });
+    }
+    meta() {
+        return extractMeta(this.$);
+    }
+    openGraph() {
+        return extractOpenGraph(this.$);
+    }
+    twitterCard() {
+        return extractTwitterCard(this.$);
+    }
+    jsonLd() {
+        return extractJsonLd(this.$);
+    }
+    forms(selector) {
+        return extractForms(this.$, selector);
+    }
+    tables(selector) {
+        return extractTables(this.$, selector);
+    }
+    scripts() {
+        return extractScripts(this.$);
+    }
+    styles() {
+        return extractStyles(this.$);
+    }
+    extract(schema) {
+        const result = {};
+        for (const [key, fieldConfig] of Object.entries(schema)) {
+            result[key] = this.extractField(fieldConfig);
+        }
+        return result;
+    }
+    extractField(field) {
+        if (typeof field === 'string') {
+            return this.text(field) || undefined;
+        }
+        const { selector, attribute, multiple, transform } = field;
+        if (multiple) {
+            const values = [];
+            this.$(selector).each((_, element) => {
+                const $el = this.$(element);
+                let value;
+                if (attribute) {
+                    value = $el.attr(attribute) || '';
+                }
+                else {
+                    value = $el.text().trim();
+                }
+                if (value) {
+                    values.push(transform ? transform(value) : value);
+                }
+            });
+            return values;
+        }
+        else {
+            const $el = this.$(selector).first();
+            let value;
+            if (attribute) {
+                value = $el.attr(attribute) || '';
+            }
+            else {
+                value = $el.text().trim();
+            }
+            if (!value)
+                return undefined;
+            return transform ? transform(value) : value;
+        }
+    }
+    title() {
+        const title = this.$('title').first().text().trim();
+        return title || undefined;
+    }
+    body() {
+        return new ScrapeElement(this.$('body').first(), this.$);
+    }
+    head() {
+        return new ScrapeElement(this.$('head').first(), this.$);
+    }
+    html() {
+        return this.$.html() || '';
+    }
+    root() {
+        return new ScrapeElement(this.$.root(), this.$);
+    }
+    exists(selector) {
+        return this.$(selector).length > 0;
+    }
+    count(selector) {
+        return this.$(selector).length;
+    }
+    findByText(text, selector) {
+        const baseSelector = selector || '*';
+        const elements = [];
+        this.$(baseSelector).each((_, element) => {
+            const $el = this.$(element);
+            if ($el.text().includes(text)) {
+                elements.push(new ScrapeElement($el, this.$));
+            }
+        });
+        return elements;
+    }
+    findByExactText(text, selector) {
+        const baseSelector = selector || '*';
+        const elements = [];
+        this.$(baseSelector).each((_, element) => {
+            const $el = this.$(element);
+            if ($el.text().trim() === text) {
+                elements.push(new ScrapeElement($el, this.$));
+            }
+        });
+        return elements;
+    }
+    findByData(name, value) {
+        const selector = value !== undefined
+            ? `[data-${name}="${value}"]`
+            : `[data-${name}]`;
+        return this.selectAll(selector);
+    }
+    get raw() {
+        return this.$;
+    }
+    get baseUrl() {
+        return this.options.baseUrl;
+    }
+}

package/dist/browser/scrape/element.d.ts

@@ -0,0 +1,49 @@
+import type { Cheerio, CheerioAPI } from 'cheerio';
+import type { Element } from 'domhandler';
+export declare class ScrapeElement {
+    private $el;
+    private $;
+    constructor($el: Cheerio<Element>, $: CheerioAPI);
+    find(selector: string): ScrapeElement;
+    parent(selector?: string): ScrapeElement;
+    children(selector?: string): ScrapeElement;
+    siblings(selector?: string): ScrapeElement;
+    next(selector?: string): ScrapeElement;
+    prev(selector?: string): ScrapeElement;
+    nextAll(selector?: string): ScrapeElement;
+    prevAll(selector?: string): ScrapeElement;
+    closest(selector: string): ScrapeElement;
+    first(): ScrapeElement;
+    last(): ScrapeElement;
+    eq(index: number): ScrapeElement;
+    filter(selector: string): ScrapeElement;
+    not(selector: string): ScrapeElement;
+    has(selector: string): ScrapeElement;
+    add(selector: string): ScrapeElement;
+    parents(selector?: string): ScrapeElement;
+    contents(): ScrapeElement;
+    text(): string;
+    html(): string | null;
+    outerHtml(): string;
+    attr(name: string): string | undefined;
+    attrs(): Record<string, string>;
+    data(name?: string): unknown;
+    val(): string | string[] | undefined;
+    prop(name: string): unknown;
+    exists(): boolean;
+    get length(): number;
+    is(selector: string): boolean;
+    hasClass(className: string): boolean;
+    index(selector?: string): number;
+    each(callback: (el: ScrapeElement, index: number) => void): this;
+    map<T>(callback: (el: ScrapeElement, index: number) => T): T[];
+    toArray(): ScrapeElement[];
+    reduce<T>(callback: (acc: T, el: ScrapeElement, index: number) => T, initialValue: T): T;
+    some(callback: (el: ScrapeElement, index: number) => boolean): boolean;
+    every(callback: (el: ScrapeElement, index: number) => boolean): boolean;
+    tagName(): string | undefined;
+    clone(): ScrapeElement;
+    toString(): string;
+    get raw(): Cheerio<Element>;
+    get(index?: number): Element | undefined;
+}

package/dist/browser/scrape/element.js

@@ -0,0 +1,176 @@
+export class ScrapeElement {
+    $el;
+    $;
+    constructor($el, $) {
+        this.$el = $el;
+        this.$ = $;
+    }
+    find(selector) {
+        return new ScrapeElement(this.$el.find(selector), this.$);
+    }
+    parent(selector) {
+        const parent = selector ? this.$el.parent(selector) : this.$el.parent();
+        return new ScrapeElement(parent, this.$);
+    }
+    children(selector) {
+        const children = selector ? this.$el.children(selector) : this.$el.children();
+        return new ScrapeElement(children, this.$);
+    }
+    siblings(selector) {
+        const siblings = selector ? this.$el.siblings(selector) : this.$el.siblings();
+        return new ScrapeElement(siblings, this.$);
+    }
+    next(selector) {
+        const next = selector ? this.$el.next(selector) : this.$el.next();
+        return new ScrapeElement(next, this.$);
+    }
+    prev(selector) {
+        const prev = selector ? this.$el.prev(selector) : this.$el.prev();
+        return new ScrapeElement(prev, this.$);
+    }
+    nextAll(selector) {
+        const nextAll = selector ? this.$el.nextAll(selector) : this.$el.nextAll();
+        return new ScrapeElement(nextAll, this.$);
+    }
+    prevAll(selector) {
+        const prevAll = selector ? this.$el.prevAll(selector) : this.$el.prevAll();
+        return new ScrapeElement(prevAll, this.$);
+    }
+    closest(selector) {
+        return new ScrapeElement(this.$el.closest(selector), this.$);
+    }
+    first() {
+        return new ScrapeElement(this.$el.first(), this.$);
+    }
+    last() {
+        return new ScrapeElement(this.$el.last(), this.$);
+    }
+    eq(index) {
+        return new ScrapeElement(this.$el.eq(index), this.$);
+    }
+    filter(selector) {
+        return new ScrapeElement(this.$el.filter(selector), this.$);
+    }
+    not(selector) {
+        return new ScrapeElement(this.$el.not(selector), this.$);
+    }
+    has(selector) {
+        return new ScrapeElement(this.$el.has(selector), this.$);
+    }
+    add(selector) {
+        return new ScrapeElement(this.$el.add(selector), this.$);
+    }
+    parents(selector) {
+        const parents = selector ? this.$el.parents(selector) : this.$el.parents();
+        return new ScrapeElement(parents, this.$);
+    }
+    contents() {
+        return new ScrapeElement(this.$el.contents(), this.$);
+    }
+    text() {
+        return this.$el.text().trim();
+    }
+    html() {
+        return this.$el.html();
+    }
+    outerHtml() {
+        return this.$.html(this.$el) || '';
+    }
+    attr(name) {
+        return this.$el.attr(name);
+    }
+    attrs() {
+        const attributes = {};
+        const el = this.$el.get(0);
+        if (el && 'attribs' in el) {
+            Object.assign(attributes, el.attribs);
+        }
+        return attributes;
+    }
+    data(name) {
+        if (name) {
+            return this.$el.data(name);
+        }
+        return this.$el.data();
+    }
+    val() {
+        return this.$el.val();
+    }
+    prop(name) {
+        return this.$el.prop(name);
+    }
+    exists() {
+        return this.$el.length > 0;
+    }
+    get length() {
+        return this.$el.length;
+    }
+    is(selector) {
+        return this.$el.is(selector);
+    }
+    hasClass(className) {
+        return this.$el.hasClass(className);
+    }
+    index(selector) {
+        return selector ? this.$el.index(selector) : this.$el.index();
+    }
+    each(callback) {
+        this.$el.each((index, element) => {
+            callback(new ScrapeElement(this.$(element), this.$), index);
+        });
+        return this;
+    }
+    map(callback) {
+        const results = [];
+        this.$el.each((index, element) => {
+            results.push(callback(new ScrapeElement(this.$(element), this.$), index));
+        });
+        return results;
+    }
+    toArray() {
+        return this.$el.toArray().map((element) => new ScrapeElement(this.$(element), this.$));
+    }
+    reduce(callback, initialValue) {
+        let accumulator = initialValue;
+        this.$el.each((index, element) => {
+            accumulator = callback(accumulator, new ScrapeElement(this.$(element), this.$), index);
+        });
+        return accumulator;
+    }
+    some(callback) {
+        let found = false;
+        this.$el.each((index, element) => {
+            if (callback(new ScrapeElement(this.$(element), this.$), index)) {
+                found = true;
+                return false;
+            }
+        });
+        return found;
+    }
+    every(callback) {
+        let allMatch = true;
+        this.$el.each((index, element) => {
+            if (!callback(new ScrapeElement(this.$(element), this.$), index)) {
+                allMatch = false;
+                return false;
+            }
+        });
+        return allMatch;
+    }
+    tagName() {
+        const el = this.$el.get(0);
+        return el ? el.tagName?.toLowerCase() : undefined;
+    }
+    clone() {
+        return new ScrapeElement(this.$el.clone(), this.$);
+    }
+    toString() {
+        return this.outerHtml();
+    }
+    get raw() {
+        return this.$el;
+    }
+    get(index = 0) {
+        return this.$el.get(index);
+    }
+}

package/dist/browser/scrape/extractors.d.ts

@@ -0,0 +1,16 @@
+import type { CheerioAPI } from 'cheerio';
+import type { ExtractedLink, ExtractedImage, ExtractedMeta, OpenGraphData, TwitterCardData, JsonLdData, ExtractedForm, ExtractedTable, ExtractedScript, ExtractedStyle, LinkExtractionOptions, ImageExtractionOptions } from './types.js';
+export declare function extractLinks($: CheerioAPI, options?: LinkExtractionOptions & {
+    baseUrl?: string;
+}): ExtractedLink[];
+export declare function extractImages($: CheerioAPI, options?: ImageExtractionOptions & {
+    baseUrl?: string;
+}): ExtractedImage[];
+export declare function extractMeta($: CheerioAPI): ExtractedMeta;
+export declare function extractOpenGraph($: CheerioAPI): OpenGraphData;
+export declare function extractTwitterCard($: CheerioAPI): TwitterCardData;
+export declare function extractJsonLd($: CheerioAPI): JsonLdData[];
+export declare function extractForms($: CheerioAPI, selector?: string): ExtractedForm[];
+export declare function extractTables($: CheerioAPI, selector?: string): ExtractedTable[];
+export declare function extractScripts($: CheerioAPI): ExtractedScript[];
+export declare function extractStyles($: CheerioAPI): ExtractedStyle[];