recker 1.0.2-0 → 1.0.3

package/dist/scrape/document.js

@@ -0,0 +1,198 @@
+import { load } from 'cheerio';
+import { ScrapeElement } from './element.js';
+import { extractLinks, extractImages, extractMeta, extractOpenGraph, extractTwitterCard, extractJsonLd, extractForms, extractTables, extractScripts, extractStyles, } from './extractors.js';
+export class ScrapeDocument {
+    $;
+    options;
+    constructor(html, options) {
+        this.$ = load(html);
+        this.options = options || {};
+    }
+    select(selector) {
+        return new ScrapeElement(this.$(selector), this.$);
+    }
+    selectFirst(selector) {
+        return new ScrapeElement(this.$(selector).first(), this.$);
+    }
+    selectAll(selector) {
+        const elements = [];
+        this.$(selector).each((_, element) => {
+            elements.push(new ScrapeElement(this.$(element), this.$));
+        });
+        return elements;
+    }
+    query(selector) {
+        return this.select(selector);
+    }
+    queryAll(selector) {
+        return this.selectAll(selector);
+    }
+    text(selector) {
+        return this.$(selector).first().text().trim();
+    }
+    texts(selector) {
+        const texts = [];
+        this.$(selector).each((_, element) => {
+            const text = this.$(element).text().trim();
+            if (text) {
+                texts.push(text);
+            }
+        });
+        return texts;
+    }
+    attr(selector, attribute) {
+        return this.$(selector).first().attr(attribute);
+    }
+    attrs(selector, attribute) {
+        const attrs = [];
+        this.$(selector).each((_, element) => {
+            const value = this.$(element).attr(attribute);
+            if (value !== undefined) {
+                attrs.push(value);
+            }
+        });
+        return attrs;
+    }
+    innerHtml(selector) {
+        return this.$(selector).first().html();
+    }
+    outerHtml(selector) {
+        const el = this.$(selector).first();
+        return this.$.html(el) || '';
+    }
+    links(options) {
+        return extractLinks(this.$, {
+            ...options,
+            baseUrl: this.options.baseUrl,
+        });
+    }
+    images(options) {
+        return extractImages(this.$, {
+            ...options,
+            baseUrl: this.options.baseUrl,
+        });
+    }
+    meta() {
+        return extractMeta(this.$);
+    }
+    openGraph() {
+        return extractOpenGraph(this.$);
+    }
+    twitterCard() {
+        return extractTwitterCard(this.$);
+    }
+    jsonLd() {
+        return extractJsonLd(this.$);
+    }
+    forms(selector) {
+        return extractForms(this.$, selector);
+    }
+    tables(selector) {
+        return extractTables(this.$, selector);
+    }
+    scripts() {
+        return extractScripts(this.$);
+    }
+    styles() {
+        return extractStyles(this.$);
+    }
+    extract(schema) {
+        const result = {};
+        for (const [key, fieldConfig] of Object.entries(schema)) {
+            result[key] = this.extractField(fieldConfig);
+        }
+        return result;
+    }
+    extractField(field) {
+        if (typeof field === 'string') {
+            return this.text(field) || undefined;
+        }
+        const { selector, attribute, multiple, transform } = field;
+        if (multiple) {
+            const values = [];
+            this.$(selector).each((_, element) => {
+                const $el = this.$(element);
+                let value;
+                if (attribute) {
+                    value = $el.attr(attribute) || '';
+                }
+                else {
+                    value = $el.text().trim();
+                }
+                if (value) {
+                    values.push(transform ? transform(value) : value);
+                }
+            });
+            return values;
+        }
+        else {
+            const $el = this.$(selector).first();
+            let value;
+            if (attribute) {
+                value = $el.attr(attribute) || '';
+            }
+            else {
+                value = $el.text().trim();
+            }
+            if (!value)
+                return undefined;
+            return transform ? transform(value) : value;
+        }
+    }
+    title() {
+        const title = this.$('title').first().text().trim();
+        return title || undefined;
+    }
+    body() {
+        return new ScrapeElement(this.$('body').first(), this.$);
+    }
+    head() {
+        return new ScrapeElement(this.$('head').first(), this.$);
+    }
+    html() {
+        return this.$.html() || '';
+    }
+    root() {
+        return new ScrapeElement(this.$.root(), this.$);
+    }
+    exists(selector) {
+        return this.$(selector).length > 0;
+    }
+    count(selector) {
+        return this.$(selector).length;
+    }
+    findByText(text, selector) {
+        const baseSelector = selector || '*';
+        const elements = [];
+        this.$(baseSelector).each((_, element) => {
+            const $el = this.$(element);
+            if ($el.text().includes(text)) {
+                elements.push(new ScrapeElement($el, this.$));
+            }
+        });
+        return elements;
+    }
+    findByExactText(text, selector) {
+        const baseSelector = selector || '*';
+        const elements = [];
+        this.$(baseSelector).each((_, element) => {
+            const $el = this.$(element);
+            if ($el.text().trim() === text) {
+                elements.push(new ScrapeElement($el, this.$));
+            }
+        });
+        return elements;
+    }
+    findByData(name, value) {
+        const selector = value !== undefined
+            ? `[data-${name}="${value}"]`
+            : `[data-${name}]`;
+        return this.selectAll(selector);
+    }
+    get raw() {
+        return this.$;
+    }
+    get baseUrl() {
+        return this.options.baseUrl;
+    }
+}

package/dist/scrape/element.d.ts

@@ -0,0 +1,50 @@
+import type { Cheerio, CheerioAPI } from 'cheerio';
+import type { Element } from 'domhandler';
+export declare class ScrapeElement {
+    private $el;
+    private $;
+    constructor($el: Cheerio<Element>, $: CheerioAPI);
+    find(selector: string): ScrapeElement;
+    parent(selector?: string): ScrapeElement;
+    children(selector?: string): ScrapeElement;
+    siblings(selector?: string): ScrapeElement;
+    next(selector?: string): ScrapeElement;
+    prev(selector?: string): ScrapeElement;
+    nextAll(selector?: string): ScrapeElement;
+    prevAll(selector?: string): ScrapeElement;
+    closest(selector: string): ScrapeElement;
+    first(): ScrapeElement;
+    last(): ScrapeElement;
+    eq(index: number): ScrapeElement;
+    filter(selector: string): ScrapeElement;
+    not(selector: string): ScrapeElement;
+    has(selector: string): ScrapeElement;
+    add(selector: string): ScrapeElement;
+    parents(selector?: string): ScrapeElement;
+    contents(): ScrapeElement;
+    text(): string;
+    html(): string | null;
+    outerHtml(): string;
+    attr(name: string): string | undefined;
+    attrs(): Record<string, string>;
+    data(name?: string): unknown;
+    val(): string | string[] | undefined;
+    prop(name: string): unknown;
+    exists(): boolean;
+    get length(): number;
+    is(selector: string): boolean;
+    hasClass(className: string): boolean;
+    index(selector?: string): number;
+    each(callback: (el: ScrapeElement, index: number) => void): this;
+    map<T>(callback: (el: ScrapeElement, index: number) => T): T[];
+    toArray(): ScrapeElement[];
+    reduce<T>(callback: (acc: T, el: ScrapeElement, index: number) => T, initialValue: T): T;
+    some(callback: (el: ScrapeElement, index: number) => boolean): boolean;
+    every(callback: (el: ScrapeElement, index: number) => boolean): boolean;
+    tagName(): string | undefined;
+    clone(): ScrapeElement;
+    toString(): string;
+    get raw(): Cheerio<Element>;
+    get(index?: number): Element | undefined;
+}
+//# sourceMappingURL=element.d.ts.map

package/dist/scrape/element.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"element.d.ts","sourceRoot":"","sources":["../../src/scrape/element.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EAAE,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AACnD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAE1C,qBAAa,aAAa;IACxB,OAAO,CAAC,GAAG,CAAmB;IAC9B,OAAO,CAAC,CAAC,CAAa;gBAEV,GAAG,EAAE,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,EAAE,UAAU;IAUhD,IAAI,CAAC,QAAQ,EAAE,MAAM,GAAG,aAAa;IAOrC,MAAM,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,aAAa;IAQxC,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,aAAa;IAQ1C,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,aAAa;IAQ1C,IAAI,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,aAAa;IAQtC,IAAI,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,aAAa;IAQtC,OAAO,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,aAAa;IAQzC,OAAO,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,aAAa;IAQzC,OAAO,CAAC,QAAQ,EAAE,MAAM,GAAG,aAAa;IAOxC,KAAK,IAAI,aAAa;IAOtB,IAAI,IAAI,aAAa;IAOrB,EAAE,CAAC,KAAK,EAAE,MAAM,GAAG,aAAa;IAOhC,MAAM,CAAC,QAAQ,EAAE,MAAM,GAAG,aAAa;IAOvC,GAAG,CAAC,QAAQ,EAAE,MAAM,GAAG,aAAa;IAOpC,GAAG,CAAC,QAAQ,EAAE,MAAM,GAAG,aAAa;IAOpC,GAAG,CAAC,QAAQ,EAAE,MAAM,GAAG,aAAa;IAOpC,OAAO,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,aAAa;IAQzC,QAAQ,IAAI,aAAa;IASzB,IAAI,IAAI,MAAM;IAOd,IAAI,IAAI,MAAM,GAAG,IAAI;IAOrB,SAAS,IAAI,MAAM;IAOnB,IAAI,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS;IAOtC,KAAK,IAAI,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC;IAY/B,IAAI,CAAC,IAAI,CAAC,EAAE,MAAM,GAAG,OAAO;IAU5B,GAAG,IAAI,MAAM,GAAG,MAAM,EAAE,GAAG,SAAS;IAOpC,IAAI,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO;IAS3B,MAAM,IAAI,OAAO;IAOjB,IAAI,MAAM,IAAI,MAAM,CAEnB;IAKD,EAAE,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO;IAO7B,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO;IAOpC,KAAK,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,MAAM;IAShC,IAAI,CAAC,QAAQ,EAAE,CAAC,EAAE,EAAE,aAAa,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,GAAG,IAAI;IAUhE,GAAG,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,EAAE,aAAa,EAAE,KAAK,EAAE,MAAM,KAAK,CAAC,GAAG,CAAC,EAAE;IAW9D,OAAO,IAAI,aAAa,EAAE;IAS1B,MAAM,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,EAAE,aAAa,EAAE,KAAK,EAAE,MAAM,KAAK,CAAC,EAAE,YAAY,EAAE,CAAC,GAAG,CAAC;IAexF,IAAI,CAAC,QAAQ,EAAE,CAAC,EAAE,EAAE,aAAa,EAAE,KAAK,EAAE,MAAM,KAAK,OAAO,GAAG,OAAO;IActE,KAAK,CAAC,QAAQ,EAAE,CAAC,EAAE,EAAE,aAAa,EAAE,KAAK,EAAE,MAAM,KAAK,OAAO,GAAG,OAAO;IAgBvE,OAAO,IAAI,MAAM,GAAG,SAAS;IAQ7B,KAAK,IAAI,aAAa;IAOtB,QAAQ,IAAI,MAAM;IASlB,IAAI,GAAG,IAAI,OAAO,CAAC,OAAO,CAAC,CAE1B;IAKD,GAAG,CAAC,KAAK,GAAE,MAAU,GAAG,OAAO,GAAG,SAAS;CAG5C"}

package/dist/scrape/element.js

@@ -0,0 +1,176 @@
+export class ScrapeElement {
+    $el;
+    $;
+    constructor($el, $) {
+        this.$el = $el;
+        this.$ = $;
+    }
+    find(selector) {
+        return new ScrapeElement(this.$el.find(selector), this.$);
+    }
+    parent(selector) {
+        const parent = selector ? this.$el.parent(selector) : this.$el.parent();
+        return new ScrapeElement(parent, this.$);
+    }
+    children(selector) {
+        const children = selector ? this.$el.children(selector) : this.$el.children();
+        return new ScrapeElement(children, this.$);
+    }
+    siblings(selector) {
+        const siblings = selector ? this.$el.siblings(selector) : this.$el.siblings();
+        return new ScrapeElement(siblings, this.$);
+    }
+    next(selector) {
+        const next = selector ? this.$el.next(selector) : this.$el.next();
+        return new ScrapeElement(next, this.$);
+    }
+    prev(selector) {
+        const prev = selector ? this.$el.prev(selector) : this.$el.prev();
+        return new ScrapeElement(prev, this.$);
+    }
+    nextAll(selector) {
+        const nextAll = selector ? this.$el.nextAll(selector) : this.$el.nextAll();
+        return new ScrapeElement(nextAll, this.$);
+    }
+    prevAll(selector) {
+        const prevAll = selector ? this.$el.prevAll(selector) : this.$el.prevAll();
+        return new ScrapeElement(prevAll, this.$);
+    }
+    closest(selector) {
+        return new ScrapeElement(this.$el.closest(selector), this.$);
+    }
+    first() {
+        return new ScrapeElement(this.$el.first(), this.$);
+    }
+    last() {
+        return new ScrapeElement(this.$el.last(), this.$);
+    }
+    eq(index) {
+        return new ScrapeElement(this.$el.eq(index), this.$);
+    }
+    filter(selector) {
+        return new ScrapeElement(this.$el.filter(selector), this.$);
+    }
+    not(selector) {
+        return new ScrapeElement(this.$el.not(selector), this.$);
+    }
+    has(selector) {
+        return new ScrapeElement(this.$el.has(selector), this.$);
+    }
+    add(selector) {
+        return new ScrapeElement(this.$el.add(selector), this.$);
+    }
+    parents(selector) {
+        const parents = selector ? this.$el.parents(selector) : this.$el.parents();
+        return new ScrapeElement(parents, this.$);
+    }
+    contents() {
+        return new ScrapeElement(this.$el.contents(), this.$);
+    }
+    text() {
+        return this.$el.text().trim();
+    }
+    html() {
+        return this.$el.html();
+    }
+    outerHtml() {
+        return this.$.html(this.$el) || '';
+    }
+    attr(name) {
+        return this.$el.attr(name);
+    }
+    attrs() {
+        const attributes = {};
+        const el = this.$el.get(0);
+        if (el && 'attribs' in el) {
+            Object.assign(attributes, el.attribs);
+        }
+        return attributes;
+    }
+    data(name) {
+        if (name) {
+            return this.$el.data(name);
+        }
+        return this.$el.data();
+    }
+    val() {
+        return this.$el.val();
+    }
+    prop(name) {
+        return this.$el.prop(name);
+    }
+    exists() {
+        return this.$el.length > 0;
+    }
+    get length() {
+        return this.$el.length;
+    }
+    is(selector) {
+        return this.$el.is(selector);
+    }
+    hasClass(className) {
+        return this.$el.hasClass(className);
+    }
+    index(selector) {
+        return selector ? this.$el.index(selector) : this.$el.index();
+    }
+    each(callback) {
+        this.$el.each((index, element) => {
+            callback(new ScrapeElement(this.$(element), this.$), index);
+        });
+        return this;
+    }
+    map(callback) {
+        const results = [];
+        this.$el.each((index, element) => {
+            results.push(callback(new ScrapeElement(this.$(element), this.$), index));
+        });
+        return results;
+    }
+    toArray() {
+        return this.$el.toArray().map((element) => new ScrapeElement(this.$(element), this.$));
+    }
+    reduce(callback, initialValue) {
+        let accumulator = initialValue;
+        this.$el.each((index, element) => {
+            accumulator = callback(accumulator, new ScrapeElement(this.$(element), this.$), index);
+        });
+        return accumulator;
+    }
+    some(callback) {
+        let found = false;
+        this.$el.each((index, element) => {
+            if (callback(new ScrapeElement(this.$(element), this.$), index)) {
+                found = true;
+                return false;
+            }
+        });
+        return found;
+    }
+    every(callback) {
+        let allMatch = true;
+        this.$el.each((index, element) => {
+            if (!callback(new ScrapeElement(this.$(element), this.$), index)) {
+                allMatch = false;
+                return false;
+            }
+        });
+        return allMatch;
+    }
+    tagName() {
+        const el = this.$el.get(0);
+        return el ? el.tagName?.toLowerCase() : undefined;
+    }
+    clone() {
+        return new ScrapeElement(this.$el.clone(), this.$);
+    }
+    toString() {
+        return this.outerHtml();
+    }
+    get raw() {
+        return this.$el;
+    }
+    get(index = 0) {
+        return this.$el.get(index);
+    }
+}

package/dist/scrape/extractors.d.ts

@@ -0,0 +1,17 @@
+import type { CheerioAPI } from 'cheerio';
+import type { ExtractedLink, ExtractedImage, ExtractedMeta, OpenGraphData, TwitterCardData, JsonLdData, ExtractedForm, ExtractedTable, ExtractedScript, ExtractedStyle, LinkExtractionOptions, ImageExtractionOptions } from './types.js';
+export declare function extractLinks($: CheerioAPI, options?: LinkExtractionOptions & {
+    baseUrl?: string;
+}): ExtractedLink[];
+export declare function extractImages($: CheerioAPI, options?: ImageExtractionOptions & {
+    baseUrl?: string;
+}): ExtractedImage[];
+export declare function extractMeta($: CheerioAPI): ExtractedMeta;
+export declare function extractOpenGraph($: CheerioAPI): OpenGraphData;
+export declare function extractTwitterCard($: CheerioAPI): TwitterCardData;
+export declare function extractJsonLd($: CheerioAPI): JsonLdData[];
+export declare function extractForms($: CheerioAPI, selector?: string): ExtractedForm[];
+export declare function extractTables($: CheerioAPI, selector?: string): ExtractedTable[];
+export declare function extractScripts($: CheerioAPI): ExtractedScript[];
+export declare function extractStyles($: CheerioAPI): ExtractedStyle[];
+//# sourceMappingURL=extractors.d.ts.map

package/dist/scrape/extractors.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"extractors.d.ts","sourceRoot":"","sources":["../../src/scrape/extractors.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AAC1C,OAAO,KAAK,EACV,aAAa,EACb,cAAc,EACd,aAAa,EACb,aAAa,EACb,eAAe,EACf,UAAU,EACV,aAAa,EAEb,cAAc,EACd,eAAe,EACf,cAAc,EACd,qBAAqB,EACrB,sBAAsB,EACvB,MAAM,YAAY,CAAC;AAoDpB,wBAAgB,YAAY,CAC1B,CAAC,EAAE,UAAU,EACb,OAAO,CAAC,EAAE,qBAAqB,GAAG;IAAE,OAAO,CAAC,EAAE,MAAM,CAAA;CAAE,GACrD,aAAa,EAAE,CAwBjB;AAKD,wBAAgB,aAAa,CAC3B,CAAC,EAAE,UAAU,EACb,OAAO,CAAC,EAAE,sBAAsB,GAAG;IAAE,OAAO,CAAC,EAAE,MAAM,CAAA;CAAE,GACtD,cAAc,EAAE,CA4BlB;AAKD,wBAAgB,WAAW,CAAC,CAAC,EAAE,UAAU,GAAG,aAAa,CA0DxD;AAKD,wBAAgB,gBAAgB,CAAC,CAAC,EAAE,UAAU,GAAG,aAAa,CAiD7D;AAKD,wBAAgB,kBAAkB,CAAC,CAAC,EAAE,UAAU,GAAG,eAAe,CAqCjE;AAKD,wBAAgB,aAAa,CAAC,CAAC,EAAE,UAAU,GAAG,UAAU,EAAE,CAwBzD;AAKD,wBAAgB,YAAY,CAAC,CAAC,EAAE,UAAU,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,aAAa,EAAE,CA+C9E;AAKD,wBAAgB,aAAa,CAAC,CAAC,EAAE,UAAU,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,cAAc,EAAE,CA2DhF;AAKD,wBAAgB,cAAc,CAAC,CAAC,EAAE,UAAU,GAAG,eAAe,EAAE,CAqB/D;AAKD,wBAAgB,aAAa,CAAC,CAAC,EAAE,UAAU,GAAG,cAAc,EAAE,CAyB7D"}