recker 1.0.43 → 1.0.44

package/dist/scrape/element.js

@@ -1,176 +1,390 @@
+import { is } from 'css-select';
+import { NodeType, parse, } from './parser/index.js';
+import Matcher from './parser/matcher.js';
 export class ScrapeElement {
-    $el;
-    $;
-    constructor($el, $) {
-        this.$el = $el;
-        this.$ = $;
+    elements;
+    constructor(elements) {
+        if (Array.isArray(elements)) {
+            this.elements = elements.filter((el) => el.nodeType === NodeType.ELEMENT_NODE);
+        }
+        else {
+            this.elements =
+                elements.nodeType === NodeType.ELEMENT_NODE
+                    ? [elements]
+                    : [];
+        }
     }
     find(selector) {
-        return new ScrapeElement(this.$el.find(selector), this.$);
+        const found = [];
+        this.elements.forEach((el) => {
+            found.push(...el.querySelectorAll(selector));
+        });
+        return new ScrapeElement(found);
     }
     parent(selector) {
-        const parent = selector ? this.$el.parent(selector) : this.$el.parent();
-        return new ScrapeElement(parent, this.$);
+        const parents = [];
+        this.elements.forEach((el) => {
+            const p = el.parentNode;
+            if (p && p.nodeType === NodeType.ELEMENT_NODE) {
+                if (!selector || is(p, selector, { adapter: Matcher })) {
+                    if (!parents.includes(p)) {
+                        parents.push(p);
+                    }
+                }
+            }
+        });
+        return new ScrapeElement(parents);
     }
     children(selector) {
-        const children = selector ? this.$el.children(selector) : this.$el.children();
-        return new ScrapeElement(children, this.$);
+        const children = [];
+        this.elements.forEach((el) => {
+            el.childNodes.forEach((child) => {
+                if (child.nodeType === NodeType.ELEMENT_NODE) {
+                    const childEl = child;
+                    if (!selector || is(childEl, selector, { adapter: Matcher })) {
+                        children.push(childEl);
+                    }
+                }
+            });
+        });
+        return new ScrapeElement(children);
     }
     siblings(selector) {
-        const siblings = selector ? this.$el.siblings(selector) : this.$el.siblings();
-        return new ScrapeElement(siblings, this.$);
+        const siblings = [];
+        this.elements.forEach((el) => {
+            const parent = el.parentNode;
+            if (parent) {
+                parent.childNodes.forEach((child) => {
+                    if (child !== el &&
+                        child.nodeType === NodeType.ELEMENT_NODE) {
+                        const childEl = child;
+                        if (!selector || is(childEl, selector, { adapter: Matcher })) {
+                            if (!siblings.includes(childEl)) {
+                                siblings.push(childEl);
+                            }
+                        }
+                    }
+                });
+            }
+        });
+        return new ScrapeElement(siblings);
     }
     next(selector) {
-        const next = selector ? this.$el.next(selector) : this.$el.next();
-        return new ScrapeElement(next, this.$);
+        const nexts = [];
+        this.elements.forEach((el) => {
+            let curr = el.nextElementSibling;
+            while (curr) {
+                if (!selector || is(curr, selector, { adapter: Matcher })) {
+                    nexts.push(curr);
+                    break;
+                }
+                break;
+            }
+        });
+        return new ScrapeElement(nexts);
     }
     prev(selector) {
-        const prev = selector ? this.$el.prev(selector) : this.$el.prev();
-        return new ScrapeElement(prev, this.$);
+        const prevs = [];
+        this.elements.forEach((el) => {
+            let curr = el.previousElementSibling;
+            if (curr) {
+                if (!selector || is(curr, selector, { adapter: Matcher })) {
+                    prevs.push(curr);
+                }
+            }
+        });
+        return new ScrapeElement(prevs);
     }
     nextAll(selector) {
-        const nextAll = selector ? this.$el.nextAll(selector) : this.$el.nextAll();
-        return new ScrapeElement(nextAll, this.$);
+        const nexts = [];
+        this.elements.forEach((el) => {
+            let curr = el.nextElementSibling;
+            while (curr) {
+                if (!selector || is(curr, selector, { adapter: Matcher })) {
+                    nexts.push(curr);
+                }
+                curr = curr.nextElementSibling;
+            }
+        });
+        return new ScrapeElement(nexts);
     }
     prevAll(selector) {
-        const prevAll = selector ? this.$el.prevAll(selector) : this.$el.prevAll();
-        return new ScrapeElement(prevAll, this.$);
+        const prevs = [];
+        this.elements.forEach((el) => {
+            let curr = el.previousElementSibling;
+            while (curr) {
+                if (!selector || is(curr, selector, { adapter: Matcher })) {
+                    prevs.push(curr);
+                }
+                curr = curr.previousElementSibling;
+            }
+        });
+        return new ScrapeElement(prevs);
     }
     closest(selector) {
-        return new ScrapeElement(this.$el.closest(selector), this.$);
+        const closests = [];
+        this.elements.forEach((el) => {
+            const found = el.closest(selector);
+            if (found) {
+                if (!closests.includes(found)) {
+                    closests.push(found);
+                }
+            }
+        });
+        return new ScrapeElement(closests);
     }
     first() {
-        return new ScrapeElement(this.$el.first(), this.$);
+        return new ScrapeElement(this.elements.slice(0, 1));
     }
     last() {
-        return new ScrapeElement(this.$el.last(), this.$);
+        return new ScrapeElement(this.elements.slice(-1));
     }
     eq(index) {
-        return new ScrapeElement(this.$el.eq(index), this.$);
-    }
-    filter(selector) {
-        return new ScrapeElement(this.$el.filter(selector), this.$);
+        if (index < 0)
+            index = this.elements.length + index;
+        return new ScrapeElement(this.elements.slice(index, index + 1));
+    }
+    filter(selectorOrCallback) {
+        if (typeof selectorOrCallback === 'string') {
+            const filtered = this.elements.filter((el) => is(el, selectorOrCallback, { adapter: Matcher }));
+            return new ScrapeElement(filtered);
+        }
+        else {
+            const filtered = this.elements.filter((el, index) => selectorOrCallback(new ScrapeElement(el), index));
+            return new ScrapeElement(filtered);
+        }
     }
     not(selector) {
-        return new ScrapeElement(this.$el.not(selector), this.$);
+        const filtered = this.elements.filter((el) => !is(el, selector, { adapter: Matcher }));
+        return new ScrapeElement(filtered);
     }
     has(selector) {
-        return new ScrapeElement(this.$el.has(selector), this.$);
+        const has = this.elements.filter((el) => {
+            return el.querySelector(selector) !== null;
+        });
+        return new ScrapeElement(has);
     }
     add(selector) {
-        return new ScrapeElement(this.$el.add(selector), this.$);
+        if (selector.trim().startsWith('<')) {
+            const parsed = parse(selector);
+            return new ScrapeElement([...this.elements, ...parsed.children]);
+        }
+        return this;
     }
     parents(selector) {
-        const parents = selector ? this.$el.parents(selector) : this.$el.parents();
-        return new ScrapeElement(parents, this.$);
+        const parents = [];
+        this.elements.forEach((el) => {
+            let p = el.parentNode;
+            while (p && p.nodeType === NodeType.ELEMENT_NODE) {
+                const pEl = p;
+                if (!selector || is(pEl, selector, { adapter: Matcher })) {
+                    if (!parents.includes(pEl)) {
+                        parents.push(pEl);
+                    }
+                }
+                p = p.parentNode;
+            }
+        });
+        return new ScrapeElement(parents);
+    }
+    parentsUntil(selector) {
+        const parents = [];
+        this.elements.forEach((el) => {
+            let p = el.parentNode;
+            while (p && p.nodeType === NodeType.ELEMENT_NODE) {
+                const pEl = p;
+                if (is(pEl, selector, { adapter: Matcher })) {
+                    break;
+                }
+                if (!parents.includes(pEl)) {
+                    parents.push(pEl);
+                }
+                p = p.parentNode;
+            }
+        });
+        return new ScrapeElement(parents);
+    }
+    nextUntil(selector) {
+        const nexts = [];
+        this.elements.forEach((el) => {
+            let curr = el.nextElementSibling;
+            while (curr) {
+                if (is(curr, selector, { adapter: Matcher })) {
+                    break;
+                }
+                nexts.push(curr);
+                curr = curr.nextElementSibling;
+            }
+        });
+        return new ScrapeElement(nexts);
+    }
+    prevUntil(selector) {
+        const prevs = [];
+        this.elements.forEach((el) => {
+            let curr = el.previousElementSibling;
+            while (curr) {
+                if (is(curr, selector, { adapter: Matcher })) {
+                    break;
+                }
+                prevs.push(curr);
+                curr = curr.previousElementSibling;
+            }
+        });
+        return new ScrapeElement(prevs);
+    }
+    slice(start, end) {
+        return new ScrapeElement(this.elements.slice(start, end));
     }
     contents() {
-        return new ScrapeElement(this.$el.contents(), this.$);
+        const contents = [];
+        this.elements.forEach(el => {
+            el.childNodes.forEach(child => {
+                if (child.nodeType === NodeType.ELEMENT_NODE) {
+                    contents.push(child);
+                }
+            });
+        });
+        return new ScrapeElement(contents);
     }
     text() {
-        return this.$el.text().trim();
+        return this.elements.map((el) => el.text).join('');
     }
     html() {
-        return this.$el.html();
+        if (this.elements.length > 0) {
+            return this.elements[0].innerHTML;
+        }
+        return null;
     }
     outerHtml() {
-        return this.$.html(this.$el) || '';
+        if (this.elements.length > 0) {
+            return this.elements[0].outerHTML;
+        }
+        return '';
     }
     attr(name) {
-        return this.$el.attr(name);
+        if (this.elements.length > 0) {
+            return this.elements[0].getAttribute(name);
+        }
+        return undefined;
     }
     attrs() {
-        const attributes = {};
-        const el = this.$el.get(0);
-        if (el && 'attribs' in el) {
-            Object.assign(attributes, el.attribs);
+        if (this.elements.length > 0) {
+            return this.elements[0].attributes;
         }
-        return attributes;
+        return {};
     }
     data(name) {
+        if (this.elements.length === 0)
+            return undefined;
+        const el = this.elements[0];
+        const attrs = el.attributes;
         if (name) {
-            return this.$el.data(name);
+            return attrs[`data-${name}`];
+        }
+        const dataAttrs = {};
+        for (const key in attrs) {
+            if (key.startsWith('data-')) {
+                const shortKey = key.slice(5);
+                dataAttrs[shortKey] = attrs[key];
+            }
         }
-        return this.$el.data();
+        return dataAttrs;
     }
     val() {
-        return this.$el.val();
+        if (this.elements.length === 0)
+            return undefined;
+        const el = this.elements[0];
+        if (el.tagName === 'INPUT') {
+            return el.getAttribute('value');
+        }
+        if (el.tagName === 'TEXTAREA') {
+            return el.text;
+        }
+        if (el.tagName === 'SELECT') {
+            const option = el.querySelector('option[selected]') || el.querySelector('option');
+            return option ? option.getAttribute('value') ?? option.text : undefined;
+        }
+        return el.getAttribute('value');
     }
     prop(name) {
-        return this.$el.prop(name);
+        if (this.elements.length === 0)
+            return undefined;
+        const el = this.elements[0];
+        if (name === 'tagName')
+            return el.tagName;
+        if (name in el)
+            return el[name];
+        return this.attr(name);
     }
     exists() {
-        return this.$el.length > 0;
+        return this.elements.length > 0;
     }
     get length() {
-        return this.$el.length;
+        return this.elements.length;
     }
     is(selector) {
-        return this.$el.is(selector);
+        return this.elements.some((el) => is(el, selector, { adapter: Matcher }));
     }
     hasClass(className) {
-        return this.$el.hasClass(className);
+        return this.elements.some((el) => el.classList.contains(className));
     }
     index(selector) {
-        return selector ? this.$el.index(selector) : this.$el.index();
+        if (this.elements.length === 0)
+            return -1;
+        const el = this.elements[0];
+        const parent = el.parentNode;
+        if (!parent)
+            return 0;
+        const siblings = parent.childNodes.filter(n => n.nodeType === NodeType.ELEMENT_NODE);
+        if (selector) {
+            return siblings.findIndex(s => s === el);
+        }
+        return siblings.indexOf(el);
     }
     each(callback) {
-        this.$el.each((index, element) => {
-            callback(new ScrapeElement(this.$(element), this.$), index);
+        this.elements.forEach((el, index) => {
+            callback(new ScrapeElement(el), index);
         });
         return this;
     }
     map(callback) {
-        const results = [];
-        this.$el.each((index, element) => {
-            results.push(callback(new ScrapeElement(this.$(element), this.$), index));
+        return this.elements.map((el, index) => {
+            return callback(new ScrapeElement(el), index);
         });
-        return results;
     }
     toArray() {
-        return this.$el.toArray().map((element) => new ScrapeElement(this.$(element), this.$));
+        return this.elements.map((el) => new ScrapeElement(el));
     }
     reduce(callback, initialValue) {
         let accumulator = initialValue;
-        this.$el.each((index, element) => {
-            accumulator = callback(accumulator, new ScrapeElement(this.$(element), this.$), index);
+        this.elements.forEach((el, index) => {
+            accumulator = callback(accumulator, new ScrapeElement(el), index);
         });
         return accumulator;
     }
     some(callback) {
-        let found = false;
-        this.$el.each((index, element) => {
-            if (callback(new ScrapeElement(this.$(element), this.$), index)) {
-                found = true;
-                return false;
-            }
-        });
-        return found;
+        return this.elements.some((el, index) => callback(new ScrapeElement(el), index));
     }
     every(callback) {
-        let allMatch = true;
-        this.$el.each((index, element) => {
-            if (!callback(new ScrapeElement(this.$(element), this.$), index)) {
-                allMatch = false;
-                return false;
-            }
-        });
-        return allMatch;
+        return this.elements.every((el, index) => callback(new ScrapeElement(el), index));
     }
     tagName() {
-        const el = this.$el.get(0);
-        return el ? el.tagName?.toLowerCase() : undefined;
+        if (this.elements.length > 0) {
+            return this.elements[0].tagName?.toLowerCase();
+        }
+        return undefined;
     }
     clone() {
-        return new ScrapeElement(this.$el.clone(), this.$);
+        const clones = this.elements.map(el => el.clone());
+        return new ScrapeElement(clones);
     }
     toString() {
         return this.outerHtml();
     }
     get raw() {
-        return this.$el;
+        return this.elements;
     }
     get(index = 0) {
-        return this.$el.get(index);
+        return this.elements[index];
     }
 }

package/dist/scrape/extractors.d.ts

@@ -1,16 +1,16 @@
-import type { CheerioAPI } from 'cheerio';
+import { HTMLElement } from './parser/index.js';
 import type { ExtractedLink, ExtractedImage, ExtractedMeta, OpenGraphData, TwitterCardData, JsonLdData, ExtractedForm, ExtractedTable, ExtractedScript, ExtractedStyle, LinkExtractionOptions, ImageExtractionOptions } from './types.js';
-export declare function extractLinks($: CheerioAPI, options?: LinkExtractionOptions & {
+export declare function extractLinks(root: HTMLElement, options?: LinkExtractionOptions & {
     baseUrl?: string;
 }): ExtractedLink[];
-export declare function extractImages($: CheerioAPI, options?: ImageExtractionOptions & {
+export declare function extractImages(root: HTMLElement, options?: ImageExtractionOptions & {
     baseUrl?: string;
 }): ExtractedImage[];
-export declare function extractMeta($: CheerioAPI): ExtractedMeta;
-export declare function extractOpenGraph($: CheerioAPI): OpenGraphData;
-export declare function extractTwitterCard($: CheerioAPI): TwitterCardData;
-export declare function extractJsonLd($: CheerioAPI): JsonLdData[];
-export declare function extractForms($: CheerioAPI, selector?: string): ExtractedForm[];
-export declare function extractTables($: CheerioAPI, selector?: string): ExtractedTable[];
-export declare function extractScripts($: CheerioAPI): ExtractedScript[];
-export declare function extractStyles($: CheerioAPI): ExtractedStyle[];
+export declare function extractMeta(root: HTMLElement): ExtractedMeta;
+export declare function extractOpenGraph(root: HTMLElement): OpenGraphData;
+export declare function extractTwitterCard(root: HTMLElement): TwitterCardData;
+export declare function extractJsonLd(root: HTMLElement): JsonLdData[];
+export declare function extractForms(root: HTMLElement, selector?: string): ExtractedForm[];
+export declare function extractTables(root: HTMLElement, selector?: string): ExtractedTable[];
+export declare function extractScripts(root: HTMLElement): ExtractedScript[];
+export declare function extractStyles(root: HTMLElement): ExtractedStyle[];