recker 1.0.43 → 1.0.44

package/dist/scrape/extractors.js

@@ -1,10 +1,13 @@
+import { NodeType } from './parser/index.js';
 function resolveUrl(url, baseUrl) {
     if (!url)
         return '';
     if (!baseUrl)
         return url;
     try {
-        if (url.startsWith('http://') || url.startsWith('https://') || url.startsWith('//')) {
+        if (url.startsWith('http://') ||
+            url.startsWith('https://') ||
+            url.startsWith('//')) {
             return url;
         }
         return new URL(url, baseUrl).href;
@@ -32,64 +35,76 @@ function classifyLinkType(href, baseUrl) {
             return undefined;
         }
     }
-    if (href.startsWith('http://') || href.startsWith('https://') || href.startsWith('//')) {
+    if (href.startsWith('http://') ||
+        href.startsWith('https://') ||
+        href.startsWith('//')) {
         return 'external';
     }
     return 'internal';
 }
-export function extractLinks($, options) {
+export function extractLinks(root, options) {
     const selector = options?.selector || 'a[href]';
     const links = [];
-    $(selector).each((_, element) => {
-        const $el = $(element);
-        let href = $el.attr('href') || '';
+    root.querySelectorAll(selector).forEach((element) => {
+        let href = element.getAttribute('href') || '';
         if (options?.absolute && options?.baseUrl) {
             href = resolveUrl(href, options.baseUrl);
         }
+        const imgs = element.querySelectorAll('img');
+        const svgs = element.querySelectorAll('svg');
         links.push({
             href,
-            text: $el.text().trim(),
-            rel: $el.attr('rel'),
-            target: $el.attr('target'),
-            title: $el.attr('title'),
+            text: element.text.trim(),
+            rel: element.getAttribute('rel'),
+            target: element.getAttribute('target'),
+            title: element.getAttribute('title'),
             type: classifyLinkType(href, options?.baseUrl),
+            hasImage: imgs.length > 0,
+            hasImageWithAlt: imgs.filter((img) => !!img.getAttribute('alt')?.trim()).length > 0,
+            hasSvg: svgs.length > 0,
+            hasSvgWithTitle: svgs.filter((svg) => svg.querySelectorAll('title').length > 0 ||
+                !!svg.getAttribute('aria-label')).length > 0,
+            ariaLabel: element.getAttribute('aria-label'),
         });
     });
     return links;
 }
-export function extractImages($, options) {
+export function extractImages(root, options) {
     const selector = options?.selector || 'img[src]';
     const images = [];
-    $(selector).each((_, element) => {
-        const $el = $(element);
-        let src = $el.attr('src') || '';
+    root.querySelectorAll(selector).forEach((element) => {
+        let src = element.getAttribute('src') || '';
         if (options?.absolute && options?.baseUrl) {
             src = resolveUrl(src, options.baseUrl);
         }
-        const width = $el.attr('width');
-        const height = $el.attr('height');
+        const width = element.getAttribute('width');
+        const height = element.getAttribute('height');
         images.push({
             src,
-            alt: $el.attr('alt'),
-            title: $el.attr('title'),
+            alt: element.getAttribute('alt'),
+            title: element.getAttribute('title'),
             width: width ? parseInt(width, 10) : undefined,
             height: height ? parseInt(height, 10) : undefined,
-            srcset: $el.attr('srcset'),
-            loading: $el.attr('loading'),
-            decoding: $el.attr('decoding'),
+            srcset: element.getAttribute('srcset'),
+            loading: element.getAttribute('loading'),
+            decoding: element.getAttribute('decoding'),
         });
     });
     return images;
 }
-export function extractMeta($) {
+export function extractMeta(root) {
     const meta = {};
-    meta.title = $('title').first().text().trim() || undefined;
-    const charsetMeta = $('meta[charset]').attr('charset');
+    const titleEl = root.querySelector('title');
+    meta.title = titleEl ? titleEl.text.trim() || undefined : undefined;
+    const charsetMeta = root.querySelector('meta[charset]');
     if (charsetMeta) {
-        meta.charset = charsetMeta;
+        meta.charset = charsetMeta.getAttribute('charset');
     }
     else {
-        const contentType = $('meta[http-equiv="Content-Type"]').attr('content');
+        const contentTypeMeta = root.querySelector('meta[http-equiv="Content-Type"]');
+        const contentType = contentTypeMeta
+            ? contentTypeMeta.getAttribute('content')
+            : null;
         if (contentType) {
             const charsetMatch = contentType.match(/charset=([^;]+)/i);
             if (charsetMatch) {
@@ -97,14 +112,13 @@ export function extractMeta($) {
             }
         }
     }
-    const canonical = $('link[rel="canonical"]').attr('href');
-    if (canonical) {
-        meta.canonical = canonical;
+    const canonicalEl = root.querySelector('link[rel="canonical"]');
+    if (canonicalEl) {
+        meta.canonical = canonicalEl.getAttribute('href');
     }
-    $('meta[name]').each((_, element) => {
-        const $el = $(element);
-        const name = $el.attr('name')?.toLowerCase();
-        const content = $el.attr('content');
+    root.querySelectorAll('meta[name]').forEach((element) => {
+        const name = element.getAttribute('name')?.toLowerCase();
+        const content = element.getAttribute('content');
         if (!name || !content)
             return;
         switch (name) {
@@ -112,7 +126,10 @@ export function extractMeta($) {
                 meta.description = content;
                 break;
             case 'keywords':
-                meta.keywords = content.split(',').map((k) => k.trim()).filter(Boolean);
+                meta.keywords = content
+                    .split(',')
+                    .map((k) => k.trim())
+                    .filter(Boolean);
                 break;
             case 'author':
                 meta.author = content;
@@ -129,13 +146,12 @@ export function extractMeta($) {
     });
     return meta;
 }
-export function extractOpenGraph($) {
+export function extractOpenGraph(root) {
     const og = {};
     const images = [];
-    $('meta[property^="og:"]').each((_, element) => {
-        const $el = $(element);
-        const property = $el.attr('property');
-        const content = $el.attr('content');
+    root.querySelectorAll('meta[property^="og:"]').forEach((element) => {
+        const property = element.getAttribute('property');
+        const content = element.getAttribute('content');
         if (!property || !content)
             return;
         const key = property.replace('og:', '');
@@ -173,12 +189,11 @@ export function extractOpenGraph($) {
     }
     return og;
 }
-export function extractTwitterCard($) {
+export function extractTwitterCard(root) {
     const twitter = {};
-    $('meta[name^="twitter:"]').each((_, element) => {
-        const $el = $(element);
-        const name = $el.attr('name');
-        const content = $el.attr('content');
+    root.querySelectorAll('meta[name^="twitter:"]').forEach((element) => {
+        const name = element.getAttribute('name');
+        const content = element.getAttribute('content');
         if (!name || !content)
             return;
         const key = name.replace('twitter:', '');
@@ -207,10 +222,10 @@ export function extractTwitterCard($) {
     });
     return twitter;
 }
-export function extractJsonLd($) {
+export function extractJsonLd(root) {
     const results = [];
-    $('script[type="application/ld+json"]').each((_, element) => {
-        const content = $(element).html();
+    root.querySelectorAll('script[type="application/ld+json"]').forEach((element) => {
+        const content = element.innerHTML;
         if (!content)
             return;
         try {
@@ -230,125 +245,142 @@ export function extractJsonLd($) {
     });
     return results;
 }
-export function extractForms($, selector) {
+export function extractForms(root, selector) {
     const forms = [];
     const formSelector = selector || 'form';
-    $(formSelector).each((_, formElement) => {
-        const $form = $(formElement);
+    root.querySelectorAll(formSelector).forEach((formElement) => {
         const fields = [];
-        $form.find('input, select, textarea').each((_, fieldElement) => {
-            const $field = $(fieldElement);
+        formElement.querySelectorAll('input, select, textarea').forEach((fieldElement) => {
             const tagName = fieldElement.tagName?.toLowerCase();
-            const type = $field.attr('type') || (tagName === 'textarea' ? 'textarea' : tagName === 'select' ? 'select' : 'text');
+            const type = fieldElement.getAttribute('type') ||
+                (tagName === 'textarea'
+                    ? 'textarea'
+                    : tagName === 'select'
+                        ? 'select'
+                        : 'text');
+            let value = fieldElement.getAttribute('value');
+            if (tagName === 'textarea') {
+                value = fieldElement.text;
+            }
             const field = {
-                name: $field.attr('name'),
+                name: fieldElement.getAttribute('name'),
                 type,
-                value: $field.val(),
-                placeholder: $field.attr('placeholder'),
-                required: $field.attr('required') !== undefined,
+                value: value,
+                placeholder: fieldElement.getAttribute('placeholder'),
+                required: fieldElement.hasAttribute('required'),
             };
             if (tagName === 'select') {
                 field.options = [];
-                $field.find('option').each((_, optionElement) => {
-                    const $option = $(optionElement);
+                fieldElement.querySelectorAll('option').forEach((optionElement) => {
                     field.options.push({
-                        value: $option.attr('value') || $option.text(),
-                        text: $option.text().trim(),
+                        value: optionElement.getAttribute('value') || optionElement.text,
+                        text: optionElement.text.trim(),
                     });
                 });
+                const selected = fieldElement.querySelector('option[selected]');
+                if (selected) {
+                    field.value = selected.getAttribute('value') || selected.text;
+                }
             }
             fields.push(field);
         });
         forms.push({
-            action: $form.attr('action'),
-            method: $form.attr('method')?.toUpperCase(),
-            name: $form.attr('name'),
-            id: $form.attr('id'),
+            action: formElement.getAttribute('action'),
+            method: formElement.getAttribute('method')?.toUpperCase(),
+            name: formElement.getAttribute('name'),
+            id: formElement.getAttribute('id'),
             fields,
         });
     });
     return forms;
 }
-export function extractTables($, selector) {
+export function extractTables(root, selector) {
     const tables = [];
     const tableSelector = selector || 'table';
-    $(tableSelector).each((_, tableElement) => {
-        const $table = $(tableElement);
+    root.querySelectorAll(tableSelector).forEach((tableElement) => {
         const headers = [];
         const rows = [];
-        const caption = $table.find('caption').first().text().trim() || undefined;
-        const $headerCells = $table.find('thead th, thead td, tr:first-child th');
-        if ($headerCells.length > 0) {
-            $headerCells.each((_, th) => {
-                headers.push($(th).text().trim());
+        const captionEl = tableElement.querySelector('caption');
+        const caption = captionEl ? captionEl.text.trim() || undefined : undefined;
+        let headerRow = null;
+        const thead = tableElement.querySelector('thead');
+        if (thead) {
+            thead.querySelectorAll('th, td').forEach(cell => {
+                headers.push(cell.text.trim());
             });
         }
-        const $rows = $table.find('tbody tr, tr').not(':has(th):not(:first-child)');
-        $rows.each((_, tr) => {
+        if (headers.length === 0) {
+            const tr = tableElement.querySelector('tr');
+            if (tr) {
+                tr.childNodes.forEach(child => {
+                    if (child.nodeType === NodeType.ELEMENT_NODE) {
+                        const el = child;
+                        if (['TH', 'TD'].includes(el.tagName)) {
+                            headers.push(el.text.trim());
+                        }
+                    }
+                });
+                if (headers.length > 0) {
+                    headerRow = tr;
+                }
+            }
+        }
+        const allRows = tableElement.querySelectorAll('tr');
+        allRows.forEach((tr) => {
+            if (tr.parentNode && tr.parentNode.tagName === 'THEAD') {
+                return;
+            }
+            if (tr === headerRow) {
+                return;
+            }
             const row = [];
-            $(tr).find('td').each((_, td) => {
-                row.push($(td).text().trim());
+            tr.querySelectorAll('td, th').forEach(cell => {
+                row.push(cell.text.trim());
             });
             if (row.length > 0) {
                 rows.push(row);
             }
         });
-        if (headers.length === 0 && rows.length > 0) {
-            const $firstRowCells = $table.find('tr:first-child td, tr:first-child th');
-            if ($firstRowCells.length > 0) {
-                $firstRowCells.each((_, cell) => {
-                    headers.push($(cell).text().trim());
-                });
-                if ($table.find('tr:first-child th').length > 0) {
-                }
-                else {
-                    const firstRow = [];
-                    $table.find('tr:first-child td').each((_, td) => {
-                        firstRow.push($(td).text().trim());
-                    });
-                    if (firstRow.length > 0 && !rows.some(r => r.join('') === firstRow.join(''))) {
-                        rows.unshift(firstRow);
-                    }
-                }
-            }
+        if (headers.length > 0 || rows.length > 0) {
+            tables.push({
+                caption,
+                headers,
+                rows,
+            });
         }
-        tables.push({ headers, rows, caption });
     });
     return tables;
 }
-export function extractScripts($) {
+export function extractScripts(root) {
     const scripts = [];
-    $('script').each((_, element) => {
-        const $el = $(element);
-        const src = $el.attr('src');
-        const inline = !src ? $el.html()?.trim() : undefined;
+    root.querySelectorAll('script').forEach((element) => {
+        const src = element.getAttribute('src');
+        const inline = !src ? element.innerHTML?.trim() : undefined;
         if (!src && !inline)
             return;
         scripts.push({
             src,
-            type: $el.attr('type'),
-            async: $el.attr('async') !== undefined,
-            defer: $el.attr('defer') !== undefined,
+            type: element.getAttribute('type'),
+            async: element.hasAttribute('async'),
+            defer: element.hasAttribute('defer'),
             inline,
         });
     });
     return scripts;
 }
-export function extractStyles($) {
+export function extractStyles(root) {
     const styles = [];
-    $('link[rel="stylesheet"]').each((_, element) => {
-        const $el = $(element);
+    root.querySelectorAll('link[rel="stylesheet"]').forEach((element) => {
         styles.push({
-            href: $el.attr('href'),
-            media: $el.attr('media'),
+            href: element.getAttribute('href'),
+            media: element.getAttribute('media'),
         });
     });
-    $('style').each((_, element) => {
-        const $el = $(element);
-        const inline = $el.html()?.trim();
+    root.querySelectorAll('style').forEach((element) => {
+        const inline = element.innerHTML?.trim();
         if (inline) {
             styles.push({
-                media: $el.attr('media'),
+                media: element.getAttribute('media'),
                 inline,
             });
         }

package/dist/scrape/index.d.ts

@@ -1,3 +1,5 @@
+export { parse as parseHtmlSync, HTMLElement, TextNode, CommentNode, Node, NodeType, valid, } from './parser/index.js';
+export type { Options as ParserOptions } from './parser/index.js';
 export { ScrapeDocument } from './document.js';
 export { ScrapeElement } from './element.js';
 export { Spider, spider } from './spider.js';

package/dist/scrape/index.js

@@ -1,3 +1,4 @@
+export { parse as parseHtmlSync, HTMLElement, TextNode, CommentNode, Node, NodeType, valid, } from './parser/index.js';
 export { ScrapeDocument } from './document.js';
 export { ScrapeElement } from './element.js';
 export { Spider, spider } from './spider.js';

package/dist/scrape/parser/back.d.ts

@@ -0,0 +1 @@
+export default function arr_back<T>(arr: T[]): T;

package/dist/scrape/parser/back.js

@@ -0,0 +1,3 @@
+export default function arr_back(arr) {
+    return arr[arr.length - 1];
+}

package/dist/scrape/parser/index.d.ts

@@ -0,0 +1,20 @@
+import CommentNode from './nodes/comment.js';
+import HTMLElement, { Options } from './nodes/html.js';
+import Node from './nodes/node.js';
+import TextNode from './nodes/text.js';
+import NodeType from './nodes/type.js';
+import baseParse from './parse.js';
+import valid from './valid.js';
+export type { Options } from './nodes/html.js';
+export { HTMLElement, CommentNode, valid, Node, TextNode, NodeType };
+export declare function parse(data: string, options?: Partial<Options>): HTMLElement;
+export declare namespace parse {
+    var parse: typeof baseParse;
+    var HTMLElement: typeof import("./nodes/html.js").default;
+    var CommentNode: typeof import("./nodes/comment.js").default;
+    var valid: typeof import("./valid.js").default;
+    var Node: typeof import("./nodes/node.js").default;
+    var TextNode: typeof import("./nodes/text.js").default;
+    var NodeType: typeof import("./nodes/type.js").default;
+}
+export default parse;

package/dist/scrape/parser/index.js

@@ -0,0 +1,19 @@
+import CommentNode from './nodes/comment.js';
+import HTMLElement from './nodes/html.js';
+import Node from './nodes/node.js';
+import TextNode from './nodes/text.js';
+import NodeType from './nodes/type.js';
+import baseParse from './parse.js';
+import valid from './valid.js';
+export { HTMLElement, CommentNode, valid, Node, TextNode, NodeType };
+export function parse(data, options = {}) {
+    return baseParse(data, options);
+}
+export default parse;
+parse.parse = baseParse;
+parse.HTMLElement = HTMLElement;
+parse.CommentNode = CommentNode;
+parse.valid = valid;
+parse.Node = Node;
+parse.TextNode = TextNode;
+parse.NodeType = NodeType;

package/dist/scrape/parser/matcher.d.ts

@@ -0,0 +1,30 @@
+import type HTMLElement from './nodes/html.js';
+import type Node from './nodes/node.js';
+export declare type Predicate = (node: Node) => node is HTMLElement;
+declare function isTag(node: Node): node is HTMLElement;
+declare function getAttributeValue(elem: HTMLElement, name: string): string | undefined;
+declare function getName(elem: HTMLElement): string;
+declare function getChildren(node: Node): Node[];
+declare function getParent(node: Node): HTMLElement | null;
+declare function getText(node: Node): string;
+declare function removeSubsets(nodes: Node[]): Node[];
+declare function existsOne(test: Predicate, elems: Node[]): boolean;
+declare function getSiblings(node: Node): Node[];
+declare function hasAttrib(elem: HTMLElement, name: string): boolean;
+declare function findOne(test: Predicate, elems: Node[]): HTMLElement | null;
+declare function findAll(test: Predicate, nodes: Node[]): Node[];
+declare const _default: {
+    isTag: typeof isTag;
+    getAttributeValue: typeof getAttributeValue;
+    getName: typeof getName;
+    getChildren: typeof getChildren;
+    getParent: typeof getParent;
+    getText: typeof getText;
+    removeSubsets: typeof removeSubsets;
+    existsOne: typeof existsOne;
+    getSiblings: typeof getSiblings;
+    hasAttrib: typeof hasAttrib;
+    findOne: typeof findOne;
+    findAll: typeof findAll;
+};
+export default _default;

package/dist/scrape/parser/matcher.js

@@ -0,0 +1,99 @@
+import NodeType from './nodes/type.js';
+function isTag(node) {
+    return !!node && node.nodeType === NodeType.ELEMENT_NODE;
+}
+function getAttributeValue(elem, name) {
+    return isTag(elem) ? elem.getAttribute(name) : undefined;
+}
+function getName(elem) {
+    return ((elem && elem.rawTagName) || '').toLowerCase();
+}
+function getChildren(node) {
+    return node && node.childNodes;
+}
+function getParent(node) {
+    return node ? node.parentNode : null;
+}
+function getText(node) {
+    return node.text;
+}
+function removeSubsets(nodes) {
+    let idx = nodes.length;
+    let node;
+    let ancestor;
+    let replace;
+    while (--idx > -1) {
+        node = ancestor = nodes[idx];
+        nodes[idx] = null;
+        replace = true;
+        while (ancestor) {
+            if (nodes.indexOf(ancestor) > -1) {
+                replace = false;
+                nodes.splice(idx, 1);
+                break;
+            }
+            ancestor = getParent(ancestor);
+        }
+        if (replace) {
+            nodes[idx] = node;
+        }
+    }
+    return nodes;
+}
+function existsOne(test, elems) {
+    return elems.some((elem) => {
+        return isTag(elem)
+            ? test(elem) || existsOne(test, getChildren(elem))
+            : false;
+    });
+}
+function getSiblings(node) {
+    const parent = getParent(node);
+    return parent ? getChildren(parent) : [];
+}
+function hasAttrib(elem, name) {
+    return getAttributeValue(elem, name) !== undefined;
+}
+function findOne(test, elems) {
+    let elem = null;
+    for (let i = 0, l = elems?.length; i < l && !elem; i++) {
+        const el = elems[i];
+        if (test(el)) {
+            elem = el;
+        }
+        else {
+            const childs = getChildren(el);
+            if (childs && childs.length > 0) {
+                elem = findOne(test, childs);
+            }
+        }
+    }
+    return elem;
+}
+function findAll(test, nodes) {
+    let result = [];
+    for (let i = 0, j = nodes.length; i < j; i++) {
+        if (!isTag(nodes[i]))
+            continue;
+        if (test(nodes[i]))
+            result.push(nodes[i]);
+        const childs = getChildren(nodes[i]);
+        if (childs)
+            result = result.concat(findAll(test, childs));
+    }
+    return result;
+}
+export default {
+    isTag,
+    getAttributeValue,
+    getName,
+    getChildren,
+    getParent,
+    getText,
+    removeSubsets,
+    existsOne,
+    getSiblings,
+    hasAttrib,
+    findOne,
+    findAll,
+};

package/dist/scrape/parser/nodes/comment.d.ts

@@ -0,0 +1,12 @@
+import type HTMLElement from './html.js';
+import Node from './node.js';
+import NodeType from './type.js';
+export default class CommentNode extends Node {
+    rawText: string;
+    rawTagName: string;
+    clone(): CommentNode;
+    constructor(rawText: string, parentNode?: HTMLElement | null, range?: [number, number], rawTagName?: string);
+    nodeType: NodeType;
+    get text(): string;
+    toString(): string;
+}

package/dist/scrape/parser/nodes/comment.js

@@ -0,0 +1,21 @@
+import Node from './node.js';
+import NodeType from './type.js';
+export default class CommentNode extends Node {
+    rawText;
+    rawTagName;
+    clone() {
+        return new CommentNode(this.rawText, null, undefined, this.rawTagName);
+    }
+    constructor(rawText, parentNode = null, range, rawTagName = '!--') {
+        super(parentNode, range);
+        this.rawText = rawText;
+        this.rawTagName = rawTagName;
+    }
+    nodeType = NodeType.COMMENT_NODE;
+    get text() {
+        return this.rawText;
+    }
+    toString() {
+        return `<!--${this.rawText}-->`;
+    }
+}