recker 1.0.43 → 1.0.44

package/dist/presets/sentry.js

@@ -0,0 +1,48 @@
+export function sentry(options) {
+    const headers = {
+        'Authorization': `Bearer ${options.authToken}`,
+        'Content-Type': 'application/json',
+    };
+    return {
+        baseUrl: 'https://sentry.io/api/0',
+        headers,
+        timeout: 30 * 1000,
+        retry: {
+            maxAttempts: 3,
+            backoff: 'exponential',
+            delay: 1000,
+            statusCodes: [408, 429, 500, 502, 503, 504],
+        }
+    };
+}
+function parseDSN(dsn) {
+    const url = new URL(dsn);
+    const publicKey = url.username;
+    const host = url.host;
+    const projectId = url.pathname.slice(1);
+    return { publicKey, host, projectId };
+}
+export function sentryIngest(options) {
+    const { publicKey, host, projectId } = parseDSN(options.dsn);
+    return {
+        baseUrl: `https://${host}/api/${projectId}`,
+        headers: {
+            'Content-Type': 'application/x-sentry-envelope',
+            'X-Sentry-Auth': `Sentry sentry_version=7, sentry_client=recker/1.0, sentry_key=${publicKey}`,
+        },
+        timeout: 10 * 1000,
+        retry: {
+            maxAttempts: 2,
+            backoff: 'exponential',
+            delay: 500,
+            statusCodes: [408, 429, 500, 502, 503, 504],
+        }
+    };
+}
+export function createSentryEnvelope(event) {
+    const eventId = event.event_id || crypto.randomUUID().replace(/-/g, '');
+    const header = JSON.stringify({ event_id: eventId, sent_at: new Date().toISOString() });
+    const itemHeader = JSON.stringify({ type: 'event' });
+    const payload = JSON.stringify(event);
+    return `${header}\n${itemHeader}\n${payload}`;
+}

package/dist/presets/square.d.ts

@@ -0,0 +1,10 @@
+import { ClientOptions } from '../types/index.js';
+export interface SquarePresetOptions {
+    accessToken: string;
+    sandbox?: boolean;
+    version?: string;
+}
+export declare function square(options: SquarePresetOptions): ClientOptions;
+export declare const squarePayments: (options: SquarePresetOptions) => ClientOptions;
+export declare const squareCatalog: (options: SquarePresetOptions) => ClientOptions;
+export declare const squareSandbox: (options: Omit<SquarePresetOptions, "sandbox">) => ClientOptions;

package/dist/presets/square.js

@@ -0,0 +1,33 @@
+export function square(options) {
+    const baseUrl = options.sandbox
+        ? 'https://connect.squareupsandbox.com/v2'
+        : 'https://connect.squareup.com/v2';
+    return {
+        baseUrl,
+        headers: {
+            'Authorization': `Bearer ${options.accessToken}`,
+            'Content-Type': 'application/json',
+            'Square-Version': options.version || '2024-01-18',
+        },
+        timeout: 30 * 1000,
+        retry: {
+            maxAttempts: 3,
+            backoff: 'exponential',
+            delay: 1000,
+            statusCodes: [408, 429, 500, 502, 503, 504],
+        }
+    };
+}
+export const squarePayments = (options) => ({
+    ...square(options),
+    baseUrl: options.sandbox
+        ? 'https://connect.squareupsandbox.com/v2/payments'
+        : 'https://connect.squareup.com/v2/payments',
+});
+export const squareCatalog = (options) => ({
+    ...square(options),
+    baseUrl: options.sandbox
+        ? 'https://connect.squareupsandbox.com/v2/catalog'
+        : 'https://connect.squareup.com/v2/catalog',
+});
+export const squareSandbox = (options) => square({ ...options, sandbox: true });

package/dist/recker.d.ts

@@ -6,6 +6,7 @@ import { type WebSocketOptions, type ReckerWebSocket } from './websocket/client.
 import { createWhois, type WhoisResult, type WhoisOptions } from './utils/whois.js';
 import { type DNSClientOptions, type DNSClient } from './dns/index.js';
 import type { AIClientConfig } from './types/ai.js';
+import { uploadFile } from './utils/upload.js';
 export declare function get(url: string, options?: RequestOptions): RequestPromise;
 export declare function post(url: string, options?: RequestOptions): RequestPromise;
 export declare function put(url: string, options?: RequestOptions): RequestPromise;
@@ -38,6 +39,8 @@ export declare const recker: {
         extend: (defaults: Partial<import("./types/ai.js").ChatOptions>) => import("./types/ai.js").AIClient;
         readonly metrics: import("./types/ai.js").AIMetrics;
     };
+    har: import("./plugins/har-recorder.js").HarRecorder;
+    upload: typeof uploadFile;
     client: (options?: ExtendedClientOptions) => Client;
     dnsClient: (options?: DNSClientOptions) => DNSClient;
     whoisClient: typeof createWhois;

package/dist/recker.js

@@ -5,6 +5,8 @@ import { createWebSocket } from './websocket/client.js';
 import { whois as whoisLookup, isDomainAvailable, createWhois } from './utils/whois.js';
 import { createDNS } from './dns/index.js';
 import { createAI } from './ai/index.js';
+import { harRecorder } from './plugins/har-recorder.js';
+import { uploadFile } from './utils/upload.js';
 let _defaultClient = null;
 let _defaultDns = null;
 let _defaultAi = null;
@@ -87,6 +89,8 @@ export const recker = {
     dnsSecurity,
     ws,
     ai: aiNamespace,
+    har: harRecorder,
+    upload: uploadFile,
     client: (options) => createClient(options),
     dnsClient: (options) => createDNS(options),
     whoisClient: createWhois,

package/dist/scrape/document.d.ts

@@ -1,11 +1,12 @@
-import type { CheerioAPI } from 'cheerio';
 import { ScrapeElement } from './element.js';
+import { HTMLElement } from './parser/index.js';
 import type { ScrapeOptions, ExtractionSchema, ExtractedLink, ExtractedImage, ExtractedMeta, OpenGraphData, TwitterCardData, JsonLdData, ExtractedForm, ExtractedTable, ExtractedScript, ExtractedStyle, LinkExtractionOptions, ImageExtractionOptions } from './types.js';
 export declare class ScrapeDocument {
-    private $;
+    private rootElement;
     private options;
-    constructor($: CheerioAPI, options?: ScrapeOptions);
+    constructor(root: HTMLElement, options?: ScrapeOptions);
     static create(html: string, options?: ScrapeOptions): Promise<ScrapeDocument>;
+    static createSync(html: string, options?: ScrapeOptions): ScrapeDocument;
     select(selector: string): ScrapeElement;
     selectFirst(selector: string): ScrapeElement;
     selectAll(selector: string): ScrapeElement[];
@@ -39,6 +40,6 @@ export declare class ScrapeDocument {
     findByText(text: string, selector?: string): ScrapeElement[];
     findByExactText(text: string, selector?: string): ScrapeElement[];
     findByData(name: string, value?: string): ScrapeElement[];
-    get raw(): CheerioAPI;
+    get raw(): HTMLElement;
     get baseUrl(): string | undefined;
 }

package/dist/scrape/document.js

@@ -1,37 +1,32 @@
-import { requireOptional } from '../utils/optional-require.js';
 import { ScrapeElement } from './element.js';
+import { parse } from './parser/index.js';
 import { extractLinks, extractImages, extractMeta, extractOpenGraph, extractTwitterCard, extractJsonLd, extractForms, extractTables, extractScripts, extractStyles, } from './extractors.js';
-let cheerioModule = null;
-async function loadCheerio() {
-    if (cheerioModule) {
-        return cheerioModule;
-    }
-    cheerioModule = await requireOptional('cheerio', 'recker/scrape');
-    return cheerioModule;
-}
 export class ScrapeDocument {
-    $;
+    rootElement;
     options;
-    constructor($, options) {
-        this.$ = $;
+    constructor(root, options) {
+        this.rootElement = root;
         this.options = options || {};
     }
     static async create(html, options) {
-        const { load } = await loadCheerio();
-        return new ScrapeDocument(load(html), options);
+        const root = parse(html);
+        return new ScrapeDocument(root, options);
+    }
+    static createSync(html, options) {
+        const root = parse(html);
+        return new ScrapeDocument(root, options);
     }
     select(selector) {
-        return new ScrapeElement(this.$(selector), this.$);
+        const elements = this.rootElement.querySelectorAll(selector);
+        return new ScrapeElement(elements);
     }
     selectFirst(selector) {
-        return new ScrapeElement(this.$(selector).first(), this.$);
+        const element = this.rootElement.querySelector(selector);
+        return new ScrapeElement(element ? [element] : []);
     }
     selectAll(selector) {
-        const elements = [];
-        this.$(selector).each((_, element) => {
-            elements.push(new ScrapeElement(this.$(element), this.$));
-        });
-        return elements;
+        const elements = this.rootElement.querySelectorAll(selector);
+        return elements.map((el) => new ScrapeElement(el));
     }
     query(selector) {
         return this.select(selector);
@@ -40,78 +35,87 @@ export class ScrapeDocument {
         return this.selectAll(selector);
     }
     text(selector) {
-        return this.$(selector).first().text().trim();
+        const el = this.rootElement.querySelector(selector);
+        return el ? el.text.trim() : '';
     }
     texts(selector) {
-        const texts = [];
-        this.$(selector).each((_, element) => {
-            const text = this.$(element).text().trim();
-            if (text) {
-                texts.push(text);
-            }
-        });
-        return texts;
+        const elements = this.rootElement.querySelectorAll(selector);
+        return elements
+            .map((el) => el.text.trim())
+            .filter((text) => text.length > 0);
     }
     attr(selector, attribute) {
-        return this.$(selector).first().attr(attribute);
+        const el = this.rootElement.querySelector(selector);
+        return el ? el.getAttribute(attribute) : undefined;
     }
     attrs(selector, attribute) {
+        const elements = this.rootElement.querySelectorAll(selector);
         const attrs = [];
-        this.$(selector).each((_, element) => {
-            const value = this.$(element).attr(attribute);
-            if (value !== undefined) {
-                attrs.push(value);
+        elements.forEach((el) => {
+            const val = el.getAttribute(attribute);
+            if (val !== undefined) {
+                attrs.push(val);
             }
         });
         return attrs;
     }
     innerHtml(selector) {
-        return this.$(selector).first().html();
+        const el = this.rootElement.querySelector(selector);
+        return el ? el.innerHTML : null;
     }
     outerHtml(selector) {
-        const el = this.$(selector).first();
-        return this.$.html(el) || '';
+        const el = this.rootElement.querySelector(selector);
+        return el ? el.outerHTML : '';
     }
     links(options) {
-        return extractLinks(this.$, {
+        return extractLinks(this.rootElement, {
             ...options,
             baseUrl: this.options.baseUrl,
         });
     }
     images(options) {
-        return extractImages(this.$, {
+        return extractImages(this.rootElement, {
             ...options,
             baseUrl: this.options.baseUrl,
         });
     }
     meta() {
-        return extractMeta(this.$);
+        return extractMeta(this.rootElement);
     }
     openGraph() {
-        return extractOpenGraph(this.$);
+        return extractOpenGraph(this.rootElement);
     }
     twitterCard() {
-        return extractTwitterCard(this.$);
+        return extractTwitterCard(this.rootElement);
     }
     jsonLd() {
-        return extractJsonLd(this.$);
+        return extractJsonLd(this.rootElement);
     }
     forms(selector) {
-        return extractForms(this.$, selector);
+        return extractForms(this.rootElement, selector);
     }
     tables(selector) {
-        return extractTables(this.$, selector);
+        return extractTables(this.rootElement, selector);
     }
     scripts() {
-        return extractScripts(this.$);
+        return extractScripts(this.rootElement);
     }
     styles() {
-        return extractStyles(this.$);
+        return extractStyles(this.rootElement);
     }
     extract(schema) {
         const result = {};
         for (const [key, fieldConfig] of Object.entries(schema)) {
-            result[key] = this.extractField(fieldConfig);
+            const value = this.extractField(fieldConfig);
+            if (value === undefined) {
+                const isOptional = typeof fieldConfig === 'object' && fieldConfig.optional === true;
+                if (!isOptional) {
+                    result[key] = undefined;
+                }
+            }
+            else {
+                result[key] = value;
+            }
         }
         return result;
     }
@@ -120,77 +124,86 @@ export class ScrapeDocument {
             return this.text(field) || undefined;
         }
         const { selector, attribute, multiple, transform } = field;
+        const defaultValue = 'default' in field ? field.default : undefined;
         if (multiple) {
+            const elements = this.rootElement.querySelectorAll(selector);
             const values = [];
-            this.$(selector).each((_, element) => {
-                const $el = this.$(element);
+            elements.forEach(el => {
                 let value;
                 if (attribute) {
-                    value = $el.attr(attribute) || '';
+                    value = el.getAttribute(attribute) || '';
                 }
                 else {
-                    value = $el.text().trim();
+                    value = el.text.trim();
                 }
                 if (value) {
                     values.push(transform ? transform(value) : value);
                 }
             });
+            if (values.length === 0 && defaultValue !== undefined) {
+                return defaultValue;
+            }
             return values;
         }
         else {
-            const $el = this.$(selector).first();
+            const el = this.rootElement.querySelector(selector);
+            if (!el) {
+                return defaultValue;
+            }
             let value;
             if (attribute) {
-                value = $el.attr(attribute) || '';
+                value = el.getAttribute(attribute) || '';
             }
             else {
-                value = $el.text().trim();
+                value = el.text.trim();
+            }
+            if (!value) {
+                return defaultValue;
             }
-            if (!value)
-                return undefined;
             return transform ? transform(value) : value;
         }
     }
     title() {
-        const title = this.$('title').first().text().trim();
+        const el = this.rootElement.querySelector('title');
+        const title = el ? el.text.trim() : '';
         return title || undefined;
     }
     body() {
-        return new ScrapeElement(this.$('body').first(), this.$);
+        const el = this.rootElement.querySelector('body');
+        return new ScrapeElement(el ? [el] : []);
     }
     head() {
-        return new ScrapeElement(this.$('head').first(), this.$);
+        const el = this.rootElement.querySelector('head');
+        return new ScrapeElement(el ? [el] : []);
     }
     html() {
-        return this.$.html() || '';
+        return this.rootElement.outerHTML;
     }
     root() {
-        return new ScrapeElement(this.$.root(), this.$);
+        return new ScrapeElement(this.rootElement);
     }
     exists(selector) {
-        return this.$(selector).length > 0;
+        return this.rootElement.querySelector(selector) !== null;
     }
     count(selector) {
-        return this.$(selector).length;
+        return this.rootElement.querySelectorAll(selector).length;
     }
     findByText(text, selector) {
-        const baseSelector = selector || '*';
         const elements = [];
-        this.$(baseSelector).each((_, element) => {
-            const $el = this.$(element);
-            if ($el.text().includes(text)) {
-                elements.push(new ScrapeElement($el, this.$));
+        const candidates = selector ? this.rootElement.querySelectorAll(selector) : this.rootElement.querySelectorAll('*');
+        candidates.forEach(el => {
+            if (el.text.includes(text)) {
+                elements.push(new ScrapeElement(el));
             }
         });
         return elements;
     }
     findByExactText(text, selector) {
-        const baseSelector = selector || '*';
         const elements = [];
-        this.$(baseSelector).each((_, element) => {
-            const $el = this.$(element);
-            if ($el.text().trim() === text) {
-                elements.push(new ScrapeElement($el, this.$));
+        const candidates = selector ? this.rootElement.querySelectorAll(selector) : this.rootElement.querySelectorAll('*');
+        candidates.forEach(el => {
+            if (el.text.trim() === text) {
+                elements.push(new ScrapeElement(el));
             }
         });
         return elements;
@@ -202,7 +215,7 @@ export class ScrapeDocument {
         return this.selectAll(selector);
     }
     get raw() {
-        return this.$;
+        return this.rootElement;
     }
     get baseUrl() {
         return this.options.baseUrl;

package/dist/scrape/element.d.ts

@@ -1,9 +1,7 @@
-import type { Cheerio, CheerioAPI } from 'cheerio';
-import type { Element } from 'domhandler';
+import { HTMLElement, Node } from './parser/index.js';
 export declare class ScrapeElement {
-    private $el;
-    private $;
-    constructor($el: Cheerio<Element>, $: CheerioAPI);
+    private elements;
+    constructor(elements: HTMLElement | HTMLElement[] | Node | Node[]);
     find(selector: string): ScrapeElement;
     parent(selector?: string): ScrapeElement;
     children(selector?: string): ScrapeElement;
@@ -16,11 +14,15 @@ export declare class ScrapeElement {
     first(): ScrapeElement;
     last(): ScrapeElement;
     eq(index: number): ScrapeElement;
-    filter(selector: string): ScrapeElement;
+    filter(selectorOrCallback: string | ((el: ScrapeElement, index: number) => boolean)): ScrapeElement;
     not(selector: string): ScrapeElement;
     has(selector: string): ScrapeElement;
     add(selector: string): ScrapeElement;
     parents(selector?: string): ScrapeElement;
+    parentsUntil(selector: string): ScrapeElement;
+    nextUntil(selector: string): ScrapeElement;
+    prevUntil(selector: string): ScrapeElement;
+    slice(start: number, end?: number): ScrapeElement;
     contents(): ScrapeElement;
     text(): string;
     html(): string | null;
@@ -44,6 +46,6 @@ export declare class ScrapeElement {
     tagName(): string | undefined;
     clone(): ScrapeElement;
     toString(): string;
-    get raw(): Cheerio<Element>;
-    get(index?: number): Element | undefined;
+    get raw(): HTMLElement[];
+    get(index?: number): HTMLElement | undefined;
 }