recker 1.0.28 → 1.0.29-next.604e03b

package/dist/presets/huggingface.js

@@ -1,4 +1,10 @@
 export function huggingface(options) {
+    const _aiConfig = {
+        provider: 'huggingface',
+        apiKey: options.apiKey,
+        model: options.model ?? 'meta-llama/Meta-Llama-3-70B-Instruct',
+        memory: { maxPairs: 12 },
+    };
     return {
         baseUrl: 'https://api-inference.huggingface.co',
         headers: {
@@ -11,6 +17,7 @@ export function huggingface(options) {
             backoff: 'exponential',
             delay: 1000,
             statusCodes: [408, 429, 500, 502, 503, 504]
-        }
+        },
+        _aiConfig,
     };
 }

package/dist/presets/mistral.d.ts

@@ -1,5 +1,7 @@
 import { ClientOptions } from '../types/index.js';
+import type { ClientOptionsWithAI } from '../types/ai-client.js';
 export interface MistralPresetOptions {
     apiKey: string;
+    model?: string;
 }
-export declare function mistral(options: MistralPresetOptions): ClientOptions;
+export declare function mistral(options: MistralPresetOptions): ClientOptions & ClientOptionsWithAI;

package/dist/presets/mistral.js

@@ -1,9 +1,14 @@
 export function mistral(options) {
+    const _aiConfig = {
+        provider: 'mistral',
+        apiKey: options.apiKey,
+        model: options.model ?? 'mistral-large-3',
+        memory: { maxPairs: 12 },
+    };
     return {
         baseUrl: 'https://api.mistral.ai/v1',
         headers: {
             'Authorization': `Bearer ${options.apiKey}`,
-            'Content-Type': 'application/json',
         },
         timeout: 5 * 60 * 1000,
         retry: {
@@ -11,6 +16,7 @@ export function mistral(options) {
             backoff: 'exponential',
             delay: 1000,
             statusCodes: [408, 429, 500, 502, 503, 504]
-        }
+        },
+        _aiConfig,
     };
 }

package/dist/presets/openai.d.ts

@@ -1,7 +1,9 @@
 import { ClientOptions } from '../types/index.js';
+import type { ClientOptionsWithAI } from '../types/ai-client.js';
 export interface OpenAIPresetOptions {
     apiKey: string;
     organization?: string;
     project?: string;
+    model?: string;
 }
-export declare function openai(options: OpenAIPresetOptions): ClientOptions;
+export declare function openai(options: OpenAIPresetOptions): ClientOptions & ClientOptionsWithAI;

package/dist/presets/openai.js

@@ -1,7 +1,6 @@
 export function openai(options) {
     const headers = {
         'Authorization': `Bearer ${options.apiKey}`,
-        'Content-Type': 'application/json'
     };
     if (options.organization) {
         headers['OpenAI-Organization'] = options.organization;
@@ -9,6 +8,13 @@ export function openai(options) {
     if (options.project) {
         headers['OpenAI-Project'] = options.project;
     }
+    const _aiConfig = {
+        provider: 'openai',
+        apiKey: options.apiKey,
+        model: options.model ?? 'gpt-5.1',
+        organization: options.organization,
+        memory: { maxPairs: 12 },
+    };
     return {
         baseUrl: 'https://api.openai.com/v1',
         headers,
@@ -18,6 +24,7 @@ export function openai(options) {
             backoff: 'exponential',
             delay: 1000,
             statusCodes: [408, 409, 429, 500, 502, 503, 504]
-        }
+        },
+        _aiConfig,
     };
 }

package/dist/presets/perplexity.d.ts

@@ -1,5 +1,7 @@
 import { ClientOptions } from '../types/index.js';
+import type { ClientOptionsWithAI } from '../types/ai-client.js';
 export interface PerplexityPresetOptions {
     apiKey: string;
+    model?: string;
 }
-export declare function perplexity(options: PerplexityPresetOptions): ClientOptions;
+export declare function perplexity(options: PerplexityPresetOptions): ClientOptions & ClientOptionsWithAI;

package/dist/presets/perplexity.js

@@ -1,9 +1,14 @@
 export function perplexity(options) {
+    const _aiConfig = {
+        provider: 'perplexity',
+        apiKey: options.apiKey,
+        model: options.model ?? 'sonar-pro',
+        memory: { maxPairs: 12 },
+    };
     return {
         baseUrl: 'https://api.perplexity.ai',
         headers: {
             'Authorization': `Bearer ${options.apiKey}`,
-            'Content-Type': 'application/json',
         },
         timeout: 5 * 60 * 1000,
         retry: {
@@ -11,6 +16,7 @@ export function perplexity(options) {
             backoff: 'exponential',
             delay: 1000,
             statusCodes: [408, 429, 500, 502, 503, 504]
-        }
+        },
+        _aiConfig,
     };
 }

package/dist/presets/registry.d.ts

@@ -1,4 +1,5 @@
 import { ClientOptions } from '../types/index.js';
+import type { AIProvider } from '../types/ai.js';
 export type PresetFactory = (options: any) => ClientOptions;
 export interface PresetInfo {
     name: string;
@@ -8,6 +9,9 @@ export interface PresetInfo {
     category: 'ai' | 'cloud' | 'saas' | 'devtools';
     requiredAuth: string[];
     docsUrl?: string;
+    aiProvider?: AIProvider;
+    defaultModel?: string;
+    chatEndpoint?: string;
 }
 export declare const presetRegistry: PresetInfo[];
 export declare function detectPreset(url: string): PresetInfo | undefined;

package/dist/presets/registry.js

@@ -43,6 +43,9 @@ export const presetRegistry = [
         category: 'ai',
         requiredAuth: ['apiKey'],
         docsUrl: 'https://platform.openai.com/docs',
+        aiProvider: 'openai',
+        defaultModel: 'gpt-4o',
+        chatEndpoint: '/chat/completions',
     },
     {
         name: 'anthropic',
@@ -52,6 +55,9 @@ export const presetRegistry = [
         category: 'ai',
         requiredAuth: ['apiKey'],
         docsUrl: 'https://docs.anthropic.com/',
+        aiProvider: 'anthropic',
+        defaultModel: 'claude-sonnet-4-20250514',
+        chatEndpoint: '/messages',
     },
     {
         name: 'gemini',
@@ -61,6 +67,9 @@ export const presetRegistry = [
         category: 'ai',
         requiredAuth: ['apiKey'],
         docsUrl: 'https://ai.google.dev/docs',
+        aiProvider: 'google',
+        defaultModel: 'gemini-2.0-flash',
+        chatEndpoint: '/models/{model}:generateContent',
     },
     {
         name: 'cohere',
@@ -70,6 +79,9 @@ export const presetRegistry = [
         category: 'ai',
         requiredAuth: ['apiKey'],
         docsUrl: 'https://docs.cohere.com/',
+        aiProvider: 'cohere',
+        defaultModel: 'command-r-plus',
+        chatEndpoint: '/chat',
     },
     {
         name: 'mistral',
@@ -79,6 +91,9 @@ export const presetRegistry = [
         category: 'ai',
         requiredAuth: ['apiKey'],
         docsUrl: 'https://docs.mistral.ai/',
+        aiProvider: 'mistral',
+        defaultModel: 'mistral-large-latest',
+        chatEndpoint: '/chat/completions',
     },
     {
         name: 'groq',
@@ -88,6 +103,9 @@ export const presetRegistry = [
         category: 'ai',
         requiredAuth: ['apiKey'],
         docsUrl: 'https://console.groq.com/docs',
+        aiProvider: 'groq',
+        defaultModel: 'llama-3.3-70b-versatile',
+        chatEndpoint: '/openai/v1/chat/completions',
     },
     {
         name: 'together',
@@ -97,6 +115,9 @@ export const presetRegistry = [
         category: 'ai',
         requiredAuth: ['apiKey'],
         docsUrl: 'https://docs.together.ai/',
+        aiProvider: 'together',
+        defaultModel: 'meta-llama/Llama-3.3-70B-Instruct-Turbo',
+        chatEndpoint: '/chat/completions',
     },
     {
         name: 'replicate',
@@ -106,6 +127,9 @@ export const presetRegistry = [
         category: 'ai',
         requiredAuth: ['apiKey'],
         docsUrl: 'https://replicate.com/docs',
+        aiProvider: 'replicate',
+        defaultModel: 'meta/llama-2-70b-chat',
+        chatEndpoint: '/predictions',
     },
     {
         name: 'huggingface',
@@ -115,6 +139,9 @@ export const presetRegistry = [
         category: 'ai',
         requiredAuth: ['apiKey'],
         docsUrl: 'https://huggingface.co/docs/api-inference',
+        aiProvider: 'huggingface',
+        defaultModel: 'meta-llama/Meta-Llama-3-70B-Instruct',
+        chatEndpoint: '/models/{model}',
     },
     {
         name: 'perplexity',
@@ -124,6 +151,9 @@ export const presetRegistry = [
         category: 'ai',
         requiredAuth: ['apiKey'],
         docsUrl: 'https://docs.perplexity.ai/',
+        aiProvider: 'perplexity',
+        defaultModel: 'llama-3.1-sonar-large-128k-online',
+        chatEndpoint: '/chat/completions',
     },
     {
         name: 'deepseek',
@@ -133,6 +163,9 @@ export const presetRegistry = [
         category: 'ai',
         requiredAuth: ['apiKey'],
         docsUrl: 'https://platform.deepseek.com/docs',
+        aiProvider: 'deepseek',
+        defaultModel: 'deepseek-chat',
+        chatEndpoint: '/chat/completions',
     },
     {
         name: 'fireworks',
@@ -142,6 +175,9 @@ export const presetRegistry = [
         category: 'ai',
         requiredAuth: ['apiKey'],
         docsUrl: 'https://docs.fireworks.ai/',
+        aiProvider: 'fireworks',
+        defaultModel: 'accounts/fireworks/models/llama-v3p1-70b-instruct',
+        chatEndpoint: '/inference/v1/chat/completions',
     },
     {
         name: 'xai',
@@ -151,6 +187,9 @@ export const presetRegistry = [
         category: 'ai',
         requiredAuth: ['apiKey'],
         docsUrl: 'https://docs.x.ai/',
+        aiProvider: 'xai',
+        defaultModel: 'grok-2',
+        chatEndpoint: '/v1/chat/completions',
     },
     {
         name: 'grok',
@@ -160,6 +199,9 @@ export const presetRegistry = [
         category: 'ai',
         requiredAuth: ['apiKey'],
         docsUrl: 'https://docs.x.ai/',
+        aiProvider: 'xai',
+        defaultModel: 'grok-2',
+        chatEndpoint: '/v1/chat/completions',
     },
     {
         name: 'azure-openai',
@@ -169,6 +211,9 @@ export const presetRegistry = [
         category: 'ai',
         requiredAuth: ['resourceName', 'apiKey'],
         docsUrl: 'https://learn.microsoft.com/en-us/azure/ai-services/openai/',
+        aiProvider: 'azure-openai',
+        defaultModel: 'gpt-4o',
+        chatEndpoint: '/openai/deployments/{deployment}/chat/completions',
     },
     {
         name: 'cloudflare-workers-ai',
@@ -178,6 +223,9 @@ export const presetRegistry = [
         category: 'ai',
         requiredAuth: ['accountId', 'apiToken'],
         docsUrl: 'https://developers.cloudflare.com/workers-ai/',
+        aiProvider: 'cloudflare-workers-ai',
+        defaultModel: '@cf/meta/llama-3-8b-instruct',
+        chatEndpoint: '/run/{model}',
     },
     {
         name: 'aws',

package/dist/presets/replicate.d.ts

@@ -1,5 +1,7 @@
 import { ClientOptions } from '../types/index.js';
+import type { ClientOptionsWithAI } from '../types/ai-client.js';
 export interface ReplicatePresetOptions {
     apiKey: string;
+    model?: string;
 }
-export declare function replicate(options: ReplicatePresetOptions): ClientOptions;
+export declare function replicate(options: ReplicatePresetOptions): ClientOptions & ClientOptionsWithAI;

package/dist/presets/replicate.js

@@ -1,4 +1,10 @@
 export function replicate(options) {
+    const _aiConfig = {
+        provider: 'replicate',
+        apiKey: options.apiKey,
+        model: options.model ?? 'meta/llama-2-70b-chat',
+        memory: { maxPairs: 12 },
+    };
     return {
         baseUrl: 'https://api.replicate.com/v1',
         headers: {
@@ -11,6 +17,7 @@ export function replicate(options) {
             backoff: 'exponential',
             delay: 2000,
             statusCodes: [408, 429, 500, 502, 503, 504]
-        }
+        },
+        _aiConfig,
     };
 }

package/dist/presets/together.d.ts

@@ -1,5 +1,7 @@
 import { ClientOptions } from '../types/index.js';
+import type { ClientOptionsWithAI } from '../types/ai-client.js';
 export interface TogetherPresetOptions {
     apiKey: string;
+    model?: string;
 }
-export declare function together(options: TogetherPresetOptions): ClientOptions;
+export declare function together(options: TogetherPresetOptions): ClientOptions & ClientOptionsWithAI;

package/dist/presets/together.js

@@ -1,9 +1,14 @@
 export function together(options) {
+    const _aiConfig = {
+        provider: 'together',
+        apiKey: options.apiKey,
+        model: options.model ?? 'meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8',
+        memory: { maxPairs: 12 },
+    };
     return {
         baseUrl: 'https://api.together.xyz/v1',
         headers: {
             'Authorization': `Bearer ${options.apiKey}`,
-            'Content-Type': 'application/json',
         },
         timeout: 5 * 60 * 1000,
         retry: {
@@ -11,6 +16,7 @@ export function together(options) {
             backoff: 'exponential',
             delay: 1000,
             statusCodes: [408, 429, 500, 502, 503, 504]
-        }
+        },
+        _aiConfig,
     };
 }

package/dist/presets/xai.d.ts

@@ -1,6 +1,8 @@
 import { ClientOptions } from '../types/index.js';
+import type { ClientOptionsWithAI } from '../types/ai-client.js';
 export interface XAIPresetOptions {
     apiKey: string;
+    model?: string;
 }
-export declare function xai(options: XAIPresetOptions): ClientOptions;
+export declare function xai(options: XAIPresetOptions): ClientOptions & ClientOptionsWithAI;
 export declare const grok: typeof xai;

package/dist/presets/xai.js

@@ -1,9 +1,14 @@
 export function xai(options) {
+    const _aiConfig = {
+        provider: 'xai',
+        apiKey: options.apiKey,
+        model: options.model ?? 'grok-4.1',
+        memory: { maxPairs: 12 },
+    };
     return {
         baseUrl: 'https://api.x.ai/v1',
         headers: {
             'Authorization': `Bearer ${options.apiKey}`,
-            'Content-Type': 'application/json',
         },
         timeout: 5 * 60 * 1000,
         retry: {
@@ -11,7 +16,8 @@ export function xai(options) {
             backoff: 'exponential',
             delay: 1000,
             statusCodes: [408, 429, 500, 502, 503, 504]
-        }
+        },
+        _aiConfig,
     };
 }
 export const grok = xai;

package/dist/scrape/index.d.ts

@@ -1,4 +1,6 @@
 export { ScrapeDocument } from './document.js';
 export { ScrapeElement } from './element.js';
+export { Spider, spider } from './spider.js';
+export type { SpiderOptions, SpiderPageResult, SpiderProgress, SpiderResult, } from './spider.js';
 export { extractLinks, extractImages, extractMeta, extractOpenGraph, extractTwitterCard, extractJsonLd, extractForms, extractTables, extractScripts, extractStyles, } from './extractors.js';
 export type { ExtractedLink, ExtractedImage, ExtractedMeta, OpenGraphData, TwitterCardData, JsonLdData, ExtractedForm, ExtractedFormField, ExtractedTable, ExtractedScript, ExtractedStyle, ExtractionSchema, ExtractionSchemaField, ScrapeOptions, LinkExtractionOptions, ImageExtractionOptions, } from './types.js';

package/dist/scrape/index.js

@@ -1,3 +1,4 @@
 export { ScrapeDocument } from './document.js';
 export { ScrapeElement } from './element.js';
+export { Spider, spider } from './spider.js';
 export { extractLinks, extractImages, extractMeta, extractOpenGraph, extractTwitterCard, extractJsonLd, extractForms, extractTables, extractScripts, extractStyles, } from './extractors.js';

package/dist/scrape/spider.d.ts

@@ -0,0 +1,61 @@
+import type { ExtractedLink } from './types.js';
+export interface SpiderOptions {
+    maxDepth?: number;
+    maxPages?: number;
+    sameDomain?: boolean;
+    concurrency?: number;
+    timeout?: number;
+    delay?: number;
+    exclude?: RegExp[];
+    include?: RegExp[];
+    userAgent?: string;
+    respectRobotsTxt?: boolean;
+    onPage?: (result: SpiderPageResult) => void;
+    onProgress?: (progress: SpiderProgress) => void;
+}
+export interface SpiderPageResult {
+    url: string;
+    status: number;
+    title: string;
+    depth: number;
+    links: ExtractedLink[];
+    duration: number;
+    error?: string;
+}
+export interface SpiderProgress {
+    crawled: number;
+    queued: number;
+    total: number;
+    currentUrl: string;
+    depth: number;
+}
+export interface SpiderResult {
+    startUrl: string;
+    pages: SpiderPageResult[];
+    visited: Set<string>;
+    duration: number;
+    errors: Array<{
+        url: string;
+        error: string;
+    }>;
+}
+export declare class Spider {
+    private options;
+    private client;
+    private pool;
+    private visited;
+    private queue;
+    private results;
+    private errors;
+    private baseHost;
+    private running;
+    private aborted;
+    private pendingCount;
+    constructor(options?: SpiderOptions);
+    crawl(startUrl: string): Promise<SpiderResult>;
+    private crawlPage;
+    abort(): void;
+    isRunning(): boolean;
+    getProgress(): SpiderProgress;
+}
+export declare function spider(url: string, options?: SpiderOptions): Promise<SpiderResult>;