recker 1.0.73 → 1.0.75-next.2e5a94f

package/dist/mcp/cli.js

@@ -1,7 +1,7 @@
 #!/usr/bin/env node
 import { RekCommand as Command } from '../cli/router.js';
 import { MCPServer } from './server.js';
-import { listCategories, validateCategories, estimateCategoryTokens, } from './profiles.js';
+import { listCategories, DEFAULT_CATEGORY, validateCategories, estimateCategoryTokens, } from './profiles.js';
 import { LEGACY_TOOL_GROUPS } from './legacy-tool-groups.js';
 const program = new Command('recker-mcp');
 program
@@ -13,7 +13,7 @@ program
     .option('--docs-path <path>', 'Path to documentation directory')
     .option('--examples-path <path>', 'Path to examples directory')
     .option('--src-path <path>', 'Path to source directory')
-    .option('-c, --category <categories>', 'Tool categories to enable (comma-separated): minimal, docs, network, dns, seo, security, scrape, video, ai, protocols, parsing, streaming, full')
+    .option('-c, --category <categories>', 'Tool categories to enable (comma-separated): minimal, docs, network, dns, seo, security, scrape, video, ai, protocols, parsing, streaming, template, full')
     .option('--list-categories', 'List available categories and exit')
     .option('--no-docs', 'Disable documentation tools (search, get, examples, schema, suggest)')
     .option('--no-http', 'Disable HTTP request tool')
@@ -50,6 +50,7 @@ program
         console.log('');
         process.exit(0);
     }
+    const useExplicitCategory = Boolean(opts.category);
     if (opts.category) {
         const categoryNames = opts.category.split(',').map((p) => p.trim());
         const validation = validateCategories(categoryNames);
@@ -99,6 +100,7 @@ program
         console.error(`Invalid transport mode: ${transport}. Use: stdio, http, or sse`);
         process.exit(1);
     }
+    const effectiveCategory = useExplicitCategory ? opts.category : (!opts.only && !opts.filter ? DEFAULT_CATEGORY : undefined);
     const server = new MCPServer({
         transport,
         port,
@@ -106,8 +108,8 @@ program
         docsPath: opts.docsPath,
         examplesPath: opts.examplesPath,
         srcPath: opts.srcPath,
-        category: opts.category,
-        toolsFilter: !opts.category && toolsFilter.length > 0 ? toolsFilter : undefined,
+        category: effectiveCategory,
+        toolsFilter: !effectiveCategory && toolsFilter.length > 0 ? toolsFilter : undefined,
     });
     if (transport !== 'stdio') {
         console.log('╔═══════════════════════════════════════════════════════════════════╗');
@@ -117,15 +119,15 @@ program
         console.log(`  Transport: ${transport}`);
         console.log(`  Port:      ${port}`);
         console.log(`  Debug:     ${opts.debug ? 'enabled' : 'disabled'}`);
-        if (opts.category) {
-            const tokens = estimateCategoryTokens(opts.category);
-            console.log(`  Category:  ${opts.category} (~${tokens} tokens)`);
+        if (effectiveCategory) {
+            const tokens = estimateCategoryTokens(effectiveCategory);
+            console.log(`  Category:  ${effectiveCategory} (~${tokens} tokens)`);
         }
         else if (toolsFilter.length > 0) {
             console.log(`  Filters:   ${toolsFilter.join(', ')}`);
         }
         else {
-            console.log(`  Category:  full (all tools enabled)`);
+            console.log(`  Category:  minimal (default)`);
         }
         console.log('');
         console.log('  Available tools:');

package/dist/mcp/profiles.d.ts

@@ -1,4 +1,4 @@
-export type CategoryName = 'minimal' | 'docs' | 'network' | 'dns' | 'seo' | 'security' | 'scrape' | 'video' | 'ai' | 'protocols' | 'parsing' | 'streaming' | 'full';
+export type CategoryName = 'minimal' | 'docs' | 'network' | 'dns' | 'seo' | 'security' | 'scrape' | 'video' | 'ai' | 'protocols' | 'parsing' | 'streaming' | 'template' | 'full';
 export interface Category {
     name: CategoryName;
     description: string;

package/dist/mcp/profiles.js

@@ -118,11 +118,10 @@ export const categories = {
     },
     ai: {
         name: 'ai',
-        description: 'Multi-provider AI chat, embeddings, and comparison',
+        description: 'Multi-provider AI chat and comparison',
         icon: '🤖',
         tools: [
             'rek_ai_chat',
-            'rek_ai_embed',
             'rek_ai_providers',
             'rek_ai_tokens',
             'rek_ai_compare',
@@ -173,14 +172,39 @@ export const categories = {
         ],
         estimatedTokens: 900,
     },
+    template: {
+        name: 'template',
+        description: 'Template rendering, validation, parsing, and helper metadata',
+        icon: '📝',
+        tools: [
+            'rek_template_render',
+            'rek_template_validate',
+            'rek_template_parse',
+            'rek_template_variables',
+            'rek_template_check',
+            'rek_template_helpers',
+        ],
+        estimatedTokens: 1800,
+    },
     full: {
         name: 'full',
         description: 'All available tools (high context cost)',
         icon: '🌟',
         tools: ['*'],
-        estimatedTokens: 18000,
+        estimatedTokens: 0,
     },
 };
+function getAllConcreteProfileTools() {
+    const toolNames = new Set();
+    for (const category of Object.values(categories)) {
+        for (const tool of category.tools) {
+            if (tool !== '*') {
+                toolNames.add(tool);
+            }
+        }
+    }
+    return toolNames;
+}
 export const DEFAULT_CATEGORY = 'minimal';
 export const profiles = categories;
 export const DEFAULT_PROFILE = DEFAULT_CATEGORY;
@@ -226,7 +250,7 @@ export function estimateCategoryTokens(categoryNames) {
         if (!category)
             continue;
         if (category.tools.includes('*')) {
-            return category.estimatedTokens;
+            return getAllConcreteProfileTools().size * 300;
         }
         for (const tool of category.tools) {
             if (!seenTools.has(tool)) {
@@ -239,12 +263,13 @@ export function estimateCategoryTokens(categoryNames) {
 }
 export const estimateProfileTokens = estimateCategoryTokens;
 export function listCategories() {
+    const toolCount = getAllConcreteProfileTools().size;
     return Object.values(categories).map((p) => ({
         name: p.name,
         description: p.description,
         icon: p.icon,
-        toolCount: p.tools.includes('*') ? -1 : p.tools.length,
-        estimatedTokens: p.estimatedTokens,
+        toolCount: p.tools.includes('*') ? toolCount : p.tools.length,
+        estimatedTokens: p.estimatedTokens || (p.tools.includes('*') ? toolCount * 300 : p.tools.length * 300),
     }));
 }
 export const listProfiles = listCategories;

package/dist/mcp/tools/categories.js

@@ -97,7 +97,6 @@ export const toolCategories = {
     rek_dns_toolkit: 'security',
     rek_security_headers: 'security',
     rek_ai_chat: 'ai',
-    rek_ai_embed: 'ai',
     rek_ai_providers: 'ai',
     rek_ai_tokens: 'ai',
     rek_ai_compare: 'ai',

package/dist/plugins/auth/aws-sigv4.d.ts

@@ -8,3 +8,4 @@ export interface AWSSignatureV4Options {
 }
 export declare function awsSignatureV4(options: AWSSignatureV4Options): Middleware;
 export declare function awsSignatureV4Plugin(options: AWSSignatureV4Options): Plugin;
+export declare function clearAwsSigV4PluginCache(): void;

package/dist/plugins/auth/aws-sigv4.js

@@ -81,8 +81,25 @@ export function awsSignatureV4(options) {
         return next(newReq);
     };
 }
+const awsSigV4PluginCache = new Map();
 export function awsSignatureV4Plugin(options) {
-    return (client) => {
-        client.use(awsSignatureV4(options));
+    const cacheKey = [
+        options.accessKeyId,
+        options.secretAccessKey,
+        options.region,
+        options.service,
+        options.sessionToken || '',
+    ].join('|');
+    const cached = awsSigV4PluginCache.get(cacheKey);
+    if (cached)
+        return cached;
+    const safeOptions = { ...options };
+    const plugin = (client) => {
+        client.use(awsSignatureV4(safeOptions));
     };
+    awsSigV4PluginCache.set(cacheKey, plugin);
+    return plugin;
+}
+export function clearAwsSigV4PluginCache() {
+    awsSigV4PluginCache.clear();
 }

package/dist/plugins/retry.js

@@ -1,4 +1,5 @@
-import { HttpError, NetworkError, TimeoutError } from '../core/errors.js';
+import { HttpError, NetworkError, TimeoutError, classifyTransportError } from '../core/errors.js';
+import { getRequestContext } from '../core-runtime/request-context.js';
 function calculateDelay(attempt, baseDelay, maxDelay, strategy, useJitter) {
     let calculatedDelay;
     switch (strategy) {
@@ -54,6 +55,13 @@ export function retryPlugin(options = {}) {
         if (error instanceof HttpError) {
             return statusCodes.includes(error.status);
         }
+        const classification = classifyTransportError(error);
+        if (classification && classification.canRetry === false) {
+            return false;
+        }
+        if (classification) {
+            return classification.canRetry;
+        }
         if (error && typeof error === 'object' && 'code' in error) {
             const code = error.code;
             return code === 'ECONNRESET' || code === 'ETIMEDOUT' || code === 'ENOTFOUND';
@@ -62,6 +70,24 @@ export function retryPlugin(options = {}) {
     };
     const shouldRetry = options.shouldRetry || defaultShouldRetry;
     return (client) => {
+        const emitRequestRetry = (error, attempt, delayMs, req) => {
+            const eventBus = client.runtimeEventBus;
+            if (!eventBus?.emit) {
+                return;
+            }
+            const context = getRequestContext(req);
+            if (!context) {
+                return;
+            }
+            const classification = classifyTransportError(error);
+            eventBus.emit('request:retry', {
+                context,
+                req,
+                attempt,
+                delayMs,
+                reason: classification?.reason || error?.message || 'retry requested'
+            });
+        };
         const middleware = async (req, next) => {
             let attempt = 0;
             while (true) {
@@ -80,6 +106,7 @@ export function retryPlugin(options = {}) {
                             delayMs = calculateDelay(attempt, baseDelay, maxDelay, backoffStrategy, useJitter);
                         }
                         const err = new HttpError(res, req);
+                        emitRequestRetry(err, attempt, delayMs, req);
                         if (onRetry) {
                             onRetry(attempt, err, delayMs);
                         }
@@ -96,6 +123,7 @@ export function retryPlugin(options = {}) {
                 catch (error) {
                     if (attempt < maxAttempts && shouldRetry(error)) {
                         const delayMs = calculateDelay(attempt, baseDelay, maxDelay, backoffStrategy, useJitter);
+                        emitRequestRetry(error, attempt, delayMs, req);
                         if (onRetry) {
                             onRetry(attempt, error, delayMs);
                         }

package/dist/presets/aws.d.ts

@@ -10,6 +10,7 @@ export interface AWSPresetOptions {
 export type AWSService = 's3' | 'dynamodb' | 'lambda' | 'sqs' | 'sns' | 'ses' | 'secretsmanager' | 'ssm' | 'sts' | 'iam' | 'ec2' | 'ecs' | 'eks' | 'cloudwatch' | 'logs' | 'events' | 'kinesis' | 'firehose' | 'apigateway' | 'execute-api' | 'cognito-idp' | 'cognito-identity' | 'kms' | 'athena' | 'glue' | 'stepfunctions' | 'states' | 'bedrock' | 'bedrock-runtime';
 export declare function aws(options: AWSPresetOptions): ClientOptions;
 export declare function awsS3(options: Omit<AWSPresetOptions, 'service'>): ClientOptions;
+export declare function s3(options: Omit<AWSPresetOptions, 'service'>): ClientOptions;
 export declare function awsDynamoDB(options: Omit<AWSPresetOptions, 'service'>): ClientOptions;
 export declare function awsLambda(options: Omit<AWSPresetOptions, 'service'>): ClientOptions;
 export declare function awsSQS(options: Omit<AWSPresetOptions, 'service'>): ClientOptions;

package/dist/presets/aws.js

@@ -1,4 +1,62 @@
 import { awsSignatureV4Plugin } from '../plugins/auth.js';
+const S3_STORAGE_PRESET = {
+    timeout: {
+        lookup: 5000,
+        connect: 10000,
+        secureConnect: 10000,
+        response: 30000,
+        request: 60000,
+    },
+    http2: 'performance',
+    expectContinue: 2 * 1024 * 1024,
+    observability: false,
+    concurrency: {
+        agent: {
+            connections: 100,
+            pipelining: 10,
+            keepAlive: true,
+            keepAliveTimeout: 4000,
+            keepAliveMaxTimeout: 600000,
+            keepAliveTimeoutThreshold: 1000,
+            maxRequestsPerClient: 100,
+            clientTtl: null,
+            maxCachedSessions: 100,
+        },
+    },
+    retry: {
+        maxDelay: 30000,
+    },
+};
+function createAwsS3Preset(options) {
+    const baseConfig = aws({
+        ...options,
+        service: 's3',
+    });
+    const baseConcurrency = baseConfig.concurrency;
+    let concurrency;
+    if (typeof baseConcurrency === 'number') {
+        concurrency = baseConcurrency;
+    }
+    else {
+        const mergedConcurrency = {};
+        if (baseConcurrency && typeof baseConcurrency === 'object') {
+            Object.assign(mergedConcurrency, S3_STORAGE_PRESET.concurrency, baseConcurrency);
+        }
+        else {
+            Object.assign(mergedConcurrency, S3_STORAGE_PRESET.concurrency);
+        }
+        concurrency = mergedConcurrency;
+    }
+    return {
+        ...baseConfig,
+        ...S3_STORAGE_PRESET,
+        retry: {
+            ...baseConfig.retry,
+            ...S3_STORAGE_PRESET.retry,
+        },
+        concurrency,
+    };
+}
 function getServiceEndpoint(service, region) {
     const globalServices = ['iam', 'sts', 'cloudfront', 'route53'];
     if (globalServices.includes(service)) {
@@ -43,7 +101,10 @@ export function aws(options) {
     };
 }
 export function awsS3(options) {
-    return aws({ ...options, service: 's3' });
+    return createAwsS3Preset(options);
+}
+export function s3(options) {
+    return createAwsS3Preset(options);
 }
 export function awsDynamoDB(options) {
     return aws({ ...options, service: 'dynamodb' });

package/dist/recker.d.ts

@@ -10,6 +10,7 @@ import { uploadFile } from './utils/upload.js';
 import { VideoBuilder } from './video/builder.js';
 import { getExtractorName, listExtractors } from './extractors/index.js';
 import type { ExtractorResult } from './extractors/base.js';
+import { type GoogleSearchAdvancedOptions, type GoogleSearchResponse } from './search/index.js';
 export declare function get(url: string, options?: RequestOptions): RequestPromise;
 export declare function post(url: string, options?: RequestOptions): RequestPromise;
 export declare function put(url: string, options?: RequestOptions): RequestPromise;
@@ -25,6 +26,7 @@ export declare function ws(url: string, options?: WebSocketOptions): ReckerWebSo
 export declare function video(url: string): VideoBuilder;
 export declare function isVideoSupported(url: string): Promise<boolean>;
 export declare function extractVideo(url: string): Promise<ExtractorResult>;
+export declare function searchGoogle(query: string, options?: GoogleSearchAdvancedOptions): Promise<GoogleSearchResponse>;
 export declare const recker: {
     get: typeof get;
     post: typeof post;
@@ -81,6 +83,7 @@ export declare const recker: {
         XVideosQuickies: typeof import("./extractors/xvideos.js").XVideosQuickiesExtractor;
         Generic: typeof import("./extractors/generic.js").GenericExtractor;
     };
+    searchGoogle: typeof searchGoogle;
     listExtractors: typeof listExtractors;
     getExtractorName: typeof getExtractorName;
     ai: {

package/dist/recker.js

@@ -8,6 +8,7 @@ import { harRecorder } from './plugins/har-recorder.js';
 import { uploadFile } from './utils/upload.js';
 import { createVideoBuilder } from './video/builder.js';
 import { extract, extractors, isSupported, getExtractorName, listExtractors } from './extractors/index.js';
+import { searchGoogleAdvanced } from './search/index.js';
 let _defaultClient = null;
 let _defaultDns = null;
 let _defaultAi = null;
@@ -74,6 +75,9 @@ export async function isVideoSupported(url) {
 export async function extractVideo(url) {
     return extract(url, getDefaultClient());
 }
+export function searchGoogle(query, options) {
+    return searchGoogleAdvanced(query, options);
+}
 const aiNamespace = {
     chat: (...args) => getDefaultAi().chat(...args),
     stream: (...args) => getDefaultAi().stream(...args),
@@ -100,6 +104,7 @@ export const recker = {
     isVideoSupported,
     extractVideo,
     extractors,
+    searchGoogle,
     listExtractors,
     getExtractorName,
     ai: aiNamespace,

package/dist/runner/request-runner.d.ts

@@ -3,6 +3,12 @@ export interface RunnerOptions {
     retries?: number;
     retryDelay?: number;
 }
+export interface RunnerRunOptions {
+    priority?: number;
+    retries?: number;
+    signal?: AbortSignal;
+    deadlineMs?: number;
+}
 export interface RequestTask<T = any> {
     id: string;
     fn: () => Promise<T>;
@@ -35,10 +41,14 @@ export declare class RequestRunner extends SimpleEmitter {
     private queue;
     private activeCount;
     private paused;
-    private results;
     private stats;
     private startTime;
     private pendingRetries;
+    private isCancelled;
+    private cancelReason;
+    private retryTimers;
+    private timeoutId?;
+    private abortUnsubscribe?;
     constructor(options?: RunnerOptions);
     add<T>(fn: () => Promise<T>, options?: {
         priority?: number;
@@ -47,13 +57,13 @@ export declare class RequestRunner extends SimpleEmitter {
         trackTotal?: boolean;
         resolve?: (value: T | Error) => void;
     }): void;
-    run<T>(items: any[], processor: (item: any, index: number) => Promise<T>, options?: {
-        priority?: number;
-        retries?: number;
-    }): Promise<RunnerResult<T>>;
+    run<T>(items: any[], processor: (item: any, index: number) => Promise<T>, options?: RunnerRunOptions): Promise<RunnerResult<T>>;
     private queueTask;
     private scheduleRetry;
     private processNext;
+    private cancelAll;
+    private resolveQueue;
+    private resolveTask;
     getProgress(): {
         total: number;
         completed: number;