recker 1.0.29 → 1.0.30

package/dist/core/client.d.ts

@@ -1,4 +1,5 @@
 import { ClientOptions, Middleware, ReckerRequest, ReckerResponse, RequestOptions, CacheStorage, PageResult } from '../types/index.js';
+import type { ClientAI, ClientOptionsWithAI } from '../types/ai-client.js';
 import { RequestPromise } from './request-promise.js';
 import { PaginationOptions } from '../plugins/pagination.js';
 import { RetryOptions } from '../plugins/retry.js';
@@ -36,7 +37,9 @@ export declare class Client {
     private cookieJar?;
     private cookieIgnoreInvalid;
     private defaultTimeout?;
-    constructor(options?: ExtendedClientOptions);
+    private _aiConfig?;
+    private _ai?;
+    constructor(options?: ExtendedClientOptions & Partial<ClientOptionsWithAI>);
     private createLoggingMiddleware;
     private createMaxSizeMiddleware;
     private setupCookieJar;
@@ -113,6 +116,8 @@ export declare class Client {
     whois(query: string, options?: WhoisOptions): Promise<WhoisResult>;
     isDomainAvailable(domain: string, options?: WhoisOptions): Promise<boolean>;
     hls(manifestUrl: string, options?: HlsOptions): HlsPromise;
+    get ai(): ClientAI;
+    get hasAI(): boolean;
 }
 export declare function createClient(options?: ExtendedClientOptions): Client;
 export {};

package/dist/core/client.js

@@ -1,4 +1,5 @@
 import { consoleLogger } from '../types/index.js';
+import { ClientAIImpl } from '../ai/client-ai.js';
 import { HttpRequest } from './request.js';
 import { UndiciTransport } from '../transport/undici.js';
 import { RequestPromise } from './request-promise.js';
@@ -41,6 +42,8 @@ export class Client {
     cookieJar;
     cookieIgnoreInvalid = false;
     defaultTimeout;
+    _aiConfig;
+    _ai;
     constructor(options = {}) {
         this.baseUrl = options.baseUrl || '';
         this.middlewares = options.middlewares || [];
@@ -150,6 +153,9 @@ export class Client {
         if (options.cookies) {
             this.setupCookieJar(options.cookies);
         }
+        if (options._aiConfig) {
+            this._aiConfig = options._aiConfig;
+        }
         if (this.maxResponseSize !== undefined) {
             this.middlewares.push(this.createMaxSizeMiddleware(this.maxResponseSize));
         }
@@ -661,6 +667,18 @@ export class Client {
     hls(manifestUrl, options = {}) {
         return new HlsPromise(this, manifestUrl, options);
     }
+    get ai() {
+        if (!this._ai) {
+            if (!this._aiConfig) {
+                throw new ConfigurationError('AI features require an AI-enabled preset. Use createClient(openai({...})), createClient(anthropic({...})), etc.', { configKey: '_aiConfig' });
+            }
+            this._ai = new ClientAIImpl(this, this._aiConfig);
+        }
+        return this._ai;
+    }
+    get hasAI() {
+        return this._aiConfig !== undefined;
+    }
 }
 export function createClient(options = {}) {
     return new Client(options);

package/dist/mcp/server.js

@@ -8,6 +8,9 @@ import { createHybridSearch } from './search/index.js';
 import { UnsupportedError } from '../core/errors.js';
 import { getIpInfo, isValidIP, isGeoIPAvailable, isBogon, isIPv6 } from './ip-intel.js';
 import { networkTools, networkToolHandlers } from './tools/network.js';
+import { seoTools, seoToolHandlers } from './tools/seo.js';
+import { scrapeTools, scrapeToolHandlers } from './tools/scrape.js';
+import { securityTools, securityToolHandlers } from './tools/security.js';
 import { ToolRegistry } from './tools/registry.js';
 import { loadToolModules } from './tools/loader.js';
 export class MCPServer {
@@ -45,6 +48,18 @@ export class MCPServer {
             tools: networkTools,
             handlers: networkToolHandlers
         });
+        this.toolRegistry.registerModule({
+            tools: seoTools,
+            handlers: seoToolHandlers
+        });
+        this.toolRegistry.registerModule({
+            tools: scrapeTools,
+            handlers: scrapeToolHandlers
+        });
+        this.toolRegistry.registerModule({
+            tools: securityTools,
+            handlers: securityToolHandlers
+        });
     }
     indexReady = null;
     async ensureIndexReady() {

package/dist/mcp/tools/scrape.d.ts

@@ -0,0 +1,3 @@
+import type { MCPTool, MCPToolResult } from '../types.js';
+export declare const scrapeTools: MCPTool[];
+export declare const scrapeToolHandlers: Record<string, (args: Record<string, unknown>) => Promise<MCPToolResult>>;

package/dist/mcp/tools/scrape.js

@@ -0,0 +1,156 @@
+import { createClient } from '../../core/client.js';
+import { ScrapeDocument } from '../../scrape/document.js';
+async function scrapeUrl(args) {
+    const url = String(args.url || '');
+    const selectors = args.selectors;
+    const extract = args.extract;
+    const selector = args.selector;
+    if (!url) {
+        return {
+            content: [{ type: 'text', text: 'Error: url is required' }],
+            isError: true,
+        };
+    }
+    try {
+        const client = createClient({ timeout: 30000 });
+        const response = await client.get(url);
+        const html = await response.text();
+        const doc = await ScrapeDocument.create(html, { baseUrl: url });
+        const output = {
+            url,
+            title: doc.title(),
+        };
+        if (selector) {
+            const elements = doc.selectAll(selector);
+            output.results = elements.map(el => ({
+                text: el.text(),
+                html: el.html(),
+                tag: el.tagName(),
+                attrs: el.attrs(),
+            }));
+            output.count = elements.length;
+        }
+        if (selectors && Object.keys(selectors).length > 0) {
+            const extracted = {};
+            for (const [key, sel] of Object.entries(selectors)) {
+                const isMultiple = sel.endsWith('[]');
+                const actualSel = isMultiple ? sel.slice(0, -2) : sel;
+                if (isMultiple) {
+                    extracted[key] = doc.texts(actualSel);
+                }
+                else {
+                    extracted[key] = doc.text(actualSel);
+                }
+            }
+            output.data = extracted;
+        }
+        const extractSet = new Set(extract || []);
+        if (extractSet.has('links') || extractSet.has('all')) {
+            const links = doc.links({ absolute: true });
+            output.links = links.slice(0, 100).map(l => ({
+                href: l.href,
+                text: l.text?.slice(0, 100),
+                rel: l.rel,
+            }));
+            output.linkCount = links.length;
+        }
+        if (extractSet.has('images') || extractSet.has('all')) {
+            const images = doc.images({ absolute: true });
+            output.images = images.slice(0, 50).map(img => ({
+                src: img.src,
+                alt: img.alt,
+                width: img.width,
+                height: img.height,
+            }));
+            output.imageCount = images.length;
+        }
+        if (extractSet.has('meta') || extractSet.has('all')) {
+            output.meta = doc.meta();
+        }
+        if (extractSet.has('og') || extractSet.has('all')) {
+            output.openGraph = doc.openGraph();
+        }
+        if (extractSet.has('twitter') || extractSet.has('all')) {
+            output.twitterCard = doc.twitterCard();
+        }
+        if (extractSet.has('jsonld') || extractSet.has('all')) {
+            output.jsonLd = doc.jsonLd();
+        }
+        if (extractSet.has('tables') || extractSet.has('all')) {
+            const tables = doc.tables();
+            output.tables = tables.slice(0, 10).map(t => ({
+                headers: t.headers,
+                rows: t.rows.slice(0, 50),
+            }));
+            output.tableCount = tables.length;
+        }
+        if (extractSet.has('forms') || extractSet.has('all')) {
+            output.forms = doc.forms();
+        }
+        if (extractSet.has('headings')) {
+            output.headings = {
+                h1: doc.texts('h1'),
+                h2: doc.texts('h2'),
+                h3: doc.texts('h3'),
+            };
+        }
+        return {
+            content: [{
+                    type: 'text',
+                    text: JSON.stringify(output, null, 2),
+                }],
+        };
+    }
+    catch (error) {
+        return {
+            content: [{
+                    type: 'text',
+                    text: `Scrape failed: ${error.message}`,
+                }],
+            isError: true,
+        };
+    }
+}
+export const scrapeTools = [
+    {
+        name: 'rek_scrape',
+        description: `Scrape a web page and extract data using CSS selectors.
+
+Supports multiple extraction modes:
+- Single selector: Extract elements matching one CSS selector
+- Selector map: Extract multiple fields at once
+- Built-in extractors: links, images, meta, og, twitter, jsonld, tables, forms, headings
+
+Examples:
+- Get all product titles: selector=".product-title"
+- Extract multiple fields: selectors={"title":"h1","price":".price","desc":".description"}
+- Get all links and images: extract=["links","images"]
+- Full extraction: extract=["all"]`,
+        inputSchema: {
+            type: 'object',
+            properties: {
+                url: {
+                    type: 'string',
+                    description: 'URL to scrape',
+                },
+                selector: {
+                    type: 'string',
+                    description: 'Single CSS selector to extract elements (e.g., ".product-card", "article h2")',
+                },
+                selectors: {
+                    type: 'object',
+                    description: 'Map of field names to CSS selectors. Add [] suffix for multiple values (e.g., {"title":"h1","links[]":"a"})',
+                },
+                extract: {
+                    type: 'array',
+                    items: { type: 'string' },
+                    description: 'Built-in extractors to run: links, images, meta, og, twitter, jsonld, tables, forms, headings, all',
+                },
+            },
+            required: ['url'],
+        },
+    },
+];
+export const scrapeToolHandlers = {
+    rek_scrape: scrapeUrl,
+};

package/dist/mcp/tools/security.d.ts

@@ -0,0 +1,3 @@
+import type { MCPTool, MCPToolResult } from '../types.js';
+export declare const securityTools: MCPTool[];
+export declare const securityToolHandlers: Record<string, (args: Record<string, unknown>) => Promise<MCPToolResult>>;