recker 1.0.29 → 1.0.31

package/dist/cli/tui/shell.js

@@ -16,6 +16,7 @@ import { getShellSearch } from './shell-search.js';
 import { openSearchPanel } from './search-panel.js';
 import { ScrollBuffer, parseScrollKey, parseMouseScroll, disableMouseReporting } from './scroll-buffer.js';
 import { analyzeSeo, SeoSpider } from '../../seo/index.js';
+import { resolvePreset } from '../presets.js';
 let highlight;
 async function initDependencies() {
     if (!highlight) {
@@ -43,6 +44,7 @@ export class RekShell {
     scrollBuffer;
     originalStdoutWrite = null;
     inScrollMode = false;
+    aiClients = new Map();
     constructor() {
         this.client = createClient({
             baseUrl: 'http://localhost',
@@ -94,8 +96,10 @@ export class RekShell {
         const commands = [
             'get', 'post', 'put', 'delete', 'patch', 'head', 'options',
             'ws', 'udp', 'load', 'chat', 'ai',
+            '@openai', '@anthropic', '@groq', '@google', '@xai', '@mistral', '@cohere', '@deepseek', '@fireworks', '@together', '@perplexity',
+            'ai:clear',
             'whois', 'tls', 'ssl', 'security', 'ip', 'dns', 'dns:propagate', 'dns:email', 'rdap', 'ping',
-            'scrap', 'spider', '$', '$text', '$attr', '$html', '$links', '$images', '$scripts', '$css', '$sourcemaps', '$unmap', '$unmap:view', '$unmap:save', '$beautify', '$beautify:save', '$table',
+            'scrap', 'spider', 'seo', '$', '$text', '$attr', '$html', '$links', '$images', '$scripts', '$css', '$sourcemaps', '$unmap', '$unmap:view', '$unmap:save', '$beautify', '$beautify:save', '$table',
             '?', 'search', 'suggest', 'example',
             'help', 'clear', 'exit', 'set', 'url', 'vars', 'env'
         ];
@@ -173,20 +177,36 @@ export class RekShell {
                         }
                         return true;
                     }
-                    const scrollKey = parseScrollKey(data);
-                    if (scrollKey) {
-                        if (scrollKey === 'quit') {
-                            if (self.inScrollMode) {
+                    try {
+                        const scrollKey = parseScrollKey(data);
+                        if (scrollKey) {
+                            if (scrollKey === 'quit') {
+                                if (self.inScrollMode) {
+                                    self.exitScrollMode();
+                                    return true;
+                                }
+                                return originalEmit(event, ...args);
+                            }
+                            self.handleScrollKey(scrollKey);
+                            return true;
+                        }
+                        if (self.inScrollMode) {
+                            if (str === '\x1b[A') {
+                                self.handleScrollKey('scrollUp');
+                                return true;
+                            }
+                            if (str === '\x1b[B') {
+                                self.handleScrollKey('scrollDown');
+                                return true;
+                            }
+                            if (str === '\x1b' || str === '\x1b\x1b') {
                                 self.exitScrollMode();
                                 return true;
                             }
-                            return originalEmit(event, ...args);
+                            return true;
                         }
-                        self.handleScrollKey(scrollKey);
-                        return true;
                     }
-                    if (self.inScrollMode) {
-                        return true;
+                    catch {
                     }
                 }
                 return originalEmit(event, ...args);
@@ -194,6 +214,9 @@ export class RekShell {
         }
     }
     handleScrollKey(key) {
+        if (!this.originalStdoutWrite) {
+            return;
+        }
         let needsRedraw = false;
         switch (key) {
             case 'pageUp':
@@ -250,11 +273,15 @@ export class RekShell {
     enterScrollMode() {
         if (this.inScrollMode)
             return;
+        if (!this.originalStdoutWrite)
+            return;
         this.inScrollMode = true;
-        this.rl.pause();
-        if (this.originalStdoutWrite) {
-            this.originalStdoutWrite('\x1b[?25l');
+        try {
+            this.rl.pause();
+        }
+        catch {
         }
+        this.originalStdoutWrite('\x1b[?25l');
         this.renderScrollView();
     }
     exitScrollMode() {
@@ -288,7 +315,7 @@ export class RekShell {
         const scrollInfo = this.scrollBuffer.isScrolledUp
             ? colors.yellow(`↑ ${this.scrollBuffer.position} lines | ${info.percent}% | `)
             : '';
-        const helpText = colors.gray('Page Up/Down • Home/End • Q to exit');
+        const helpText = colors.gray('↑↓/PgUp/PgDn • Home/End • Esc/Q to exit');
         const statusBar = `\x1b[${rows};1H\x1b[7m ${scrollInfo}${helpText} \x1b[0m`;
         this.originalStdoutWrite(statusBar);
     }
@@ -297,6 +324,21 @@ export class RekShell {
         this.rl.prompt();
     }
     async handleCommand(input) {
+        if (input.startsWith('@')) {
+            const spaceIdx = input.indexOf(' ');
+            if (spaceIdx > 1) {
+                const presetName = input.slice(1, spaceIdx).toLowerCase();
+                const message = input.slice(spaceIdx + 1).trim();
+                if (message) {
+                    await this.runAIPresetChat(presetName, message);
+                    return;
+                }
+            }
+            console.log(colors.yellow('Usage: @<preset> <message>'));
+            console.log(colors.gray('Example: @openai Hello, how are you?'));
+            console.log(colors.gray('Available AI presets: openai, anthropic, groq, google, xai, mistral, cohere'));
+            return;
+        }
         if (input.endsWith('?') && !input.startsWith('?') && input.length > 1) {
             await this.runSearch(input.slice(0, -1).trim());
             return;
@@ -312,6 +354,9 @@ export class RekShell {
             case 'clear':
                 console.clear();
                 return;
+            case 'ai:clear':
+                this.clearAIMemory(parts[1]);
+                return;
             case 'exit':
             case 'quit':
                 this.rl.close();
@@ -507,6 +552,96 @@ export class RekShell {
             await startAIChat(rl, provider, apiKey, model);
         });
     }
+    async runAIPresetChat(presetName, message) {
+        try {
+            let client = this.aiClients.get(presetName);
+            if (!client) {
+                const presetConfig = resolvePreset(presetName);
+                if (!presetConfig) {
+                    console.log(colors.red(`Unknown AI preset: @${presetName}`));
+                    console.log(colors.gray('Available AI presets: openai, anthropic, groq, google, xai, mistral, cohere, deepseek, fireworks, together, perplexity'));
+                    return;
+                }
+                if (!presetConfig._aiConfig) {
+                    console.log(colors.red(`Preset @${presetName} does not support AI features.`));
+                    console.log(colors.gray('Use an AI preset like @openai, @anthropic, @groq, etc.'));
+                    return;
+                }
+                client = createClient(presetConfig);
+                this.aiClients.set(presetName, client);
+            }
+            if (!client.hasAI) {
+                console.log(colors.red(`Preset @${presetName} does not have AI capabilities.`));
+                return;
+            }
+            const model = client._aiConfig?.model || presetName;
+            console.log(colors.gray(`\n${presetName} (${model}) is thinking...`));
+            const stream = await client.ai.chatStream(message);
+            process.stdout.write('\n');
+            for await (const event of stream) {
+                if (event.type === 'text') {
+                    process.stdout.write(colors.orange(event.content));
+                }
+                else if (event.type === 'error') {
+                    console.log(colors.red(`\nError: ${event.error}`));
+                }
+            }
+            const memory = client.ai.getMemory();
+            const pairs = Math.floor(memory.length / 2);
+            console.log(colors.reset(''));
+            console.log(colors.gray(`Memory: ${pairs}/12 pairs (${memory.length} messages)`));
+        }
+        catch (error) {
+            if (error.message?.includes('API key')) {
+                console.log(colors.red(`\nMissing API key for @${presetName}`));
+                const envVarMap = {
+                    openai: 'OPENAI_API_KEY',
+                    anthropic: 'ANTHROPIC_API_KEY',
+                    google: 'GOOGLE_API_KEY',
+                    groq: 'GROQ_API_KEY',
+                    xai: 'XAI_API_KEY',
+                    mistral: 'MISTRAL_API_KEY',
+                    cohere: 'COHERE_API_KEY',
+                    deepseek: 'DEEPSEEK_API_KEY',
+                    fireworks: 'FIREWORKS_API_KEY',
+                    together: 'TOGETHER_API_KEY',
+                    perplexity: 'PERPLEXITY_API_KEY',
+                };
+                const envVar = envVarMap[presetName] || `${presetName.toUpperCase()}_API_KEY`;
+                console.log(colors.gray(`Set ${envVar} environment variable to use this preset.`));
+            }
+            else {
+                console.log(colors.red(`\nError: ${error.message || error}`));
+            }
+        }
+    }
+    clearAIMemory(presetName) {
+        if (presetName) {
+            const client = this.aiClients.get(presetName);
+            if (client && client.hasAI) {
+                client.ai.clearMemory();
+                console.log(colors.green(`Cleared AI memory for @${presetName}`));
+            }
+            else {
+                console.log(colors.yellow(`No active AI session for @${presetName}`));
+            }
+        }
+        else {
+            let cleared = 0;
+            for (const [name, client] of this.aiClients) {
+                if (client.hasAI) {
+                    client.ai.clearMemory();
+                    cleared++;
+                }
+            }
+            if (cleared > 0) {
+                console.log(colors.green(`Cleared AI memory for ${cleared} preset(s)`));
+            }
+            else {
+                console.log(colors.yellow('No active AI sessions to clear'));
+            }
+        }
+    }
     async runLoadTest(args) {
         let targetUrl = '';
         let users = 50;
@@ -1494,7 +1629,7 @@ ${colors.bold('Network:')}
     }
     async runSpider(args) {
         let url = '';
-        let maxDepth = 3;
+        let maxDepth = 5;
         let maxPages = 100;
         let concurrency = 5;
         let seoEnabled = false;
@@ -1502,7 +1637,7 @@ ${colors.bold('Network:')}
         for (let i = 0; i < args.length; i++) {
             const arg = args[i];
             if (arg.startsWith('depth=')) {
-                maxDepth = parseInt(arg.split('=')[1]) || 4;
+                maxDepth = parseInt(arg.split('=')[1]) || 5;
             }
             else if (arg.startsWith('limit=')) {
                 maxPages = parseInt(arg.split('=')[1]) || 100;
@@ -1524,7 +1659,7 @@ ${colors.bold('Network:')}
             if (!this.baseUrl) {
                 console.log(colors.yellow('Usage: spider <url> [options]'));
                 console.log(colors.gray('  Options:'));
-                console.log(colors.gray('    depth=4           Max crawl depth'));
+                console.log(colors.gray('    depth=5           Max crawl depth'));
                 console.log(colors.gray('    limit=100         Max pages to crawl'));
                 console.log(colors.gray('    concurrency=5     Concurrent requests'));
                 console.log(colors.gray('    seo               Enable SEO analysis'));
@@ -2656,6 +2791,17 @@ ${colors.bold('Network:')}
     ${colors.green('ws <url>')}            Start interactive WebSocket session.
     ${colors.green('udp <url>')}           Send UDP packet.
 
+  ${colors.bold('AI Chat:')}
+    ${colors.green('@openai <message>')}   Chat with OpenAI (GPT) with memory.
+    ${colors.green('@anthropic <msg>')}    Chat with Anthropic (Claude) with memory.
+    ${colors.green('@groq <message>')}     Chat with Groq (fast inference).
+    ${colors.green('@google <message>')}   Chat with Google (Gemini).
+    ${colors.green('@xai <message>')}      Chat with xAI (Grok).
+    ${colors.green('@mistral <message>')}  Chat with Mistral AI.
+                             ${colors.gray('Memory:')} ${colors.white('12 pairs (24 messages)')} preserved per preset.
+                             ${colors.gray('Env:')} Set ${colors.white('OPENAI_API_KEY')}, ${colors.white('ANTHROPIC_API_KEY')}, etc.
+    ${colors.green('ai:clear [preset]')}   Clear AI memory (all or specific preset).
+
   ${colors.bold('Network Tools:')}
     ${colors.green('whois <domain>')}      WHOIS lookup (domain or IP).
     ${colors.green('tls <host> [port]')}   Inspect TLS/SSL certificate.
@@ -2687,7 +2833,7 @@ ${colors.bold('Network:')}
   ${colors.bold('Web Crawler:')}
     ${colors.green('spider <url>')}        Crawl website following internal links.
                              ${colors.gray('Options:')}
-                             ${colors.white('--depth=4')}     ${colors.gray('Maximum depth to crawl')}
+                             ${colors.white('--depth=5')}     ${colors.gray('Maximum depth to crawl')}
                              ${colors.white('--limit=100')}   ${colors.gray('Maximum pages to crawl')}
                              ${colors.white('--concurrency=5')} ${colors.gray('Parallel requests')}
 
@@ -2707,7 +2853,8 @@ ${colors.bold('Network:')}
     › get /json
     › post /post name="Neo" active:=true role:Admin
     › load /heavy-endpoint users=100 mode=stress
-    › chat openai gpt-5.1
+    › @openai What is the capital of France?
+    › @anthropic Explain quantum computing
     › spider example.com depth=2 limit=50
     `);
     }

package/dist/core/client.d.ts

@@ -1,4 +1,5 @@
 import { ClientOptions, Middleware, ReckerRequest, ReckerResponse, RequestOptions, CacheStorage, PageResult } from '../types/index.js';
+import type { ClientAI, ClientOptionsWithAI } from '../types/ai-client.js';
 import { RequestPromise } from './request-promise.js';
 import { PaginationOptions } from '../plugins/pagination.js';
 import { RetryOptions } from '../plugins/retry.js';
@@ -36,7 +37,9 @@ export declare class Client {
     private cookieJar?;
     private cookieIgnoreInvalid;
     private defaultTimeout?;
-    constructor(options?: ExtendedClientOptions);
+    private _aiConfig?;
+    private _ai?;
+    constructor(options?: ExtendedClientOptions & Partial<ClientOptionsWithAI>);
     private createLoggingMiddleware;
     private createMaxSizeMiddleware;
     private setupCookieJar;
@@ -113,6 +116,8 @@ export declare class Client {
     whois(query: string, options?: WhoisOptions): Promise<WhoisResult>;
     isDomainAvailable(domain: string, options?: WhoisOptions): Promise<boolean>;
     hls(manifestUrl: string, options?: HlsOptions): HlsPromise;
+    get ai(): ClientAI;
+    get hasAI(): boolean;
 }
 export declare function createClient(options?: ExtendedClientOptions): Client;
 export {};

package/dist/core/client.js

@@ -1,4 +1,5 @@
 import { consoleLogger } from '../types/index.js';
+import { ClientAIImpl } from '../ai/client-ai.js';
 import { HttpRequest } from './request.js';
 import { UndiciTransport } from '../transport/undici.js';
 import { RequestPromise } from './request-promise.js';
@@ -41,6 +42,8 @@ export class Client {
     cookieJar;
     cookieIgnoreInvalid = false;
     defaultTimeout;
+    _aiConfig;
+    _ai;
     constructor(options = {}) {
         this.baseUrl = options.baseUrl || '';
         this.middlewares = options.middlewares || [];
@@ -150,6 +153,9 @@ export class Client {
         if (options.cookies) {
             this.setupCookieJar(options.cookies);
         }
+        if (options._aiConfig) {
+            this._aiConfig = options._aiConfig;
+        }
         if (this.maxResponseSize !== undefined) {
             this.middlewares.push(this.createMaxSizeMiddleware(this.maxResponseSize));
         }
@@ -661,6 +667,18 @@ export class Client {
     hls(manifestUrl, options = {}) {
         return new HlsPromise(this, manifestUrl, options);
     }
+    get ai() {
+        if (!this._ai) {
+            if (!this._aiConfig) {
+                throw new ConfigurationError('AI features require an AI-enabled preset. Use createClient(openai({...})), createClient(anthropic({...})), etc.', { configKey: '_aiConfig' });
+            }
+            this._ai = new ClientAIImpl(this, this._aiConfig);
+        }
+        return this._ai;
+    }
+    get hasAI() {
+        return this._aiConfig !== undefined;
+    }
 }
 export function createClient(options = {}) {
     return new Client(options);

package/dist/mcp/server.js

@@ -8,6 +8,9 @@ import { createHybridSearch } from './search/index.js';
 import { UnsupportedError } from '../core/errors.js';
 import { getIpInfo, isValidIP, isGeoIPAvailable, isBogon, isIPv6 } from './ip-intel.js';
 import { networkTools, networkToolHandlers } from './tools/network.js';
+import { seoTools, seoToolHandlers } from './tools/seo.js';
+import { scrapeTools, scrapeToolHandlers } from './tools/scrape.js';
+import { securityTools, securityToolHandlers } from './tools/security.js';
 import { ToolRegistry } from './tools/registry.js';
 import { loadToolModules } from './tools/loader.js';
 export class MCPServer {
@@ -45,6 +48,18 @@ export class MCPServer {
             tools: networkTools,
             handlers: networkToolHandlers
         });
+        this.toolRegistry.registerModule({
+            tools: seoTools,
+            handlers: seoToolHandlers
+        });
+        this.toolRegistry.registerModule({
+            tools: scrapeTools,
+            handlers: scrapeToolHandlers
+        });
+        this.toolRegistry.registerModule({
+            tools: securityTools,
+            handlers: securityToolHandlers
+        });
     }
     indexReady = null;
     async ensureIndexReady() {

package/dist/mcp/tools/scrape.d.ts

@@ -0,0 +1,3 @@
+import type { MCPTool, MCPToolResult } from '../types.js';
+export declare const scrapeTools: MCPTool[];
+export declare const scrapeToolHandlers: Record<string, (args: Record<string, unknown>) => Promise<MCPToolResult>>;

package/dist/mcp/tools/scrape.js

@@ -0,0 +1,156 @@
+import { createClient } from '../../core/client.js';
+import { ScrapeDocument } from '../../scrape/document.js';
+async function scrapeUrl(args) {
+    const url = String(args.url || '');
+    const selectors = args.selectors;
+    const extract = args.extract;
+    const selector = args.selector;
+    if (!url) {
+        return {
+            content: [{ type: 'text', text: 'Error: url is required' }],
+            isError: true,
+        };
+    }
+    try {
+        const client = createClient({ timeout: 30000 });
+        const response = await client.get(url);
+        const html = await response.text();
+        const doc = await ScrapeDocument.create(html, { baseUrl: url });
+        const output = {
+            url,
+            title: doc.title(),
+        };
+        if (selector) {
+            const elements = doc.selectAll(selector);
+            output.results = elements.map(el => ({
+                text: el.text(),
+                html: el.html(),
+                tag: el.tagName(),
+                attrs: el.attrs(),
+            }));
+            output.count = elements.length;
+        }
+        if (selectors && Object.keys(selectors).length > 0) {
+            const extracted = {};
+            for (const [key, sel] of Object.entries(selectors)) {
+                const isMultiple = sel.endsWith('[]');
+                const actualSel = isMultiple ? sel.slice(0, -2) : sel;
+                if (isMultiple) {
+                    extracted[key] = doc.texts(actualSel);
+                }
+                else {
+                    extracted[key] = doc.text(actualSel);
+                }
+            }
+            output.data = extracted;
+        }
+        const extractSet = new Set(extract || []);
+        if (extractSet.has('links') || extractSet.has('all')) {
+            const links = doc.links({ absolute: true });
+            output.links = links.slice(0, 100).map(l => ({
+                href: l.href,
+                text: l.text?.slice(0, 100),
+                rel: l.rel,
+            }));
+            output.linkCount = links.length;
+        }
+        if (extractSet.has('images') || extractSet.has('all')) {
+            const images = doc.images({ absolute: true });
+            output.images = images.slice(0, 50).map(img => ({
+                src: img.src,
+                alt: img.alt,
+                width: img.width,
+                height: img.height,
+            }));
+            output.imageCount = images.length;
+        }
+        if (extractSet.has('meta') || extractSet.has('all')) {
+            output.meta = doc.meta();
+        }
+        if (extractSet.has('og') || extractSet.has('all')) {
+            output.openGraph = doc.openGraph();
+        }
+        if (extractSet.has('twitter') || extractSet.has('all')) {
+            output.twitterCard = doc.twitterCard();
+        }
+        if (extractSet.has('jsonld') || extractSet.has('all')) {
+            output.jsonLd = doc.jsonLd();
+        }
+        if (extractSet.has('tables') || extractSet.has('all')) {
+            const tables = doc.tables();
+            output.tables = tables.slice(0, 10).map(t => ({
+                headers: t.headers,
+                rows: t.rows.slice(0, 50),
+            }));
+            output.tableCount = tables.length;
+        }
+        if (extractSet.has('forms') || extractSet.has('all')) {
+            output.forms = doc.forms();
+        }
+        if (extractSet.has('headings')) {
+            output.headings = {
+                h1: doc.texts('h1'),
+                h2: doc.texts('h2'),
+                h3: doc.texts('h3'),
+            };
+        }
+        return {
+            content: [{
+                    type: 'text',
+                    text: JSON.stringify(output, null, 2),
+                }],
+        };
+    }
+    catch (error) {
+        return {
+            content: [{
+                    type: 'text',
+                    text: `Scrape failed: ${error.message}`,
+                }],
+            isError: true,
+        };
+    }
+}
+export const scrapeTools = [
+    {
+        name: 'rek_scrape',
+        description: `Scrape a web page and extract data using CSS selectors.
+
+Supports multiple extraction modes:
+- Single selector: Extract elements matching one CSS selector
+- Selector map: Extract multiple fields at once
+- Built-in extractors: links, images, meta, og, twitter, jsonld, tables, forms, headings
+
+Examples:
+- Get all product titles: selector=".product-title"
+- Extract multiple fields: selectors={"title":"h1","price":".price","desc":".description"}
+- Get all links and images: extract=["links","images"]
+- Full extraction: extract=["all"]`,
+        inputSchema: {
+            type: 'object',
+            properties: {
+                url: {
+                    type: 'string',
+                    description: 'URL to scrape',
+                },
+                selector: {
+                    type: 'string',
+                    description: 'Single CSS selector to extract elements (e.g., ".product-card", "article h2")',
+                },
+                selectors: {
+                    type: 'object',
+                    description: 'Map of field names to CSS selectors. Add [] suffix for multiple values (e.g., {"title":"h1","links[]":"a"})',
+                },
+                extract: {
+                    type: 'array',
+                    items: { type: 'string' },
+                    description: 'Built-in extractors to run: links, images, meta, og, twitter, jsonld, tables, forms, headings, all',
+                },
+            },
+            required: ['url'],
+        },
+    },
+];
+export const scrapeToolHandlers = {
+    rek_scrape: scrapeUrl,
+};

package/dist/mcp/tools/security.d.ts

@@ -0,0 +1,3 @@
+import type { MCPTool, MCPToolResult } from '../types.js';
+export declare const securityTools: MCPTool[];
+export declare const securityToolHandlers: Record<string, (args: Record<string, unknown>) => Promise<MCPToolResult>>;