recker 1.0.29-next.7cc1d8b → 1.0.29-next.cf0cafb

package/dist/ai/memory.d.ts

@@ -0,0 +1,35 @@
+import type { ChatMessage } from '../types/ai.js';
+import type { AIMemoryConfig } from '../types/ai-client.js';
+export declare class ConversationMemory {
+    private config;
+    private systemMessage;
+    private messages;
+    constructor(config?: AIMemoryConfig);
+    setSystemPrompt(prompt: string): void;
+    getSystemPrompt(): string;
+    addUserMessage(content: string): void;
+    addAssistantMessage(content: string): void;
+    addMessage(message: ChatMessage): void;
+    buildMessages(userPrompt: string): ChatMessage[];
+    recordResponse(content: string): void;
+    getMessages(): ChatMessage[];
+    getConversation(): readonly ChatMessage[];
+    getPairCount(): number;
+    clear(): void;
+    reset(): void;
+    setConfig(config: Partial<AIMemoryConfig>): void;
+    getConfig(): AIMemoryConfig;
+    private prune;
+    isEmpty(): boolean;
+    getMessageCount(): number;
+    toJSON(): {
+        config: AIMemoryConfig;
+        systemPrompt: string | null;
+        messages: ChatMessage[];
+    };
+    static fromJSON(data: {
+        config?: AIMemoryConfig;
+        systemPrompt?: string | null;
+        messages?: ChatMessage[];
+    }): ConversationMemory;
+}

package/dist/ai/memory.js

@@ -0,0 +1,136 @@
+const DEFAULT_MAX_PAIRS = 12;
+export class ConversationMemory {
+    config;
+    systemMessage = null;
+    messages = [];
+    constructor(config = {}) {
+        this.config = {
+            maxPairs: config.maxPairs ?? DEFAULT_MAX_PAIRS,
+            systemPrompt: config.systemPrompt ?? '',
+        };
+        if (this.config.systemPrompt) {
+            this.systemMessage = {
+                role: 'system',
+                content: this.config.systemPrompt,
+            };
+        }
+    }
+    setSystemPrompt(prompt) {
+        this.config.systemPrompt = prompt;
+        if (prompt) {
+            this.systemMessage = {
+                role: 'system',
+                content: prompt,
+            };
+        }
+        else {
+            this.systemMessage = null;
+        }
+    }
+    getSystemPrompt() {
+        return this.config.systemPrompt;
+    }
+    addUserMessage(content) {
+        this.messages.push({
+            role: 'user',
+            content,
+        });
+        this.prune();
+    }
+    addAssistantMessage(content) {
+        this.messages.push({
+            role: 'assistant',
+            content,
+        });
+        this.prune();
+    }
+    addMessage(message) {
+        if (message.role === 'system') {
+            this.setSystemPrompt(typeof message.content === 'string' ? message.content : '');
+            return;
+        }
+        this.messages.push(message);
+        this.prune();
+    }
+    buildMessages(userPrompt) {
+        this.addUserMessage(userPrompt);
+        return this.getMessages();
+    }
+    recordResponse(content) {
+        this.addAssistantMessage(content);
+    }
+    getMessages() {
+        const result = [];
+        if (this.systemMessage) {
+            result.push(this.systemMessage);
+        }
+        result.push(...this.messages);
+        return result;
+    }
+    getConversation() {
+        return this.messages;
+    }
+    getPairCount() {
+        let pairs = 0;
+        for (let i = 0; i < this.messages.length - 1; i += 2) {
+            if (this.messages[i].role === 'user' &&
+                this.messages[i + 1]?.role === 'assistant') {
+                pairs++;
+            }
+        }
+        return pairs;
+    }
+    clear() {
+        this.messages = [];
+    }
+    reset() {
+        this.messages = [];
+        this.systemMessage = null;
+        this.config.systemPrompt = '';
+    }
+    setConfig(config) {
+        if (config.maxPairs !== undefined) {
+            this.config.maxPairs = config.maxPairs;
+            this.prune();
+        }
+        if (config.systemPrompt !== undefined) {
+            this.setSystemPrompt(config.systemPrompt);
+        }
+    }
+    getConfig() {
+        return { ...this.config };
+    }
+    prune() {
+        const maxMessages = this.config.maxPairs * 2;
+        if (this.messages.length > maxMessages) {
+            const excess = this.messages.length - maxMessages;
+            const toRemove = Math.ceil(excess / 2) * 2;
+            this.messages = this.messages.slice(toRemove);
+        }
+    }
+    isEmpty() {
+        return this.messages.length === 0;
+    }
+    getMessageCount() {
+        return this.messages.length;
+    }
+    toJSON() {
+        return {
+            config: this.config,
+            systemPrompt: this.systemMessage?.content,
+            messages: [...this.messages],
+        };
+    }
+    static fromJSON(data) {
+        const memory = new ConversationMemory(data.config);
+        if (data.systemPrompt) {
+            memory.setSystemPrompt(data.systemPrompt);
+        }
+        if (data.messages) {
+            for (const msg of data.messages) {
+                memory.addMessage(msg);
+            }
+        }
+        return memory;
+    }
+}

package/dist/browser/types/ai-client.d.ts

@@ -0,0 +1,29 @@
+import type { AIResponse, AIStream, ChatMessage, AIProvider } from './ai.js';
+export interface AIMemoryConfig {
+    maxPairs?: number;
+    systemPrompt?: string;
+}
+export interface PresetAIConfig {
+    provider: AIProvider;
+    apiKey?: string;
+    model: string;
+    baseUrl?: string;
+    memory?: AIMemoryConfig;
+    organization?: string;
+    headers?: Record<string, string>;
+}
+export interface ClientAI {
+    chat(prompt: string): Promise<AIResponse>;
+    chatStream(prompt: string): Promise<AIStream>;
+    prompt(prompt: string): Promise<AIResponse>;
+    promptStream(prompt: string): Promise<AIStream>;
+    clearMemory(): void;
+    getMemory(): readonly ChatMessage[];
+    setMemoryConfig(config: Partial<AIMemoryConfig>): void;
+    getMemoryConfig(): AIMemoryConfig;
+    readonly provider: AIProvider;
+    readonly model: string;
+}
+export interface ClientOptionsWithAI {
+    _aiConfig?: PresetAIConfig;
+}

package/dist/browser/types/ai-client.js

@@ -0,0 +1 @@
+export {};

package/dist/cli/tui/scroll-buffer.js

@@ -119,17 +119,17 @@ export class ScrollBuffer extends EventEmitter {
 }
 export function parseScrollKey(data) {
     const str = data.toString();
-    if (str === '\x1b[5~' || str === '\x1bOy')
+    if (str === '\x1b[5~' || str === '\x1bOy' || str === '\x1b[5;5~' || str === '\x1b[5;2~')
         return 'pageUp';
-    if (str === '\x1b[6~' || str === '\x1bOs')
+    if (str === '\x1b[6~' || str === '\x1bOs' || str === '\x1b[6;5~' || str === '\x1b[6;2~')
         return 'pageDown';
     if (str === '\x1b[1;2A')
         return 'scrollUp';
     if (str === '\x1b[1;2B')
         return 'scrollDown';
-    if (str === '\x1b[H' || str === '\x1b[1~' || str === '\x1bOH')
+    if (str === '\x1b[H' || str === '\x1b[1~' || str === '\x1bOH' || str === '\x1b[7~')
         return 'home';
-    if (str === '\x1b[F' || str === '\x1b[4~' || str === '\x1bOF')
+    if (str === '\x1b[F' || str === '\x1b[4~' || str === '\x1bOF' || str === '\x1b[8~')
         return 'end';
     if (str === 'q' || str === 'Q')
         return 'quit';

package/dist/cli/tui/shell.js

@@ -173,20 +173,36 @@ export class RekShell {
                         }
                         return true;
                     }
-                    const scrollKey = parseScrollKey(data);
-                    if (scrollKey) {
-                        if (scrollKey === 'quit') {
-                            if (self.inScrollMode) {
+                    try {
+                        const scrollKey = parseScrollKey(data);
+                        if (scrollKey) {
+                            if (scrollKey === 'quit') {
+                                if (self.inScrollMode) {
+                                    self.exitScrollMode();
+                                    return true;
+                                }
+                                return originalEmit(event, ...args);
+                            }
+                            self.handleScrollKey(scrollKey);
+                            return true;
+                        }
+                        if (self.inScrollMode) {
+                            if (str === '\x1b[A') {
+                                self.handleScrollKey('scrollUp');
+                                return true;
+                            }
+                            if (str === '\x1b[B') {
+                                self.handleScrollKey('scrollDown');
+                                return true;
+                            }
+                            if (str === '\x1b' || str === '\x1b\x1b') {
                                 self.exitScrollMode();
                                 return true;
                             }
-                            return originalEmit(event, ...args);
+                            return true;
                         }
-                        self.handleScrollKey(scrollKey);
-                        return true;
                     }
-                    if (self.inScrollMode) {
-                        return true;
+                    catch {
                     }
                 }
                 return originalEmit(event, ...args);
@@ -194,6 +210,9 @@ export class RekShell {
         }
     }
     handleScrollKey(key) {
+        if (!this.originalStdoutWrite) {
+            return;
+        }
         let needsRedraw = false;
         switch (key) {
             case 'pageUp':
@@ -250,11 +269,15 @@ export class RekShell {
     enterScrollMode() {
         if (this.inScrollMode)
             return;
+        if (!this.originalStdoutWrite)
+            return;
         this.inScrollMode = true;
-        this.rl.pause();
-        if (this.originalStdoutWrite) {
-            this.originalStdoutWrite('\x1b[?25l');
+        try {
+            this.rl.pause();
+        }
+        catch {
         }
+        this.originalStdoutWrite('\x1b[?25l');
         this.renderScrollView();
     }
     exitScrollMode() {
@@ -288,7 +311,7 @@ export class RekShell {
         const scrollInfo = this.scrollBuffer.isScrolledUp
             ? colors.yellow(`↑ ${this.scrollBuffer.position} lines | ${info.percent}% | `)
             : '';
-        const helpText = colors.gray('Page Up/Down • Home/End • Q to exit');
+        const helpText = colors.gray('↑↓/PgUp/PgDn • Home/End • Esc/Q to exit');
         const statusBar = `\x1b[${rows};1H\x1b[7m ${scrollInfo}${helpText} \x1b[0m`;
         this.originalStdoutWrite(statusBar);
     }
@@ -1494,7 +1517,7 @@ ${colors.bold('Network:')}
     }
     async runSpider(args) {
         let url = '';
-        let maxDepth = 3;
+        let maxDepth = 5;
         let maxPages = 100;
         let concurrency = 5;
         let seoEnabled = false;
@@ -1502,7 +1525,7 @@ ${colors.bold('Network:')}
         for (let i = 0; i < args.length; i++) {
             const arg = args[i];
             if (arg.startsWith('depth=')) {
-                maxDepth = parseInt(arg.split('=')[1]) || 4;
+                maxDepth = parseInt(arg.split('=')[1]) || 5;
             }
             else if (arg.startsWith('limit=')) {
                 maxPages = parseInt(arg.split('=')[1]) || 100;
@@ -1524,7 +1547,7 @@ ${colors.bold('Network:')}
             if (!this.baseUrl) {
                 console.log(colors.yellow('Usage: spider <url> [options]'));
                 console.log(colors.gray('  Options:'));
-                console.log(colors.gray('    depth=4           Max crawl depth'));
+                console.log(colors.gray('    depth=5           Max crawl depth'));
                 console.log(colors.gray('    limit=100         Max pages to crawl'));
                 console.log(colors.gray('    concurrency=5     Concurrent requests'));
                 console.log(colors.gray('    seo               Enable SEO analysis'));
@@ -2687,7 +2710,7 @@ ${colors.bold('Network:')}
   ${colors.bold('Web Crawler:')}
     ${colors.green('spider <url>')}        Crawl website following internal links.
                              ${colors.gray('Options:')}
-                             ${colors.white('--depth=4')}     ${colors.gray('Maximum depth to crawl')}
+                             ${colors.white('--depth=5')}     ${colors.gray('Maximum depth to crawl')}
                              ${colors.white('--limit=100')}   ${colors.gray('Maximum pages to crawl')}
                              ${colors.white('--concurrency=5')} ${colors.gray('Parallel requests')}
 

package/dist/mcp/server.js

@@ -8,6 +8,9 @@ import { createHybridSearch } from './search/index.js';
 import { UnsupportedError } from '../core/errors.js';
 import { getIpInfo, isValidIP, isGeoIPAvailable, isBogon, isIPv6 } from './ip-intel.js';
 import { networkTools, networkToolHandlers } from './tools/network.js';
+import { seoTools, seoToolHandlers } from './tools/seo.js';
+import { scrapeTools, scrapeToolHandlers } from './tools/scrape.js';
+import { securityTools, securityToolHandlers } from './tools/security.js';
 import { ToolRegistry } from './tools/registry.js';
 import { loadToolModules } from './tools/loader.js';
 export class MCPServer {
@@ -45,6 +48,18 @@ export class MCPServer {
             tools: networkTools,
             handlers: networkToolHandlers
         });
+        this.toolRegistry.registerModule({
+            tools: seoTools,
+            handlers: seoToolHandlers
+        });
+        this.toolRegistry.registerModule({
+            tools: scrapeTools,
+            handlers: scrapeToolHandlers
+        });
+        this.toolRegistry.registerModule({
+            tools: securityTools,
+            handlers: securityToolHandlers
+        });
     }
     indexReady = null;
     async ensureIndexReady() {

package/dist/mcp/tools/scrape.d.ts

@@ -0,0 +1,3 @@
+import type { MCPTool, MCPToolResult } from '../types.js';
+export declare const scrapeTools: MCPTool[];
+export declare const scrapeToolHandlers: Record<string, (args: Record<string, unknown>) => Promise<MCPToolResult>>;

package/dist/mcp/tools/scrape.js

@@ -0,0 +1,156 @@
+import { createClient } from '../../core/client.js';
+import { ScrapeDocument } from '../../scrape/document.js';
+async function scrapeUrl(args) {
+    const url = String(args.url || '');
+    const selectors = args.selectors;
+    const extract = args.extract;
+    const selector = args.selector;
+    if (!url) {
+        return {
+            content: [{ type: 'text', text: 'Error: url is required' }],
+            isError: true,
+        };
+    }
+    try {
+        const client = createClient({ timeout: 30000 });
+        const response = await client.get(url);
+        const html = await response.text();
+        const doc = await ScrapeDocument.create(html, { baseUrl: url });
+        const output = {
+            url,
+            title: doc.title(),
+        };
+        if (selector) {
+            const elements = doc.selectAll(selector);
+            output.results = elements.map(el => ({
+                text: el.text(),
+                html: el.html(),
+                tag: el.tagName(),
+                attrs: el.attrs(),
+            }));
+            output.count = elements.length;
+        }
+        if (selectors && Object.keys(selectors).length > 0) {
+            const extracted = {};
+            for (const [key, sel] of Object.entries(selectors)) {
+                const isMultiple = sel.endsWith('[]');
+                const actualSel = isMultiple ? sel.slice(0, -2) : sel;
+                if (isMultiple) {
+                    extracted[key] = doc.texts(actualSel);
+                }
+                else {
+                    extracted[key] = doc.text(actualSel);
+                }
+            }
+            output.data = extracted;
+        }
+        const extractSet = new Set(extract || []);
+        if (extractSet.has('links') || extractSet.has('all')) {
+            const links = doc.links({ absolute: true });
+            output.links = links.slice(0, 100).map(l => ({
+                href: l.href,
+                text: l.text?.slice(0, 100),
+                rel: l.rel,
+            }));
+            output.linkCount = links.length;
+        }
+        if (extractSet.has('images') || extractSet.has('all')) {
+            const images = doc.images({ absolute: true });
+            output.images = images.slice(0, 50).map(img => ({
+                src: img.src,
+                alt: img.alt,
+                width: img.width,
+                height: img.height,
+            }));
+            output.imageCount = images.length;
+        }
+        if (extractSet.has('meta') || extractSet.has('all')) {
+            output.meta = doc.meta();
+        }
+        if (extractSet.has('og') || extractSet.has('all')) {
+            output.openGraph = doc.openGraph();
+        }
+        if (extractSet.has('twitter') || extractSet.has('all')) {
+            output.twitterCard = doc.twitterCard();
+        }
+        if (extractSet.has('jsonld') || extractSet.has('all')) {
+            output.jsonLd = doc.jsonLd();
+        }
+        if (extractSet.has('tables') || extractSet.has('all')) {
+            const tables = doc.tables();
+            output.tables = tables.slice(0, 10).map(t => ({
+                headers: t.headers,
+                rows: t.rows.slice(0, 50),
+            }));
+            output.tableCount = tables.length;
+        }
+        if (extractSet.has('forms') || extractSet.has('all')) {
+            output.forms = doc.forms();
+        }
+        if (extractSet.has('headings')) {
+            output.headings = {
+                h1: doc.texts('h1'),
+                h2: doc.texts('h2'),
+                h3: doc.texts('h3'),
+            };
+        }
+        return {
+            content: [{
+                    type: 'text',
+                    text: JSON.stringify(output, null, 2),
+                }],
+        };
+    }
+    catch (error) {
+        return {
+            content: [{
+                    type: 'text',
+                    text: `Scrape failed: ${error.message}`,
+                }],
+            isError: true,
+        };
+    }
+}
+export const scrapeTools = [
+    {
+        name: 'rek_scrape',
+        description: `Scrape a web page and extract data using CSS selectors.
+
+Supports multiple extraction modes:
+- Single selector: Extract elements matching one CSS selector
+- Selector map: Extract multiple fields at once
+- Built-in extractors: links, images, meta, og, twitter, jsonld, tables, forms, headings
+
+Examples:
+- Get all product titles: selector=".product-title"
+- Extract multiple fields: selectors={"title":"h1","price":".price","desc":".description"}
+- Get all links and images: extract=["links","images"]
+- Full extraction: extract=["all"]`,
+        inputSchema: {
+            type: 'object',
+            properties: {
+                url: {
+                    type: 'string',
+                    description: 'URL to scrape',
+                },
+                selector: {
+                    type: 'string',
+                    description: 'Single CSS selector to extract elements (e.g., ".product-card", "article h2")',
+                },
+                selectors: {
+                    type: 'object',
+                    description: 'Map of field names to CSS selectors. Add [] suffix for multiple values (e.g., {"title":"h1","links[]":"a"})',
+                },
+                extract: {
+                    type: 'array',
+                    items: { type: 'string' },
+                    description: 'Built-in extractors to run: links, images, meta, og, twitter, jsonld, tables, forms, headings, all',
+                },
+            },
+            required: ['url'],
+        },
+    },
+];
+export const scrapeToolHandlers = {
+    rek_scrape: scrapeUrl,
+};

package/dist/mcp/tools/security.d.ts

@@ -0,0 +1,3 @@
+import type { MCPTool, MCPToolResult } from '../types.js';
+export declare const securityTools: MCPTool[];
+export declare const securityToolHandlers: Record<string, (args: Record<string, unknown>) => Promise<MCPToolResult>>;