recker 1.0.29 → 1.0.31

package/dist/browser/types/ai-client.d.ts

@@ -0,0 +1,32 @@
+import type { AIResponse, AIStream, ChatMessage, AIProvider } from './ai.js';
+export interface AIMemoryConfig {
+    maxPairs?: number;
+    systemPrompt?: string;
+}
+export interface PresetAIConfig {
+    provider: AIProvider;
+    apiKey?: string;
+    model: string;
+    baseUrl?: string;
+    memory?: AIMemoryConfig;
+    organization?: string;
+    headers?: Record<string, string>;
+    resourceName?: string;
+    deploymentName?: string;
+    apiVersion?: string;
+}
+export interface ClientAI {
+    chat(prompt: string): Promise<AIResponse>;
+    chatStream(prompt: string): Promise<AIStream>;
+    prompt(prompt: string): Promise<AIResponse>;
+    promptStream(prompt: string): Promise<AIStream>;
+    clearMemory(): void;
+    getMemory(): readonly ChatMessage[];
+    setMemoryConfig(config: Partial<AIMemoryConfig>): void;
+    getMemoryConfig(): AIMemoryConfig;
+    readonly provider: AIProvider;
+    readonly model: string;
+}
+export interface ClientOptionsWithAI {
+    _aiConfig?: PresetAIConfig;
+}

package/dist/browser/types/ai-client.js

@@ -0,0 +1 @@
+export {};

package/dist/browser/types/ai.d.ts

@@ -1,4 +1,4 @@
-export type AIProvider = 'openai' | 'anthropic' | 'google' | 'replicate' | 'huggingface' | 'ollama' | 'custom';
+export type AIProvider = 'openai' | 'anthropic' | 'google' | 'replicate' | 'huggingface' | 'ollama' | 'groq' | 'mistral' | 'cohere' | 'together' | 'perplexity' | 'deepseek' | 'fireworks' | 'xai' | 'azure-openai' | 'cloudflare-workers-ai' | 'custom';
 export type MessageRole = 'system' | 'user' | 'assistant' | 'tool';
 export interface ChatMessage {
     role: MessageRole;

package/dist/cli/index.js

@@ -330,7 +330,17 @@ complete -F _rek_completions rek
         .alias('interactive')
         .alias('repl')
         .description('Start the interactive Rek Shell')
-        .action(async () => {
+        .option('-e, --env [path]', 'Load .env file (auto-loads from cwd by default)')
+        .action(async (options) => {
+        if (options.env !== false) {
+            try {
+                const envPath = typeof options.env === 'string' ? options.env : join(process.cwd(), '.env');
+                await fs.access(envPath);
+                await loadEnvFile(options.env);
+            }
+            catch {
+            }
+        }
         const { RekShell } = await import('./tui/shell.js');
         const shell = new RekShell();
         shell.start();
@@ -520,6 +530,397 @@ ${colors.gray('Grade:')} ${gradeColor(colors.bold(report.grade))}  ${colors.gray
             process.exit(1);
         }
     });
+    program
+        .command('spider')
+        .description('Crawl a website following internal links')
+        .argument('<url>', 'Starting URL to crawl')
+        .argument('[args...]', 'Options: depth=N limit=N concurrency=N seo output=file.json')
+        .addHelpText('after', `
+${colors.bold(colors.yellow('Examples:'))}
+  ${colors.green('$ rek spider example.com')}                           ${colors.gray('Crawl with defaults')}
+  ${colors.green('$ rek spider example.com depth=3 limit=50')}          ${colors.gray('Depth 3, max 50 pages')}
+  ${colors.green('$ rek spider example.com seo')}                       ${colors.gray('Crawl + SEO analysis')}
+  ${colors.green('$ rek spider example.com seo output=report.json')}    ${colors.gray('SEO with JSON export')}
+
+${colors.bold(colors.yellow('Options:'))}
+  ${colors.cyan('depth=N')}           Max link depth to follow (default: 5)
+  ${colors.cyan('limit=N')}           Max pages to crawl (default: 100)
+  ${colors.cyan('concurrency=N')}     Parallel requests (default: 5)
+  ${colors.cyan('seo')}               Enable SEO analysis mode
+  ${colors.cyan('output=file.json')}  Save JSON report to file
+`)
+        .action(async (url, args) => {
+        let maxDepth = 5;
+        let maxPages = 100;
+        let concurrency = 5;
+        let seoEnabled = false;
+        let outputFile = '';
+        for (const arg of args) {
+            if (arg.startsWith('depth=')) {
+                maxDepth = parseInt(arg.split('=')[1]) || 5;
+            }
+            else if (arg.startsWith('limit=')) {
+                maxPages = parseInt(arg.split('=')[1]) || 100;
+            }
+            else if (arg.startsWith('concurrency=')) {
+                concurrency = parseInt(arg.split('=')[1]) || 5;
+            }
+            else if (arg === 'seo') {
+                seoEnabled = true;
+            }
+            else if (arg.startsWith('output=')) {
+                outputFile = arg.split('=')[1] || '';
+            }
+        }
+        if (!url.startsWith('http'))
+            url = `https://${url}`;
+        const modeLabel = seoEnabled ? colors.magenta(' + SEO') : '';
+        console.log(colors.cyan(`\nSpider starting: ${url}`));
+        console.log(colors.gray(`  Depth: ${maxDepth} | Limit: ${maxPages} | Concurrency: ${concurrency}${modeLabel}`));
+        if (outputFile) {
+            console.log(colors.gray(`  Output: ${outputFile}`));
+        }
+        console.log('');
+        try {
+            if (seoEnabled) {
+                const { SeoSpider } = await import('../seo/index.js');
+                const seoSpider = new SeoSpider({
+                    maxDepth,
+                    maxPages,
+                    concurrency,
+                    sameDomain: true,
+                    delay: 100,
+                    seo: true,
+                    output: outputFile || undefined,
+                    onProgress: (progress) => {
+                        process.stdout.write(`\r${colors.gray('  Crawling:')} ${colors.cyan(progress.crawled.toString())} pages | ${colors.gray('Queue:')} ${progress.queued} | ${colors.gray('Depth:')} ${progress.depth}   `);
+                    },
+                });
+                const result = await seoSpider.crawl(url);
+                process.stdout.write('\r' + ' '.repeat(80) + '\r');
+                console.log(colors.green(`\n✔ SEO Spider complete`) + colors.gray(` (${(result.duration / 1000).toFixed(1)}s)`));
+                console.log(`  ${colors.cyan('Pages crawled')}: ${result.pages.length}`);
+                console.log(`  ${colors.cyan('Unique URLs')}: ${result.visited.size}`);
+                console.log(`  ${colors.cyan('Avg SEO Score')}: ${result.summary.avgScore}/100`);
+                const responseTimes = result.pages.filter(p => p.duration > 0).map(p => p.duration);
+                const avgResponseTime = responseTimes.length > 0
+                    ? Math.round(responseTimes.reduce((a, b) => a + b, 0) / responseTimes.length)
+                    : 0;
+                const minResponseTime = responseTimes.length > 0 ? Math.min(...responseTimes) : 0;
+                const maxResponseTime = responseTimes.length > 0 ? Math.max(...responseTimes) : 0;
+                const reqPerSec = result.duration > 0 ? (result.pages.length / (result.duration / 1000)).toFixed(1) : '0';
+                const statusCounts = new Map();
+                for (const page of result.pages) {
+                    const status = page.status || 0;
+                    statusCounts.set(status, (statusCounts.get(status) || 0) + 1);
+                }
+                let totalInternalLinks = 0;
+                let totalExternalLinks = 0;
+                let totalImages = 0;
+                let imagesWithoutAlt = 0;
+                let pagesWithoutTitle = 0;
+                let pagesWithoutDescription = 0;
+                for (const page of result.pages) {
+                    if (page.seoReport) {
+                        totalInternalLinks += page.seoReport.links?.internal || 0;
+                        totalExternalLinks += page.seoReport.links?.external || 0;
+                        totalImages += page.seoReport.images?.total || 0;
+                        imagesWithoutAlt += page.seoReport.images?.withoutAlt || 0;
+                        if (!page.seoReport.title?.text)
+                            pagesWithoutTitle++;
+                        if (!page.seoReport.metaDescription?.text)
+                            pagesWithoutDescription++;
+                    }
+                }
+                console.log(colors.bold('\n  Performance:'));
+                console.log(`    ${colors.gray('Avg Response:')}  ${avgResponseTime}ms`);
+                console.log(`    ${colors.gray('Min/Max:')}       ${minResponseTime}ms / ${maxResponseTime}ms`);
+                console.log(`    ${colors.gray('Throughput:')}    ${reqPerSec} req/s`);
+                console.log(colors.bold('\n  HTTP Status:'));
+                const sortedStatuses = Array.from(statusCounts.entries()).sort((a, b) => b[1] - a[1]);
+                for (const [status, count] of sortedStatuses.slice(0, 5)) {
+                    const statusLabel = status === 0 ? 'Error' : status.toString();
+                    const statusColor = status >= 400 || status === 0 ? colors.red :
+                        status >= 300 ? colors.yellow : colors.green;
+                    const pct = ((count / result.pages.length) * 100).toFixed(0);
+                    console.log(`    ${statusColor(statusLabel.padEnd(5))} ${count.toString().padStart(3)} (${pct}%)`);
+                }
+                console.log(colors.bold('\n  Content:'));
+                console.log(`    ${colors.gray('Internal links:')} ${totalInternalLinks.toLocaleString()}`);
+                console.log(`    ${colors.gray('External links:')} ${totalExternalLinks.toLocaleString()}`);
+                console.log(`    ${colors.gray('Images:')}         ${totalImages.toLocaleString()} (${imagesWithoutAlt} missing alt)`);
+                console.log(`    ${colors.gray('Missing title:')}  ${pagesWithoutTitle}`);
+                console.log(`    ${colors.gray('Missing desc:')}   ${pagesWithoutDescription}`);
+                console.log(colors.bold('\n  SEO Summary:'));
+                const { summary } = result;
+                console.log(`    ${colors.red('✗')} Pages with errors:     ${summary.pagesWithErrors}`);
+                console.log(`    ${colors.yellow('⚠')} Pages with warnings:   ${summary.pagesWithWarnings}`);
+                console.log(`    ${colors.magenta('⚐')} Duplicate titles:      ${summary.duplicateTitles}`);
+                console.log(`    ${colors.magenta('⚐')} Duplicate descriptions:${summary.duplicateDescriptions}`);
+                console.log(`    ${colors.magenta('⚐')} Duplicate H1s:         ${summary.duplicateH1s}`);
+                console.log(`    ${colors.gray('○')} Orphan pages:          ${summary.orphanPages}`);
+                if (result.siteWideIssues.length > 0) {
+                    console.log(colors.bold('\n  Site-Wide Issues:'));
+                    for (const issue of result.siteWideIssues.slice(0, 10)) {
+                        const icon = issue.severity === 'error' ? colors.red('✗') :
+                            issue.severity === 'warning' ? colors.yellow('⚠') : colors.gray('○');
+                        console.log(`    ${icon} ${issue.message}`);
+                        if (issue.value) {
+                            const truncatedValue = issue.value.length > 50 ? issue.value.slice(0, 47) + '...' : issue.value;
+                            console.log(`      ${colors.gray(`"${truncatedValue}"`)}`);
+                        }
+                        const uniquePaths = [...new Set(issue.affectedUrls.map(u => new URL(u).pathname))];
+                        if (uniquePaths.length <= 3) {
+                            for (const path of uniquePaths) {
+                                console.log(`      ${colors.gray('→')} ${path}`);
+                            }
+                        }
+                        else {
+                            console.log(`      ${colors.gray(`→ ${uniquePaths.length} pages affected`)}`);
+                        }
+                    }
+                    if (result.siteWideIssues.length > 10) {
+                        console.log(colors.gray(`    ... and ${result.siteWideIssues.length - 10} more issues`));
+                    }
+                }
+                const pagesWithScores = result.pages
+                    .filter(p => p.seoReport)
+                    .sort((a, b) => (a.seoReport?.score || 0) - (b.seoReport?.score || 0));
+                const seenPaths = new Set();
+                const uniquePages = pagesWithScores.filter(page => {
+                    const path = new URL(page.url).pathname;
+                    if (seenPaths.has(path))
+                        return false;
+                    seenPaths.add(path);
+                    return true;
+                });
+                if (uniquePages.length > 0) {
+                    console.log(colors.bold('\n  Pages by SEO Score:'));
+                    const worstPages = uniquePages.slice(0, 5);
+                    for (const page of worstPages) {
+                        const score = page.seoReport?.score || 0;
+                        const grade = page.seoReport?.grade || '?';
+                        const path = new URL(page.url).pathname;
+                        const scoreColor = score >= 80 ? colors.green : score >= 60 ? colors.yellow : colors.red;
+                        console.log(`    ${scoreColor(`${score.toString().padStart(3)}`)} ${colors.gray(`[${grade}]`)} ${path.slice(0, 50)}`);
+                    }
+                    if (uniquePages.length > 5) {
+                        console.log(colors.gray(`    ... and ${uniquePages.length - 5} more pages`));
+                    }
+                }
+                if (outputFile) {
+                    console.log(colors.green(`\n  Report saved to: ${outputFile}`));
+                }
+            }
+            else {
+                const { Spider } = await import('../scrape/spider.js');
+                const spider = new Spider({
+                    maxDepth,
+                    maxPages,
+                    concurrency,
+                    sameDomain: true,
+                    delay: 100,
+                    onProgress: (progress) => {
+                        process.stdout.write(`\r${colors.gray('  Crawling:')} ${colors.cyan(progress.crawled.toString())} pages | ${colors.gray('Queue:')} ${progress.queued} | ${colors.gray('Depth:')} ${progress.depth}   `);
+                    },
+                });
+                const result = await spider.crawl(url);
+                process.stdout.write('\r' + ' '.repeat(80) + '\r');
+                console.log(colors.green(`\n✔ Spider complete`) + colors.gray(` (${(result.duration / 1000).toFixed(1)}s)`));
+                console.log(`  ${colors.cyan('Pages crawled')}: ${result.pages.length}`);
+                console.log(`  ${colors.cyan('Unique URLs')}: ${result.visited.size}`);
+                console.log(`  ${colors.cyan('Errors')}: ${result.errors.length}`);
+                const byDepth = new Map();
+                for (const page of result.pages) {
+                    byDepth.set(page.depth, (byDepth.get(page.depth) || 0) + 1);
+                }
+                console.log(colors.bold('\n  Pages by depth:'));
+                for (const [depth, count] of Array.from(byDepth.entries()).sort((a, b) => a[0] - b[0])) {
+                    const bar = '█'.repeat(Math.min(count, 40));
+                    console.log(`    ${colors.gray(`d${depth}:`)} ${bar} ${count}`);
+                }
+                const topPages = [...result.pages]
+                    .filter(p => !p.error)
+                    .sort((a, b) => b.links.length - a.links.length)
+                    .slice(0, 10);
+                if (topPages.length > 0) {
+                    console.log(colors.bold('\n  Top pages by outgoing links:'));
+                    for (const page of topPages) {
+                        const title = page.title.slice(0, 40) || new URL(page.url).pathname;
+                        console.log(`    ${colors.cyan(page.links.length.toString().padStart(3))} ${title}`);
+                    }
+                }
+                const formatError = (error) => {
+                    const statusMatch = error.match(/status code (\d{3})/i);
+                    if (statusMatch) {
+                        return `HTTP ${statusMatch[1]}`;
+                    }
+                    return error.length > 50 ? error.slice(0, 47) + '...' : error;
+                };
+                if (result.errors.length > 0 && result.errors.length <= 10) {
+                    console.log(colors.bold('\n  Errors:'));
+                    for (const err of result.errors) {
+                        const path = new URL(err.url).pathname;
+                        console.log(`    ${colors.red('✗')} ${path.padEnd(30)} → ${formatError(err.error)}`);
+                    }
+                }
+                else if (result.errors.length > 10) {
+                    console.log(colors.bold('\n  Errors:'));
+                    for (const err of result.errors.slice(0, 5)) {
+                        const path = new URL(err.url).pathname;
+                        console.log(`    ${colors.red('✗')} ${path.padEnd(30)} → ${formatError(err.error)}`);
+                    }
+                    console.log(colors.gray(`    ... and ${result.errors.length - 5} more errors`));
+                }
+                if (outputFile) {
+                    const jsonOutput = {
+                        startUrl: result.startUrl,
+                        crawledAt: new Date().toISOString(),
+                        duration: result.duration,
+                        summary: {
+                            totalPages: result.pages.length,
+                            successCount: result.pages.filter(p => !p.error).length,
+                            errorCount: result.errors.length,
+                            uniqueUrls: result.visited.size,
+                        },
+                        pages: result.pages.map(p => ({
+                            url: p.url,
+                            status: p.status,
+                            title: p.title,
+                            depth: p.depth,
+                            linksCount: p.links.length,
+                            duration: p.duration,
+                            error: p.error,
+                        })),
+                        errors: result.errors,
+                    };
+                    await fs.writeFile(outputFile, JSON.stringify(jsonOutput, null, 2));
+                    console.log(colors.green(`\n  Report saved to: ${outputFile}`));
+                }
+            }
+            console.log('');
+        }
+        catch (error) {
+            console.error(colors.red(`\nSpider failed: ${error.message}`));
+            process.exit(1);
+        }
+    });
+    program
+        .command('ai')
+        .description('Send a single AI prompt (no memory/context)')
+        .argument('<preset>', 'AI preset to use (e.g., @openai, @anthropic, @groq)')
+        .argument('<prompt...>', 'The prompt to send')
+        .option('-m, --model <model>', 'Override default model')
+        .option('-t, --temperature <temp>', 'Temperature (0-1)', '0.7')
+        .option('--max-tokens <tokens>', 'Max tokens in response', '2048')
+        .option('-w, --wait', 'Wait for full response (disable streaming)')
+        .option('-j, --json', 'Output raw JSON response')
+        .option('-e, --env [path]', 'Load .env file (auto-loads from cwd if exists)')
+        .addHelpText('after', `
+${colors.bold(colors.yellow('Examples:'))}
+  ${colors.green('$ rek ai @openai "What is the capital of France?"')}
+  ${colors.green('$ rek ai @anthropic "Explain quantum computing" -m claude-sonnet-4-20250514')}
+  ${colors.green('$ rek ai @groq "Write a haiku" --wait')}
+  ${colors.green('$ rek ai @openai "Translate to Spanish: Hello world"')}
+
+${colors.bold(colors.yellow('Available AI Presets:'))}
+  ${colors.cyan('@openai')}       OpenAI (GPT-4o, GPT-5.1)
+  ${colors.cyan('@anthropic')}    Anthropic (Claude)
+  ${colors.cyan('@groq')}         Groq (fast inference)
+  ${colors.cyan('@google')}       Google (Gemini)
+  ${colors.cyan('@xai')}          xAI (Grok)
+  ${colors.cyan('@mistral')}      Mistral AI
+  ${colors.cyan('@cohere')}       Cohere
+  ${colors.cyan('@deepseek')}     DeepSeek
+  ${colors.cyan('@perplexity')}   Perplexity
+  ${colors.cyan('@together')}     Together AI
+  ${colors.cyan('@fireworks')}    Fireworks AI
+  ${colors.cyan('@replicate')}    Replicate
+  ${colors.cyan('@huggingface')}  Hugging Face
+
+${colors.bold(colors.yellow('Note:'))}
+  This command sends a single prompt without conversation memory.
+  For chat with memory, use: ${colors.cyan('rek shell')} then ${colors.cyan('@openai Your message')}
+`)
+        .action(async (preset, promptParts, options) => {
+        if (options.env !== undefined) {
+            await loadEnvFile(options.env);
+        }
+        else {
+            try {
+                const envPath = join(process.cwd(), '.env');
+                await fs.access(envPath);
+                await loadEnvFile(true);
+            }
+            catch {
+            }
+        }
+        let presetName = preset;
+        if (presetName.startsWith('@')) {
+            presetName = presetName.slice(1);
+        }
+        const presetConfig = resolvePreset(presetName);
+        if (!presetConfig) {
+            console.error(colors.red(`Unknown AI preset: @${presetName}`));
+            console.log(colors.gray('Available AI presets: @openai, @anthropic, @groq, @google, @xai, @mistral, @cohere'));
+            process.exit(1);
+        }
+        if (!presetConfig._aiConfig) {
+            console.error(colors.red(`Preset @${presetName} does not support AI features.`));
+            console.log(colors.gray('Use an AI preset like @openai, @anthropic, @groq, etc.'));
+            process.exit(1);
+        }
+        const prompt = promptParts.join(' ');
+        if (!prompt.trim()) {
+            console.error(colors.red('Error: Prompt is required'));
+            process.exit(1);
+        }
+        try {
+            const { createClient } = await import('../core/client.js');
+            const client = createClient(presetConfig);
+            if (options.model) {
+                client.ai.setMemoryConfig({ systemPrompt: undefined });
+                client._aiConfig.model = options.model;
+            }
+            if (!options.json) {
+                console.log(colors.gray(`Using @${presetName} (${client._aiConfig.model})...\n`));
+            }
+            if (options.wait || options.json) {
+                const response = await client.ai.prompt(prompt);
+                if (options.json) {
+                    console.log(JSON.stringify({
+                        content: response.content,
+                        model: response.model,
+                        usage: response.usage,
+                        finishReason: response.finishReason,
+                    }, null, 2));
+                }
+                else {
+                    console.log(response.content);
+                    if (response.usage) {
+                        console.log(colors.gray(`\n─────────────────────────────────────────`));
+                        console.log(colors.gray(`Tokens: ${response.usage.inputTokens} in / ${response.usage.outputTokens} out`));
+                    }
+                }
+            }
+            else {
+                const stream = await client.ai.promptStream(prompt);
+                for await (const event of stream) {
+                    if (event.type === 'text') {
+                        process.stdout.write(event.content);
+                    }
+                }
+                console.log('');
+            }
+        }
+        catch (error) {
+            console.error(colors.red(`AI request failed: ${error.message}`));
+            if (error.cause) {
+                console.error(colors.gray(error.cause.message || error.cause));
+            }
+            process.exit(1);
+        }
+    });
     program
         .command('ip')
         .description('Get IP address intelligence using local GeoLite2 database')

package/dist/cli/tui/scroll-buffer.js

@@ -119,17 +119,17 @@ export class ScrollBuffer extends EventEmitter {
 }
 export function parseScrollKey(data) {
     const str = data.toString();
-    if (str === '\x1b[5~' || str === '\x1bOy')
+    if (str === '\x1b[5~' || str === '\x1bOy' || str === '\x1b[5;5~' || str === '\x1b[5;2~')
         return 'pageUp';
-    if (str === '\x1b[6~' || str === '\x1bOs')
+    if (str === '\x1b[6~' || str === '\x1bOs' || str === '\x1b[6;5~' || str === '\x1b[6;2~')
         return 'pageDown';
     if (str === '\x1b[1;2A')
         return 'scrollUp';
     if (str === '\x1b[1;2B')
         return 'scrollDown';
-    if (str === '\x1b[H' || str === '\x1b[1~' || str === '\x1bOH')
+    if (str === '\x1b[H' || str === '\x1b[1~' || str === '\x1bOH' || str === '\x1b[7~')
         return 'home';
-    if (str === '\x1b[F' || str === '\x1b[4~' || str === '\x1bOF')
+    if (str === '\x1b[F' || str === '\x1b[4~' || str === '\x1bOF' || str === '\x1b[8~')
         return 'end';
     if (str === 'q' || str === 'Q')
         return 'quit';

package/dist/cli/tui/shell.d.ts

@@ -13,6 +13,7 @@ export declare class RekShell {
     private scrollBuffer;
     private originalStdoutWrite;
     private inScrollMode;
+    private aiClients;
     constructor();
     private ensureInitialized;
     private getPrompt;
@@ -31,6 +32,8 @@ export declare class RekShell {
     private handleCommand;
     private runInteractiveMode;
     private runAIChat;
+    private runAIPresetChat;
+    private clearAIMemory;
     private runLoadTest;
     private parseLine;
     private setBaseUrl;