recker 1.0.28 → 1.0.29-next.72485b7

package/README.md

@@ -109,6 +109,8 @@ See [Mini Client documentation](./docs/http/18-mini-client.md) for more.
 | **Type-Safe** | Full TypeScript with Zod schema validation. |
 | **Observable** | DNS/TCP/TLS/TTFB timing breakdown per request. |
 | **Resilient** | Retry, circuit breaker, rate limiting, deduplication. |
+| **SEO Analysis** | 250+ rules across 21 categories. Site-wide crawling with duplicate detection. |
+| **Spider Crawler** | Web crawler with URL deduplication, depth control, and concurrency. |
 | **GeoIP (Offline)** | MaxMind GeoLite2 database with bogon detection. |
 | **RDAP Support** | Modern WHOIS with IANA Bootstrap and TLD detection. |
 
@@ -133,11 +135,31 @@ console.log(response.timings);
 // { dns: 12, tcp: 8, tls: 45, firstByte: 23, total: 156 }
 ```
 
-### Scraping
+### Scraping & Spider
 
 ```typescript
+// Scrape single page
 const doc = await client.scrape('https://example.com');
 const titles = doc.selectAll('h1').map(el => el.text());
+
+// Crawl entire site
+import { spider } from 'recker/scrape';
+const result = await spider('https://example.com', { maxPages: 50 });
+console.log(`Crawled ${result.pages.length} pages`);
+```
+
+### SEO Analysis
+
+```typescript
+import { analyzeSeo, seoSpider } from 'recker/seo';
+
+// Single page analysis - 250+ checks across 21 categories
+const report = await analyzeSeo(html, { baseUrl: 'https://example.com' });
+console.log(`Score: ${report.score}/100 (${report.grade})`);
+
+// Site-wide analysis - detect duplicates and orphan pages
+const siteReport = await seoSpider('https://example.com', { seo: true });
+console.log(`Duplicate titles: ${siteReport.summary.duplicateTitles}`);
 ```
 
 ### Circuit Breaker
@@ -174,6 +196,9 @@ rek -o data.json api.com/export
 # Interactive shell
 rek shell
 
+# SEO analysis
+rek seo https://example.com
+
 # Mock servers for testing
 rek serve http    # HTTP on :3000
 rek serve ws      # WebSocket on :8080
@@ -187,6 +212,8 @@ See [CLI Documentation](./docs/cli/01-overview.md) for more.
 - **[Quick Start](./docs/http/01-quickstart.md)** - Get running in 2 minutes
 - **[Mini Client](./docs/http/18-mini-client.md)** - Maximum performance mode
 - **[CLI Guide](./docs/cli/01-overview.md)** - Terminal client documentation
+- **[SEO Analysis](./docs/http/19-seo.md)** - 250+ rules, site-wide crawling
+- **[Web Scraping](./docs/http/14-scraping.md)** - HTML parsing and Spider crawler
 - **[API Reference](./docs/reference/01-api.md)** - Complete API documentation
 - **[Configuration](./docs/http/05-configuration.md)** - Client options
 - **[Plugins](./docs/http/10-plugins.md)** - Extend functionality

package/dist/ai/memory.d.ts

@@ -0,0 +1,35 @@
+import type { ChatMessage } from '../types/ai.js';
+import type { AIMemoryConfig } from '../types/ai-client.js';
+export declare class ConversationMemory {
+    private config;
+    private systemMessage;
+    private messages;
+    constructor(config?: AIMemoryConfig);
+    setSystemPrompt(prompt: string): void;
+    getSystemPrompt(): string;
+    addUserMessage(content: string): void;
+    addAssistantMessage(content: string): void;
+    addMessage(message: ChatMessage): void;
+    buildMessages(userPrompt: string): ChatMessage[];
+    recordResponse(content: string): void;
+    getMessages(): ChatMessage[];
+    getConversation(): readonly ChatMessage[];
+    getPairCount(): number;
+    clear(): void;
+    reset(): void;
+    setConfig(config: Partial<AIMemoryConfig>): void;
+    getConfig(): AIMemoryConfig;
+    private prune;
+    isEmpty(): boolean;
+    getMessageCount(): number;
+    toJSON(): {
+        config: AIMemoryConfig;
+        systemPrompt: string | null;
+        messages: ChatMessage[];
+    };
+    static fromJSON(data: {
+        config?: AIMemoryConfig;
+        systemPrompt?: string | null;
+        messages?: ChatMessage[];
+    }): ConversationMemory;
+}

package/dist/ai/memory.js

@@ -0,0 +1,136 @@
+const DEFAULT_MAX_PAIRS = 12;
+export class ConversationMemory {
+    config;
+    systemMessage = null;
+    messages = [];
+    constructor(config = {}) {
+        this.config = {
+            maxPairs: config.maxPairs ?? DEFAULT_MAX_PAIRS,
+            systemPrompt: config.systemPrompt ?? '',
+        };
+        if (this.config.systemPrompt) {
+            this.systemMessage = {
+                role: 'system',
+                content: this.config.systemPrompt,
+            };
+        }
+    }
+    setSystemPrompt(prompt) {
+        this.config.systemPrompt = prompt;
+        if (prompt) {
+            this.systemMessage = {
+                role: 'system',
+                content: prompt,
+            };
+        }
+        else {
+            this.systemMessage = null;
+        }
+    }
+    getSystemPrompt() {
+        return this.config.systemPrompt;
+    }
+    addUserMessage(content) {
+        this.messages.push({
+            role: 'user',
+            content,
+        });
+        this.prune();
+    }
+    addAssistantMessage(content) {
+        this.messages.push({
+            role: 'assistant',
+            content,
+        });
+        this.prune();
+    }
+    addMessage(message) {
+        if (message.role === 'system') {
+            this.setSystemPrompt(typeof message.content === 'string' ? message.content : '');
+            return;
+        }
+        this.messages.push(message);
+        this.prune();
+    }
+    buildMessages(userPrompt) {
+        this.addUserMessage(userPrompt);
+        return this.getMessages();
+    }
+    recordResponse(content) {
+        this.addAssistantMessage(content);
+    }
+    getMessages() {
+        const result = [];
+        if (this.systemMessage) {
+            result.push(this.systemMessage);
+        }
+        result.push(...this.messages);
+        return result;
+    }
+    getConversation() {
+        return this.messages;
+    }
+    getPairCount() {
+        let pairs = 0;
+        for (let i = 0; i < this.messages.length - 1; i += 2) {
+            if (this.messages[i].role === 'user' &&
+                this.messages[i + 1]?.role === 'assistant') {
+                pairs++;
+            }
+        }
+        return pairs;
+    }
+    clear() {
+        this.messages = [];
+    }
+    reset() {
+        this.messages = [];
+        this.systemMessage = null;
+        this.config.systemPrompt = '';
+    }
+    setConfig(config) {
+        if (config.maxPairs !== undefined) {
+            this.config.maxPairs = config.maxPairs;
+            this.prune();
+        }
+        if (config.systemPrompt !== undefined) {
+            this.setSystemPrompt(config.systemPrompt);
+        }
+    }
+    getConfig() {
+        return { ...this.config };
+    }
+    prune() {
+        const maxMessages = this.config.maxPairs * 2;
+        if (this.messages.length > maxMessages) {
+            const excess = this.messages.length - maxMessages;
+            const toRemove = Math.ceil(excess / 2) * 2;
+            this.messages = this.messages.slice(toRemove);
+        }
+    }
+    isEmpty() {
+        return this.messages.length === 0;
+    }
+    getMessageCount() {
+        return this.messages.length;
+    }
+    toJSON() {
+        return {
+            config: this.config,
+            systemPrompt: this.systemMessage?.content,
+            messages: [...this.messages],
+        };
+    }
+    static fromJSON(data) {
+        const memory = new ConversationMemory(data.config);
+        if (data.systemPrompt) {
+            memory.setSystemPrompt(data.systemPrompt);
+        }
+        if (data.messages) {
+            for (const msg of data.messages) {
+                memory.addMessage(msg);
+            }
+        }
+        return memory;
+    }
+}

package/dist/browser/types/ai-client.d.ts

@@ -0,0 +1,29 @@
+import type { AIResponse, AIStream, ChatMessage, AIProvider } from './ai.js';
+export interface AIMemoryConfig {
+    maxPairs?: number;
+    systemPrompt?: string;
+}
+export interface PresetAIConfig {
+    provider: AIProvider;
+    apiKey?: string;
+    model: string;
+    baseUrl?: string;
+    memory?: AIMemoryConfig;
+    organization?: string;
+    headers?: Record<string, string>;
+}
+export interface ClientAI {
+    chat(prompt: string): Promise<AIResponse>;
+    chatStream(prompt: string): Promise<AIStream>;
+    prompt(prompt: string): Promise<AIResponse>;
+    promptStream(prompt: string): Promise<AIStream>;
+    clearMemory(): void;
+    getMemory(): readonly ChatMessage[];
+    setMemoryConfig(config: Partial<AIMemoryConfig>): void;
+    getMemoryConfig(): AIMemoryConfig;
+    readonly provider: AIProvider;
+    readonly model: string;
+}
+export interface ClientOptionsWithAI {
+    _aiConfig?: PresetAIConfig;
+}

package/dist/browser/types/ai-client.js

@@ -0,0 +1 @@
+export {};

package/dist/cli/tui/scroll-buffer.js

@@ -119,17 +119,17 @@ export class ScrollBuffer extends EventEmitter {
 }
 export function parseScrollKey(data) {
     const str = data.toString();
-    if (str === '\x1b[5~' || str === '\x1bOy')
+    if (str === '\x1b[5~' || str === '\x1bOy' || str === '\x1b[5;5~' || str === '\x1b[5;2~')
         return 'pageUp';
-    if (str === '\x1b[6~' || str === '\x1bOs')
+    if (str === '\x1b[6~' || str === '\x1bOs' || str === '\x1b[6;5~' || str === '\x1b[6;2~')
         return 'pageDown';
     if (str === '\x1b[1;2A')
         return 'scrollUp';
     if (str === '\x1b[1;2B')
         return 'scrollDown';
-    if (str === '\x1b[H' || str === '\x1b[1~' || str === '\x1bOH')
+    if (str === '\x1b[H' || str === '\x1b[1~' || str === '\x1bOH' || str === '\x1b[7~')
         return 'home';
-    if (str === '\x1b[F' || str === '\x1b[4~' || str === '\x1bOF')
+    if (str === '\x1b[F' || str === '\x1b[4~' || str === '\x1bOF' || str === '\x1b[8~')
         return 'end';
     if (str === 'q' || str === 'Q')
         return 'quit';

package/dist/cli/tui/shell.d.ts

@@ -51,6 +51,7 @@ export declare class RekShell {
     private runRDAP;
     private runPing;
     private runScrap;
+    private runSpider;
     private runSelect;
     private runSelectText;
     private runSelectAttr;