recker 1.0.29 → 1.0.30

package/README.md

@@ -109,6 +109,8 @@ See [Mini Client documentation](./docs/http/18-mini-client.md) for more.
 | **Type-Safe** | Full TypeScript with Zod schema validation. |
 | **Observable** | DNS/TCP/TLS/TTFB timing breakdown per request. |
 | **Resilient** | Retry, circuit breaker, rate limiting, deduplication. |
+| **SEO Analysis** | 250+ rules across 21 categories. Site-wide crawling with duplicate detection. |
+| **Spider Crawler** | Web crawler with URL deduplication, depth control, and concurrency. |
 | **GeoIP (Offline)** | MaxMind GeoLite2 database with bogon detection. |
 | **RDAP Support** | Modern WHOIS with IANA Bootstrap and TLD detection. |
 
@@ -133,11 +135,31 @@ console.log(response.timings);
 // { dns: 12, tcp: 8, tls: 45, firstByte: 23, total: 156 }
 ```
 
-### Scraping
+### Scraping & Spider
 
 ```typescript
+// Scrape single page
 const doc = await client.scrape('https://example.com');
 const titles = doc.selectAll('h1').map(el => el.text());
+
+// Crawl entire site
+import { spider } from 'recker/scrape';
+const result = await spider('https://example.com', { maxPages: 50 });
+console.log(`Crawled ${result.pages.length} pages`);
+```
+
+### SEO Analysis
+
+```typescript
+import { analyzeSeo, seoSpider } from 'recker/seo';
+
+// Single page analysis - 250+ checks across 21 categories
+const report = await analyzeSeo(html, { baseUrl: 'https://example.com' });
+console.log(`Score: ${report.score}/100 (${report.grade})`);
+
+// Site-wide analysis - detect duplicates and orphan pages
+const siteReport = await seoSpider('https://example.com', { seo: true });
+console.log(`Duplicate titles: ${siteReport.summary.duplicateTitles}`);
 ```
 
 ### Circuit Breaker
@@ -174,6 +196,9 @@ rek -o data.json api.com/export
 # Interactive shell
 rek shell
 
+# SEO analysis
+rek seo https://example.com
+
 # Mock servers for testing
 rek serve http    # HTTP on :3000
 rek serve ws      # WebSocket on :8080
@@ -187,6 +212,8 @@ See [CLI Documentation](./docs/cli/01-overview.md) for more.
 - **[Quick Start](./docs/http/01-quickstart.md)** - Get running in 2 minutes
 - **[Mini Client](./docs/http/18-mini-client.md)** - Maximum performance mode
 - **[CLI Guide](./docs/cli/01-overview.md)** - Terminal client documentation
+- **[SEO Analysis](./docs/http/19-seo.md)** - 250+ rules, site-wide crawling
+- **[Web Scraping](./docs/http/14-scraping.md)** - HTML parsing and Spider crawler
 - **[API Reference](./docs/reference/01-api.md)** - Complete API documentation
 - **[Configuration](./docs/http/05-configuration.md)** - Client options
 - **[Plugins](./docs/http/10-plugins.md)** - Extend functionality

package/dist/ai/client-ai.d.ts

@@ -0,0 +1,41 @@
+import type { AIProvider, AIResponse, AIStream, ChatMessage } from '../types/ai.js';
+import type { ClientAI, PresetAIConfig, AIMemoryConfig } from '../types/ai-client.js';
+import type { Client } from '../core/client.js';
+export declare class AIConfigurationError extends Error {
+    constructor(message: string);
+}
+export declare class ClientAIImpl implements ClientAI {
+    private readonly client;
+    private readonly config;
+    private readonly memory;
+    constructor(client: Client, config: PresetAIConfig);
+    get provider(): AIProvider;
+    get model(): string;
+    chat(prompt: string): Promise<AIResponse>;
+    chatStream(prompt: string): Promise<AIStream>;
+    prompt(prompt: string): Promise<AIResponse>;
+    promptStream(prompt: string): Promise<AIStream>;
+    clearMemory(): void;
+    getMemory(): readonly ChatMessage[];
+    setMemoryConfig(config: Partial<AIMemoryConfig>): void;
+    getMemoryConfig(): AIMemoryConfig;
+    private makeRequest;
+    private makeStreamRequest;
+    private buildRequestBody;
+    private buildAnthropicBody;
+    private buildGoogleBody;
+    private buildCohereBody;
+    private getEndpoint;
+    private getExtraHeaders;
+    private parseResponse;
+    private parseOpenAIResponse;
+    private parseAnthropicResponse;
+    private parseGoogleResponse;
+    private parseCohereResponse;
+    private buildLatency;
+    private parseSSEStream;
+    private parseStreamChunk;
+    private parseOpenAIStreamChunk;
+    private parseAnthropicStreamChunk;
+    private wrapStreamWithMemory;
+}

package/dist/ai/client-ai.js

@@ -0,0 +1,391 @@
+import { ConversationMemory } from './memory.js';
+export class AIConfigurationError extends Error {
+    constructor(message) {
+        super(message);
+        this.name = 'AIConfigurationError';
+    }
+}
+const PROVIDER_ENDPOINTS = {
+    openai: '/chat/completions',
+    anthropic: '/messages',
+    google: '/models/{model}:generateContent',
+    groq: '/chat/completions',
+    mistral: '/chat/completions',
+    cohere: '/chat',
+    together: '/chat/completions',
+    perplexity: '/chat/completions',
+    deepseek: '/chat/completions',
+    fireworks: '/chat/completions',
+    xai: '/chat/completions',
+    replicate: '/predictions',
+    huggingface: '/models/{model}/v1/chat/completions',
+    ollama: '/api/chat',
+    'azure-openai': '/chat/completions',
+    'cloudflare-workers-ai': '/ai/run/@cf/meta/llama-2-7b-chat-int8',
+    custom: '/chat/completions',
+};
+export class ClientAIImpl {
+    client;
+    config;
+    memory;
+    constructor(client, config) {
+        this.client = client;
+        this.config = config;
+        this.memory = new ConversationMemory(config.memory);
+    }
+    get provider() {
+        return this.config.provider;
+    }
+    get model() {
+        return this.config.model;
+    }
+    async chat(prompt) {
+        const messages = this.memory.buildMessages(prompt);
+        const response = await this.makeRequest(messages, false);
+        this.memory.recordResponse(response.content);
+        return response;
+    }
+    async chatStream(prompt) {
+        const messages = this.memory.buildMessages(prompt);
+        const stream = await this.makeStreamRequest(messages);
+        return this.wrapStreamWithMemory(stream);
+    }
+    async prompt(prompt) {
+        const messages = [{ role: 'user', content: prompt }];
+        return this.makeRequest(messages, false);
+    }
+    async promptStream(prompt) {
+        const messages = [{ role: 'user', content: prompt }];
+        return this.makeStreamRequest(messages);
+    }
+    clearMemory() {
+        this.memory.clear();
+    }
+    getMemory() {
+        return this.memory.getConversation();
+    }
+    setMemoryConfig(config) {
+        this.memory.setConfig(config);
+    }
+    getMemoryConfig() {
+        return this.memory.getConfig();
+    }
+    async makeRequest(messages, stream) {
+        const startTime = performance.now();
+        const body = this.buildRequestBody(messages, stream);
+        const endpoint = this.getEndpoint();
+        const response = await this.client.post(endpoint, {
+            json: body,
+            headers: this.getExtraHeaders(),
+        });
+        const data = await response.json();
+        return this.parseResponse(data, startTime);
+    }
+    async makeStreamRequest(messages) {
+        const body = this.buildRequestBody(messages, true);
+        const endpoint = this.getEndpoint();
+        const response = await this.client.post(endpoint, {
+            json: body,
+            headers: this.getExtraHeaders(),
+        });
+        return this.parseSSEStream(response.raw);
+    }
+    buildRequestBody(messages, stream) {
+        const provider = this.config.provider;
+        if (provider === 'anthropic') {
+            return this.buildAnthropicBody(messages, stream);
+        }
+        if (provider === 'google') {
+            return this.buildGoogleBody(messages, stream);
+        }
+        if (provider === 'cohere') {
+            return this.buildCohereBody(messages, stream);
+        }
+        return {
+            model: this.config.model,
+            messages: messages.map(m => ({
+                role: m.role,
+                content: m.content,
+            })),
+            stream,
+            ...(stream && { stream_options: { include_usage: true } }),
+        };
+    }
+    buildAnthropicBody(messages, stream) {
+        const systemMessages = messages.filter(m => m.role === 'system');
+        const otherMessages = messages.filter(m => m.role !== 'system');
+        return {
+            model: this.config.model,
+            max_tokens: 4096,
+            system: systemMessages.map(m => m.content).join('\n') || undefined,
+            messages: otherMessages.map(m => ({
+                role: m.role === 'assistant' ? 'assistant' : 'user',
+                content: m.content,
+            })),
+            stream,
+        };
+    }
+    buildGoogleBody(messages, stream) {
+        const contents = messages
+            .filter(m => m.role !== 'system')
+            .map(m => ({
+            role: m.role === 'assistant' ? 'model' : 'user',
+            parts: [{ text: m.content }],
+        }));
+        const systemInstruction = messages
+            .filter(m => m.role === 'system')
+            .map(m => m.content)
+            .join('\n');
+        return {
+            contents,
+            ...(systemInstruction && {
+                systemInstruction: { parts: [{ text: systemInstruction }] },
+            }),
+            generationConfig: {
+                maxOutputTokens: 4096,
+            },
+        };
+    }
+    buildCohereBody(messages, stream) {
+        const chatHistory = messages.slice(0, -1).map(m => ({
+            role: m.role === 'assistant' ? 'CHATBOT' : 'USER',
+            message: m.content,
+        }));
+        const lastMessage = messages[messages.length - 1];
+        return {
+            model: this.config.model,
+            message: typeof lastMessage.content === 'string' ? lastMessage.content : '',
+            chat_history: chatHistory.length > 0 ? chatHistory : undefined,
+            stream,
+        };
+    }
+    getEndpoint() {
+        let endpoint = PROVIDER_ENDPOINTS[this.config.provider] || '/chat/completions';
+        endpoint = endpoint.replace('{model}', this.config.model);
+        return endpoint;
+    }
+    getExtraHeaders() {
+        const headers = {};
+        if (this.config.headers) {
+            Object.assign(headers, this.config.headers);
+        }
+        return headers;
+    }
+    parseResponse(data, startTime) {
+        const provider = this.config.provider;
+        const endTime = performance.now();
+        if (provider === 'anthropic') {
+            return this.parseAnthropicResponse(data, startTime, endTime);
+        }
+        if (provider === 'google') {
+            return this.parseGoogleResponse(data, startTime, endTime);
+        }
+        if (provider === 'cohere') {
+            return this.parseCohereResponse(data, startTime, endTime);
+        }
+        return this.parseOpenAIResponse(data, startTime, endTime);
+    }
+    parseOpenAIResponse(data, startTime, endTime) {
+        const d = data;
+        const content = d.choices?.[0]?.message?.content || '';
+        const usage = {
+            inputTokens: d.usage?.prompt_tokens || 0,
+            outputTokens: d.usage?.completion_tokens || 0,
+            totalTokens: d.usage?.total_tokens || 0,
+        };
+        return {
+            content,
+            usage,
+            latency: this.buildLatency(startTime, endTime, usage.outputTokens),
+            model: d.model || this.config.model,
+            provider: this.config.provider,
+            cached: false,
+            finishReason: d.choices?.[0]?.finish_reason,
+            raw: data,
+        };
+    }
+    parseAnthropicResponse(data, startTime, endTime) {
+        const d = data;
+        const textContent = d.content?.find(c => c.type === 'text');
+        const content = textContent?.text || '';
+        const usage = {
+            inputTokens: d.usage?.input_tokens || 0,
+            outputTokens: d.usage?.output_tokens || 0,
+            totalTokens: (d.usage?.input_tokens || 0) + (d.usage?.output_tokens || 0),
+        };
+        return {
+            content,
+            usage,
+            latency: this.buildLatency(startTime, endTime, usage.outputTokens),
+            model: d.model || this.config.model,
+            provider: 'anthropic',
+            cached: false,
+            finishReason: d.stop_reason === 'end_turn' ? 'stop' : undefined,
+            raw: data,
+        };
+    }
+    parseGoogleResponse(data, startTime, endTime) {
+        const d = data;
+        const content = d.candidates?.[0]?.content?.parts?.[0]?.text || '';
+        const usage = {
+            inputTokens: d.usageMetadata?.promptTokenCount || 0,
+            outputTokens: d.usageMetadata?.candidatesTokenCount || 0,
+            totalTokens: d.usageMetadata?.totalTokenCount || 0,
+        };
+        return {
+            content,
+            usage,
+            latency: this.buildLatency(startTime, endTime, usage.outputTokens),
+            model: this.config.model,
+            provider: 'google',
+            cached: false,
+            finishReason: d.candidates?.[0]?.finishReason === 'STOP' ? 'stop' : undefined,
+            raw: data,
+        };
+    }
+    parseCohereResponse(data, startTime, endTime) {
+        const d = data;
+        const content = d.text || '';
+        const usage = {
+            inputTokens: d.meta?.tokens?.input_tokens || 0,
+            outputTokens: d.meta?.tokens?.output_tokens || 0,
+            totalTokens: (d.meta?.tokens?.input_tokens || 0) + (d.meta?.tokens?.output_tokens || 0),
+        };
+        return {
+            content,
+            usage,
+            latency: this.buildLatency(startTime, endTime, usage.outputTokens),
+            model: this.config.model,
+            provider: 'cohere',
+            cached: false,
+            finishReason: d.finish_reason === 'COMPLETE' ? 'stop' : undefined,
+            raw: data,
+        };
+    }
+    buildLatency(startTime, endTime, outputTokens) {
+        const total = endTime - startTime;
+        return {
+            ttft: total,
+            tps: outputTokens > 0 ? (outputTokens / (total / 1000)) : 0,
+            total,
+        };
+    }
+    async *parseSSEStream(response) {
+        const reader = response.body?.getReader();
+        if (!reader) {
+            throw new Error('No response body');
+        }
+        const decoder = new TextDecoder();
+        let buffer = '';
+        let firstChunkTime;
+        const startTime = performance.now();
+        try {
+            while (true) {
+                const { done, value } = await reader.read();
+                if (done) {
+                    break;
+                }
+                buffer += decoder.decode(value, { stream: true });
+                const lines = buffer.split('\n');
+                buffer = lines.pop() || '';
+                for (const line of lines) {
+                    const trimmed = line.trim();
+                    if (!trimmed || trimmed === 'data: [DONE]') {
+                        continue;
+                    }
+                    if (trimmed.startsWith('data: ')) {
+                        const jsonStr = trimmed.slice(6);
+                        try {
+                            const event = this.parseStreamChunk(jsonStr);
+                            if (event) {
+                                if (!firstChunkTime && event.type === 'text') {
+                                    firstChunkTime = performance.now();
+                                }
+                                yield event;
+                            }
+                        }
+                        catch {
+                        }
+                    }
+                }
+            }
+        }
+        finally {
+            reader.releaseLock();
+        }
+    }
+    parseStreamChunk(jsonStr) {
+        const data = JSON.parse(jsonStr);
+        const provider = this.config.provider;
+        if (provider === 'anthropic') {
+            return this.parseAnthropicStreamChunk(data);
+        }
+        return this.parseOpenAIStreamChunk(data);
+    }
+    parseOpenAIStreamChunk(data) {
+        const choice = data.choices?.[0];
+        if (!choice) {
+            if (data.usage) {
+                return {
+                    type: 'usage',
+                    usage: {
+                        inputTokens: data.usage.prompt_tokens || 0,
+                        outputTokens: data.usage.completion_tokens || 0,
+                        totalTokens: data.usage.total_tokens || 0,
+                    },
+                };
+            }
+            return null;
+        }
+        if (choice.delta?.content) {
+            return {
+                type: 'text',
+                content: choice.delta.content,
+            };
+        }
+        if (choice.finish_reason) {
+            return {
+                type: 'done',
+                finishReason: choice.finish_reason,
+            };
+        }
+        return null;
+    }
+    parseAnthropicStreamChunk(data) {
+        if (data.type === 'content_block_delta' && data.delta?.type === 'text_delta') {
+            return {
+                type: 'text',
+                content: data.delta.text || '',
+            };
+        }
+        if (data.type === 'message_stop') {
+            return {
+                type: 'done',
+                finishReason: 'stop',
+            };
+        }
+        if (data.type === 'message_delta' && data.message?.usage) {
+            return {
+                type: 'usage',
+                usage: {
+                    inputTokens: data.message.usage.input_tokens || 0,
+                    outputTokens: data.message.usage.output_tokens || 0,
+                    totalTokens: (data.message.usage.input_tokens || 0) + (data.message.usage.output_tokens || 0),
+                },
+            };
+        }
+        return null;
+    }
+    async *wrapStreamWithMemory(stream) {
+        let fullContent = '';
+        for await (const event of stream) {
+            if (event.type === 'text') {
+                fullContent += event.content;
+            }
+            yield event;
+        }
+        if (fullContent) {
+            this.memory.recordResponse(fullContent);
+        }
+    }
+}

package/dist/ai/index.d.ts

@@ -1,4 +1,6 @@
 export { UnifiedAIClient, createAI } from './client.js';
+export { ClientAIImpl, AIConfigurationError } from './client-ai.js';
+export { ConversationMemory } from './memory.js';
 export { BaseAIProvider, AIError, RateLimitError, ContextLengthError, OverloadedError, AuthenticationError } from './providers/base.js';
 export { OpenAIProvider } from './providers/openai.js';
 export { AnthropicProvider } from './providers/anthropic.js';

package/dist/ai/index.js

@@ -1,4 +1,6 @@
 export { UnifiedAIClient, createAI } from './client.js';
+export { ClientAIImpl, AIConfigurationError } from './client-ai.js';
+export { ConversationMemory } from './memory.js';
 export { BaseAIProvider, AIError, RateLimitError, ContextLengthError, OverloadedError, AuthenticationError } from './providers/base.js';
 export { OpenAIProvider } from './providers/openai.js';
 export { AnthropicProvider } from './providers/anthropic.js';

package/dist/ai/memory.d.ts

@@ -0,0 +1,35 @@
+import type { ChatMessage } from '../types/ai.js';
+import type { AIMemoryConfig } from '../types/ai-client.js';
+export declare class ConversationMemory {
+    private config;
+    private systemMessage;
+    private messages;
+    constructor(config?: AIMemoryConfig);
+    setSystemPrompt(prompt: string): void;
+    getSystemPrompt(): string;
+    addUserMessage(content: string): void;
+    addAssistantMessage(content: string): void;
+    addMessage(message: ChatMessage): void;
+    buildMessages(userPrompt: string): ChatMessage[];
+    recordResponse(content: string): void;
+    getMessages(): ChatMessage[];
+    getConversation(): readonly ChatMessage[];
+    getPairCount(): number;
+    clear(): void;
+    reset(): void;
+    setConfig(config: Partial<AIMemoryConfig>): void;
+    getConfig(): AIMemoryConfig;
+    private prune;
+    isEmpty(): boolean;
+    getMessageCount(): number;
+    toJSON(): {
+        config: AIMemoryConfig;
+        systemPrompt: string | null;
+        messages: ChatMessage[];
+    };
+    static fromJSON(data: {
+        config?: AIMemoryConfig;
+        systemPrompt?: string | null;
+        messages?: ChatMessage[];
+    }): ConversationMemory;
+}

package/dist/ai/memory.js

@@ -0,0 +1,136 @@
+const DEFAULT_MAX_PAIRS = 12;
+export class ConversationMemory {
+    config;
+    systemMessage = null;
+    messages = [];
+    constructor(config = {}) {
+        this.config = {
+            maxPairs: config.maxPairs ?? DEFAULT_MAX_PAIRS,
+            systemPrompt: config.systemPrompt ?? '',
+        };
+        if (this.config.systemPrompt) {
+            this.systemMessage = {
+                role: 'system',
+                content: this.config.systemPrompt,
+            };
+        }
+    }
+    setSystemPrompt(prompt) {
+        this.config.systemPrompt = prompt;
+        if (prompt) {
+            this.systemMessage = {
+                role: 'system',
+                content: prompt,
+            };
+        }
+        else {
+            this.systemMessage = null;
+        }
+    }
+    getSystemPrompt() {
+        return this.config.systemPrompt;
+    }
+    addUserMessage(content) {
+        this.messages.push({
+            role: 'user',
+            content,
+        });
+        this.prune();
+    }
+    addAssistantMessage(content) {
+        this.messages.push({
+            role: 'assistant',
+            content,
+        });
+        this.prune();
+    }
+    addMessage(message) {
+        if (message.role === 'system') {
+            this.setSystemPrompt(typeof message.content === 'string' ? message.content : '');
+            return;
+        }
+        this.messages.push(message);
+        this.prune();
+    }
+    buildMessages(userPrompt) {
+        this.addUserMessage(userPrompt);
+        return this.getMessages();
+    }
+    recordResponse(content) {
+        this.addAssistantMessage(content);
+    }
+    getMessages() {
+        const result = [];
+        if (this.systemMessage) {
+            result.push(this.systemMessage);
+        }
+        result.push(...this.messages);
+        return result;
+    }
+    getConversation() {
+        return this.messages;
+    }
+    getPairCount() {
+        let pairs = 0;
+        for (let i = 0; i < this.messages.length - 1; i += 2) {
+            if (this.messages[i].role === 'user' &&
+                this.messages[i + 1]?.role === 'assistant') {
+                pairs++;
+            }
+        }
+        return pairs;
+    }
+    clear() {
+        this.messages = [];
+    }
+    reset() {
+        this.messages = [];
+        this.systemMessage = null;
+        this.config.systemPrompt = '';
+    }
+    setConfig(config) {
+        if (config.maxPairs !== undefined) {
+            this.config.maxPairs = config.maxPairs;
+            this.prune();
+        }
+        if (config.systemPrompt !== undefined) {
+            this.setSystemPrompt(config.systemPrompt);
+        }
+    }
+    getConfig() {
+        return { ...this.config };
+    }
+    prune() {
+        const maxMessages = this.config.maxPairs * 2;
+        if (this.messages.length > maxMessages) {
+            const excess = this.messages.length - maxMessages;
+            const toRemove = Math.ceil(excess / 2) * 2;
+            this.messages = this.messages.slice(toRemove);
+        }
+    }
+    isEmpty() {
+        return this.messages.length === 0;
+    }
+    getMessageCount() {
+        return this.messages.length;
+    }
+    toJSON() {
+        return {
+            config: this.config,
+            systemPrompt: this.systemMessage?.content,
+            messages: [...this.messages],
+        };
+    }
+    static fromJSON(data) {
+        const memory = new ConversationMemory(data.config);
+        if (data.systemPrompt) {
+            memory.setSystemPrompt(data.systemPrompt);
+        }
+        if (data.messages) {
+            for (const msg of data.messages) {
+                memory.addMessage(msg);
+            }
+        }
+        return memory;
+    }
+}