recker 1.0.33 → 1.0.34-next.c0d9cdb

package/dist/cli/presets.d.ts

@@ -1 +1,5 @@
-export declare function resolvePreset(name: string): any;
+export interface ResolvePresetOptions {
+    throwOnError?: boolean;
+    silent?: boolean;
+}
+export declare function resolvePreset(name: string, options?: ResolvePresetOptions): any;

package/dist/cli/presets.js

@@ -10,38 +10,62 @@ const ENV_MAPPING = {
     slack: ['SLACK_TOKEN'],
     vercel: ['VERCEL_TOKEN'],
     supabase: ['SUPABASE_URL', 'SUPABASE_KEY'],
+    groq: ['GROQ_API_KEY'],
+    google: ['GOOGLE_API_KEY'],
+    xai: ['XAI_API_KEY'],
+    mistral: ['MISTRAL_API_KEY'],
+    cohere: ['COHERE_API_KEY'],
+    deepseek: ['DEEPSEEK_API_KEY'],
+    fireworks: ['FIREWORKS_API_KEY'],
+    together: ['TOGETHER_API_KEY'],
+    perplexity: ['PERPLEXITY_API_KEY'],
 };
-export function resolvePreset(name) {
+export function resolvePreset(name, options = {}) {
+    const { throwOnError = false, silent = false } = options;
     const presetFn = presets[name];
     if (!presetFn) {
-        console.error(colors.red(`Error: Unknown preset '@${name}'`));
-        process.exit(1);
+        const msg = `Unknown preset '@${name}'`;
+        if (throwOnError) {
+            throw new Error(msg);
+        }
+        if (!silent) {
+            console.error(colors.red(`Error: ${msg}`));
+        }
+        return null;
     }
     const requiredEnvs = ENV_MAPPING[name];
-    const options = {};
+    const presetOptions = {};
     if (requiredEnvs) {
         let missing = false;
         for (const envVar of requiredEnvs) {
             const value = process.env[envVar];
             if (!value) {
-                console.error(colors.yellow(`Warning: Missing env variable ${envVar} for preset @${name}`));
+                if (!silent) {
+                    console.error(colors.yellow(`Warning: Missing env variable ${envVar} for preset @${name}`));
+                }
                 missing = true;
             }
             else {
                 const key = mapEnvToOption(name, envVar);
-                options[key] = value;
+                presetOptions[key] = value;
             }
         }
-        if (missing) {
+        if (missing && !silent) {
             console.log(colors.gray(`Tip: export ${requiredEnvs.join('=... ')}=...`));
         }
     }
     try {
-        return presetFn(options);
+        return presetFn(presetOptions);
     }
     catch (error) {
-        console.error(colors.red(`Error initializing preset @${name}: ${error.message}`));
-        process.exit(1);
+        const msg = `Error initializing preset @${name}: ${error.message}`;
+        if (throwOnError) {
+            throw new Error(msg);
+        }
+        if (!silent) {
+            console.error(colors.red(msg));
+        }
+        return null;
     }
 }
 function mapEnvToOption(preset, env) {

package/dist/cli/tui/ai-chat.d.ts

@@ -1,2 +1,7 @@
 import readline from 'node:readline';
-export declare function startAIChat(rl: readline.Interface, provider?: string, apiKey?: string, model?: string): Promise<void>;
+import type { Client } from '../../core/client.js';
+export interface AIMode {
+    aiClients: Map<string, Client>;
+    variables?: Record<string, any>;
+}
+export declare function startAIChat(rl: readline.Interface, provider: string | undefined, context: AIMode): Promise<void>;

package/dist/cli/tui/ai-chat.js

@@ -1,100 +1,304 @@
 import readline from 'node:readline';
 import colors from '../../utils/colors.js';
-import { createAI } from '../../ai/client.js';
-export async function startAIChat(rl, provider = 'openai', apiKey, model) {
-    console.clear();
-    console.log(colors.bold(colors.magenta(`🤖 Rek AI Chat (${provider})`)));
-    console.log(colors.gray('Type your message. Ctrl+C to exit.'));
-    const envKey = provider === 'openai' ? 'OPENAI_API_KEY' : 'ANTHROPIC_API_KEY';
-    const key = apiKey || process.env[envKey];
-    if (!key) {
-        console.log(colors.yellow(`
-Warning: No API Key found for ${provider}.`));
-        console.log(`Please set it via environment variable ${colors.bold(envKey)} or passing it to the command.`);
-        console.log(`Example: set ${envKey}=sk-... inside the shell.`);
+import { createClient } from '../../core/client.js';
+import { resolvePreset } from '../presets.js';
+const AI_PRESETS = [
+    'openai', 'anthropic', 'groq', 'google', 'xai',
+    'mistral', 'cohere', 'deepseek', 'fireworks', 'together', 'perplexity'
+];
+const ENV_VAR_MAP = {
+    openai: 'OPENAI_API_KEY',
+    anthropic: 'ANTHROPIC_API_KEY',
+    google: 'GOOGLE_API_KEY',
+    groq: 'GROQ_API_KEY',
+    xai: 'XAI_API_KEY',
+    mistral: 'MISTRAL_API_KEY',
+    cohere: 'COHERE_API_KEY',
+    deepseek: 'DEEPSEEK_API_KEY',
+    fireworks: 'FIREWORKS_API_KEY',
+    together: 'TOGETHER_API_KEY',
+    perplexity: 'PERPLEXITY_API_KEY',
+};
+export async function startAIChat(rl, provider = 'openai', context) {
+    let currentProvider = provider.toLowerCase();
+    if (!AI_PRESETS.includes(currentProvider)) {
+        console.log(colors.red(`Unknown AI provider: ${currentProvider}`));
+        console.log(colors.gray(`Available: ${AI_PRESETS.join(', ')}`));
         return;
     }
-    const client = createAI({
-        defaultProvider: provider,
-        providers: {
-            [provider]: { apiKey: key }
-        },
-        observability: false
-    });
-    const history = [
-        { role: 'system', content: 'You are Recker AI, a helpful and concise assistant in a terminal environment.' }
-    ];
-    rl.setPrompt(colors.magenta('You › '));
+    console.clear();
+    printHeader(currentProvider);
+    let client = await getOrCreateClient(currentProvider, context);
+    if (!client)
+        return;
+    const getPrompt = () => colors.magenta(`${currentProvider} › `);
+    rl.setPrompt(getPrompt());
     rl.prompt();
+    let isGenerating = false;
+    let abortController = null;
     return new Promise((resolve) => {
+        const cleanup = () => {
+            rl.off('line', onLine);
+            rl.off('SIGINT', onSigInt);
+            if (process.stdin.isTTY) {
+                process.stdin.off('keypress', onKeypress);
+            }
+            console.log('');
+            console.log(colors.gray('Exiting AI mode...'));
+        };
         const onLine = async (line) => {
             const input = line.trim();
             if (!input) {
                 rl.prompt();
                 return;
             }
-            if (input.toLowerCase() === '/clear') {
-                history.length = 1;
-                console.clear();
-                console.log(colors.gray('Context cleared.'));
-                rl.prompt();
-                return;
+            if (input.startsWith('/')) {
+                const handled = await handleCommand(input, currentProvider, client, context, rl, () => {
+                    cleanup();
+                    resolve();
+                }, async (newProvider) => {
+                    currentProvider = newProvider;
+                    client = await getOrCreateClient(currentProvider, context);
+                    if (client) {
+                        rl.setPrompt(getPrompt());
+                        console.log(colors.green(`Switched to ${currentProvider}`));
+                        const memory = client.ai.getMemory();
+                        if (memory.length > 0) {
+                            console.log(colors.gray(`Memory: ${Math.floor(memory.length / 2)} pairs`));
+                        }
+                    }
+                    rl.prompt();
+                });
+                if (handled)
+                    return;
             }
-            if (input.toLowerCase() === '/exit') {
-                cleanup();
-                resolve();
+            if (!client || !client.hasAI) {
+                console.log(colors.red('AI client not available'));
+                rl.prompt();
                 return;
             }
-            history.push({ role: 'user', content: input });
             rl.pause();
-            process.stdout.write(colors.cyan('AI  › '));
-            let fullResponse = '';
+            isGenerating = true;
+            abortController = new AbortController();
+            const model = client._aiConfig?.model || currentProvider;
+            process.stdout.write(colors.gray(`${model} › `));
             try {
-                const stream = await client.stream({
-                    provider: provider,
-                    model: model,
-                    messages: history
-                });
-                for await (const chunk of stream) {
-                    const content = typeof chunk === 'string' ? chunk : chunk.content || chunk.delta?.content || '';
-                    if (content) {
-                        process.stdout.write(content);
-                        fullResponse += content;
+                const stream = await client.ai.chatStream(input);
+                let hasContent = false;
+                for await (const event of stream) {
+                    if (abortController?.signal.aborted) {
+                        console.log(colors.yellow('\n[Interrupted]'));
+                        break;
+                    }
+                    if (event.type === 'text') {
+                        if (!hasContent) {
+                            process.stdout.write('\n' + colors.orange(event.content));
+                            hasContent = true;
+                        }
+                        else {
+                            process.stdout.write(colors.orange(event.content));
+                        }
+                    }
+                    else if (event.type === 'error') {
+                        console.log(colors.red(`\nError: ${event.error}`));
                     }
                 }
-                process.stdout.write('\n');
-                history.push({ role: 'assistant', content: fullResponse });
+                console.log(colors.reset(''));
+                const memory = client.ai.getMemory();
+                const pairs = Math.floor(memory.length / 2);
+                console.log(colors.gray(`Memory: ${pairs}/12 pairs`));
             }
             catch (error) {
-                console.log(colors.red(`
-Error: ${error.message}`));
-                if (error.cause)
-                    console.log(colors.gray(error.cause));
+                handleError(error, currentProvider);
             }
             finally {
+                isGenerating = false;
+                abortController = null;
                 rl.resume();
                 rl.prompt();
             }
         };
-        const cleanup = () => {
-            rl.off('line', onLine);
-            rl.off('SIGINT', onSigInt);
-            process.stdin.off('keypress', onKeypress);
-        };
-        if (process.stdin.isTTY)
-            readline.emitKeypressEvents(process.stdin);
         const onKeypress = (_str, key) => {
             if (key && key.name === 'escape') {
-                cleanup();
-                resolve();
+                if (isGenerating && abortController) {
+                    abortController.abort();
+                }
+                else {
+                    cleanup();
+                    resolve();
+                }
             }
         };
-        process.stdin.on('keypress', onKeypress);
+        if (process.stdin.isTTY) {
+            readline.emitKeypressEvents(process.stdin);
+            process.stdin.on('keypress', onKeypress);
+        }
         rl.on('line', onLine);
         const onSigInt = () => {
-            cleanup();
-            resolve();
+            if (isGenerating && abortController) {
+                abortController.abort();
+            }
+            else {
+                cleanup();
+                resolve();
+            }
         };
         rl.once('SIGINT', onSigInt);
     });
 }
+async function getOrCreateClient(provider, context) {
+    let client = context.aiClients.get(provider);
+    if (client) {
+        return client;
+    }
+    try {
+        const presetConfig = resolvePreset(provider);
+        if (!presetConfig) {
+            console.log(colors.red(`Unknown preset: ${provider}`));
+            return null;
+        }
+        if (!presetConfig._aiConfig) {
+            console.log(colors.red(`Preset ${provider} does not support AI features.`));
+            return null;
+        }
+        client = createClient(presetConfig);
+        context.aiClients.set(provider, client);
+        return client;
+    }
+    catch (error) {
+        const envVar = ENV_VAR_MAP[provider] || `${provider.toUpperCase()}_API_KEY`;
+        console.log(colors.red(`Failed to initialize ${provider}`));
+        console.log(colors.gray(`Make sure ${envVar} is set.`));
+        return null;
+    }
+}
+async function handleCommand(input, currentProvider, client, context, rl, onExit, onSwitch) {
+    const parts = input.slice(1).split(/\s+/);
+    const cmd = parts[0].toLowerCase();
+    const args = parts.slice(1);
+    switch (cmd) {
+        case 'exit':
+        case 'quit':
+        case 'q':
+            onExit();
+            return true;
+        case 'clear':
+            if (client?.hasAI) {
+                client.ai.clearMemory();
+                console.log(colors.green(`Memory cleared for ${currentProvider}`));
+            }
+            rl.prompt();
+            return true;
+        case 'switch':
+        case 's':
+            const newProvider = args[0]?.toLowerCase();
+            if (!newProvider) {
+                console.log(colors.yellow('Usage: /switch <provider>'));
+                console.log(colors.gray(`Available: ${AI_PRESETS.join(', ')}`));
+                rl.prompt();
+                return true;
+            }
+            if (!AI_PRESETS.includes(newProvider)) {
+                console.log(colors.red(`Unknown provider: ${newProvider}`));
+                console.log(colors.gray(`Available: ${AI_PRESETS.join(', ')}`));
+                rl.prompt();
+                return true;
+            }
+            await onSwitch(newProvider);
+            return true;
+        case 'model':
+        case 'm':
+            const model = client?._aiConfig?.model || 'default';
+            console.log(colors.cyan(`Current model: ${model}`));
+            console.log(colors.gray('Note: Model is set by the preset configuration.'));
+            rl.prompt();
+            return true;
+        case 'memory':
+        case 'mem':
+            if (client?.hasAI) {
+                const memory = client.ai.getMemory();
+                const pairs = Math.floor(memory.length / 2);
+                console.log(colors.cyan(`Memory: ${pairs}/12 pairs (${memory.length} messages)`));
+                if (pairs > 0) {
+                    console.log(colors.gray('Last exchange:'));
+                    const lastTwo = memory.slice(-2);
+                    for (const msg of lastTwo) {
+                        const role = msg.role === 'user' ? colors.magenta('You') : colors.orange('AI');
+                        const preview = msg.content.slice(0, 100) + (msg.content.length > 100 ? '...' : '');
+                        console.log(`  ${role}: ${colors.gray(preview)}`);
+                    }
+                }
+            }
+            rl.prompt();
+            return true;
+        case 'providers':
+        case 'list':
+            console.log(colors.cyan('Available AI providers:'));
+            for (const p of AI_PRESETS) {
+                const isActive = p === currentProvider ? colors.green(' (active)') : '';
+                const hasClient = context.aiClients.has(p) ? colors.gray(' [loaded]') : '';
+                console.log(`  ${p}${isActive}${hasClient}`);
+            }
+            rl.prompt();
+            return true;
+        case 'help':
+        case 'h':
+        case '?':
+            printHelp();
+            rl.prompt();
+            return true;
+        default:
+            console.log(colors.yellow(`Unknown command: /${cmd}`));
+            console.log(colors.gray('Type /help for available commands'));
+            rl.prompt();
+            return true;
+    }
+}
+function handleError(error, provider) {
+    if (error.message?.includes('API key') || error.message?.includes('401') || error.message?.includes('Unauthorized')) {
+        const envVar = ENV_VAR_MAP[provider] || `${provider.toUpperCase()}_API_KEY`;
+        console.log(colors.red(`\nAuthentication error for ${provider}`));
+        console.log(colors.gray(`Set ${envVar} environment variable.`));
+    }
+    else if (error.message?.includes('429') || error.message?.includes('rate limit')) {
+        console.log(colors.yellow(`\nRate limited by ${provider}. Wait a moment and try again.`));
+    }
+    else {
+        console.log(colors.red(`\nError: ${error.message || error}`));
+    }
+}
+function printHeader(provider) {
+    console.log(colors.bold(colors.cyan('Rek AI Mode')));
+    console.log('');
+    console.log(`  ${colors.gray('Provider:')} ${colors.white(provider)}`);
+    console.log('');
+    console.log(`  ${colors.green('/switch')} ${colors.gray('<provider>')}  ${colors.gray('Change AI provider')}`);
+    console.log(`  ${colors.green('/clear')}              ${colors.gray('Clear conversation memory')}`);
+    console.log(`  ${colors.green('/memory')}             ${colors.gray('Show memory status')}`);
+    console.log(`  ${colors.green('/help')}               ${colors.gray('Show all commands')}`);
+    console.log('');
+    console.log(`  ${colors.yellow('ESC')} ${colors.gray('or')} ${colors.yellow('/exit')}          ${colors.gray('Exit AI mode')}`);
+    console.log('');
+    console.log(colors.gray('─'.repeat(50)));
+    console.log('');
+}
+function printHelp() {
+    console.log(`
+${colors.bold(colors.cyan('AI Mode Commands'))}
+
+  ${colors.green('/switch <provider>')}  Switch to another AI provider
+  ${colors.green('/clear')}              Clear conversation memory
+  ${colors.green('/memory')}             Show memory status
+  ${colors.green('/model')}              Show current model
+  ${colors.green('/providers')}          List available providers
+  ${colors.green('/help')}               Show this help
+  ${colors.green('/exit')}               Exit AI mode
+
+${colors.bold('Shortcuts')}
+  ${colors.gray('ESC')}                  Exit AI mode (or abort generation)
+  ${colors.gray('Ctrl+C')}               Exit AI mode
+
+${colors.bold('Available Providers')}
+  ${AI_PRESETS.join(', ')}
+`);
+}

package/dist/cli/tui/shell.js

@@ -623,12 +623,12 @@ export class RekShell {
     }
     async runAIChat(args) {
         const provider = args[0] || 'openai';
-        const model = args[1];
-        const envKeyName = provider === 'openai' ? 'OPENAI_API_KEY' : 'ANTHROPIC_API_KEY';
-        const apiKey = this.variables[envKeyName] || process.env[envKeyName];
         const { startAIChat } = await import('./ai-chat.js');
         await this.runInteractiveMode(async (rl) => {
-            await startAIChat(rl, provider, apiKey, model);
+            await startAIChat(rl, provider, {
+                aiClients: this.aiClients,
+                variables: this.variables
+            });
         });
     }
     async runAIPresetChat(presetName, message) {
@@ -4474,21 +4474,22 @@ ${colors.bold('Network:')}
                              ${colors.white('mode=realistic')} ${colors.gray('realistic | throughput | stress')}
                              ${colors.white('http2=false')}   ${colors.gray('Force HTTP/2')}
 
-    ${colors.green('chat <provider>')}     Start AI Chat.
-                             ${colors.gray('Providers:')} ${colors.white('openai')}, ${colors.white('anthropic')}
-                             ${colors.gray('Arg:')} ${colors.white('model=...')} (optional)
-
     ${colors.green('ws <url>')}            Start interactive WebSocket session.
     ${colors.green('udp <url>')}           Send UDP packet.
 
   ${colors.bold('AI Chat:')}
-    ${colors.green('@openai <message>')}   Chat with OpenAI (GPT) with memory.
-    ${colors.green('@anthropic <msg>')}    Chat with Anthropic (Claude) with memory.
-    ${colors.green('@groq <message>')}     Chat with Groq (fast inference).
-    ${colors.green('@google <message>')}   Chat with Google (Gemini).
-    ${colors.green('@xai <message>')}      Chat with xAI (Grok).
-    ${colors.green('@mistral <message>')}  Chat with Mistral AI.
-                             ${colors.gray('Memory:')} ${colors.white('12 pairs (24 messages)')} preserved per preset.
+    ${colors.green('ai [provider]')}       Enter AI mode (interactive conversation).
+                             ${colors.gray('Default: openai. Use /switch to change provider.')}
+                             ${colors.gray('Exit: ESC, Ctrl+C, or /exit')}
+                             ${colors.gray('Commands: /switch, /clear, /memory, /help')}
+
+    ${colors.green('@<provider> <msg>')}   Quick AI message (inline, no mode switch).
+                             ${colors.gray('Examples: @openai Hello!, @anthropic Explain this')}
+                             ${colors.gray('Providers: openai, anthropic, groq, google, xai,')}
+                             ${colors.gray('           mistral, cohere, deepseek, fireworks,')}
+                             ${colors.gray('           together, perplexity')}
+
+                             ${colors.gray('Memory:')} ${colors.white('12 pairs (24 messages)')} per provider.
                              ${colors.gray('Env:')} Set ${colors.white('OPENAI_API_KEY')}, ${colors.white('ANTHROPIC_API_KEY')}, etc.
     ${colors.green('ai:clear [preset]')}   Clear AI memory (all or specific preset).
 

package/dist/scrape/spider.d.ts

@@ -1,4 +1,5 @@
 import type { ExtractedLink } from './types.js';
+import { type SitemapUrl } from '../seo/validators/sitemap.js';
 export interface SpiderOptions {
     maxDepth?: number;
     maxPages?: number;
@@ -10,6 +11,8 @@ export interface SpiderOptions {
     include?: RegExp[];
     userAgent?: string;
     respectRobotsTxt?: boolean;
+    useSitemap?: boolean;
+    sitemapUrl?: string;
     onPage?: (result: SpiderPageResult) => void;
     onProgress?: (progress: SpiderProgress) => void;
 }
@@ -29,6 +32,29 @@ export interface SpiderProgress {
     currentUrl: string;
     depth: number;
 }
+export interface SitemapAnalysis {
+    found: boolean;
+    url?: string;
+    totalUrls: number;
+    crawledFromSitemap: number;
+    orphanUrls: string[];
+    missingFromSitemap: string[];
+    blockedBySitemapRobots: string[];
+    validationIssues: Array<{
+        type: string;
+        message: string;
+    }>;
+    sitemapUrls: SitemapUrl[];
+}
+export interface RobotsAnalysis {
+    found: boolean;
+    sitemaps: string[];
+    blocksAll: boolean;
+    issues: Array<{
+        type: string;
+        message: string;
+    }>;
+}
 export interface SpiderResult {
     startUrl: string;
     pages: SpiderPageResult[];
@@ -38,6 +64,8 @@ export interface SpiderResult {
         url: string;
         error: string;
     }>;
+    sitemap?: SitemapAnalysis;
+    robots?: RobotsAnalysis;
 }
 export declare class Spider {
     private options;
@@ -51,8 +79,17 @@ export declare class Spider {
     private running;
     private aborted;
     private pendingCount;
+    private sitemapUrls;
+    private sitemapUrlSet;
+    private robotsData;
+    private sitemapValidation;
+    private robotsValidation;
     constructor(options?: SpiderOptions);
     crawl(startUrl: string): Promise<SpiderResult>;
+    private fetchRobotsTxt;
+    private fetchSitemaps;
+    private buildSitemapAnalysis;
+    private buildRobotsAnalysis;
     private crawlPage;
     abort(): void;
     isRunning(): boolean;

package/dist/scrape/spider.js

@@ -1,6 +1,8 @@
 import { createClient } from '../core/client.js';
 import { ScrapeDocument } from './document.js';
 import { RequestPool } from '../utils/request-pool.js';
+import { discoverSitemaps, fetchAndValidateSitemap, } from '../seo/validators/sitemap.js';
+import { fetchAndValidateRobotsTxt, isPathAllowed, } from '../seo/validators/robots.js';
 const TRACKING_PARAMS = new Set([
     'utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content',
     'gclid', 'gclsrc', 'dclid',
@@ -84,6 +86,11 @@ export class Spider {
     running = false;
     aborted = false;
     pendingCount = 0;
+    sitemapUrls = [];
+    sitemapUrlSet = new Set();
+    robotsData = null;
+    sitemapValidation = null;
+    robotsValidation = null;
     constructor(options = {}) {
         this.options = {
             maxDepth: options.maxDepth ?? 5,
@@ -94,6 +101,8 @@ export class Spider {
             delay: options.delay ?? 100,
             userAgent: options.userAgent ?? 'Recker Spider/1.0',
             respectRobotsTxt: options.respectRobotsTxt ?? true,
+            useSitemap: options.useSitemap ?? false,
+            sitemapUrl: options.sitemapUrl,
             exclude: options.exclude,
             include: options.include,
             onPage: options.onPage,
@@ -117,6 +126,7 @@ export class Spider {
     async crawl(startUrl) {
         const startTime = performance.now();
         const normalizedStart = normalizeUrl(startUrl);
+        const baseUrl = new URL(normalizedStart).origin;
         this.baseHost = new URL(normalizedStart).hostname;
         this.visited.clear();
         this.queue = [];
@@ -125,8 +135,19 @@ export class Spider {
         this.running = true;
         this.aborted = false;
         this.pendingCount = 0;
+        this.sitemapUrls = [];
+        this.sitemapUrlSet.clear();
+        this.robotsData = null;
+        this.sitemapValidation = null;
+        this.robotsValidation = null;
+        if (this.options.respectRobotsTxt || this.options.useSitemap) {
+            await this.fetchRobotsTxt(baseUrl);
+        }
+        if (this.options.useSitemap) {
+            await this.fetchSitemaps(baseUrl);
+        }
         const pending = new Map();
-        const scheduleUrl = (item) => {
+        const scheduleUrl = (item, fromSitemap = false) => {
             const normalized = normalizeUrl(item.url);
             if (this.visited.has(normalized))
                 return;
@@ -136,6 +157,17 @@ export class Spider {
                 return;
             if (this.results.length + pending.size >= this.options.maxPages)
                 return;
+            if (this.options.respectRobotsTxt && this.robotsData) {
+                try {
+                    const urlPath = new URL(normalized).pathname;
+                    if (!isPathAllowed(this.robotsData, urlPath, this.options.userAgent)) {
+                        return;
+                    }
+                }
+                catch {
+                    return;
+                }
+            }
             this.visited.add(normalized);
             this.pendingCount++;
             const promise = this.pool.run(() => this.crawlPage({ ...item, url: normalized }))
@@ -146,6 +178,18 @@ export class Spider {
             pending.set(normalized, promise);
         };
         scheduleUrl({ url: normalizedStart, depth: 0 });
+        if (this.options.useSitemap && this.sitemapUrls.length > 0) {
+            for (const sitemapUrl of this.sitemapUrls) {
+                try {
+                    const urlHost = new URL(sitemapUrl.loc).hostname;
+                    if (urlHost === this.baseHost) {
+                        scheduleUrl({ url: sitemapUrl.loc, depth: 1 }, true);
+                    }
+                }
+                catch {
+                }
+            }
+        }
         while ((pending.size > 0 || this.queue.length > 0) && !this.aborted) {
             while (this.queue.length > 0 && !this.aborted) {
                 const item = this.queue.shift();
@@ -161,12 +205,154 @@ export class Spider {
             await Promise.all(pending.values());
         }
         this.running = false;
+        const sitemapAnalysis = this.buildSitemapAnalysis();
+        const robotsAnalysis = this.buildRobotsAnalysis();
         return {
             startUrl: normalizedStart,
             pages: this.results,
             visited: this.visited,
             duration: Math.round(performance.now() - startTime),
             errors: this.errors,
+            sitemap: this.options.useSitemap ? sitemapAnalysis : undefined,
+            robots: robotsAnalysis,
+        };
+    }
+    async fetchRobotsTxt(baseUrl) {
+        try {
+            const fetcher = async (url) => {
+                const response = await this.client.get(url);
+                return {
+                    status: response.status,
+                    text: await response.text(),
+                };
+            };
+            const result = await fetchAndValidateRobotsTxt(baseUrl, fetcher);
+            if (result.exists) {
+                this.robotsData = result.parseResult;
+                this.robotsValidation = {
+                    found: true,
+                    issues: result.issues.map(i => ({ type: i.type, message: i.message })),
+                };
+            }
+            else {
+                this.robotsValidation = {
+                    found: false,
+                    issues: [{ type: 'info', message: 'robots.txt not found' }],
+                };
+            }
+        }
+        catch (error) {
+            this.robotsValidation = {
+                found: false,
+                issues: [{ type: 'error', message: `Failed to fetch robots.txt: ${error instanceof Error ? error.message : 'Unknown error'}` }],
+            };
+        }
+    }
+    async fetchSitemaps(baseUrl) {
+        const fetcher = async (url) => {
+            const response = await this.client.get(url);
+            return {
+                status: response.status,
+                text: await response.text(),
+                headers: Object.fromEntries([...response.headers.entries()]),
+            };
+        };
+        try {
+            let sitemapUrls = [];
+            if (this.options.sitemapUrl) {
+                sitemapUrls = [this.options.sitemapUrl];
+            }
+            else if (this.robotsData?.sitemaps.length) {
+                sitemapUrls = this.robotsData.sitemaps;
+            }
+            else {
+                sitemapUrls = await discoverSitemaps(baseUrl, undefined, fetcher);
+            }
+            for (const sitemapUrl of sitemapUrls) {
+                try {
+                    const result = await fetchAndValidateSitemap(sitemapUrl, fetcher);
+                    if (result.exists && result.parseResult.valid) {
+                        this.sitemapValidation = result;
+                        if (result.parseResult.type === 'sitemapindex') {
+                            for (const childSitemap of result.parseResult.sitemaps) {
+                                try {
+                                    const childResult = await fetchAndValidateSitemap(childSitemap.loc, fetcher);
+                                    if (childResult.exists && childResult.parseResult.urls.length > 0) {
+                                        this.sitemapUrls.push(...childResult.parseResult.urls);
+                                    }
+                                }
+                                catch {
+                                }
+                            }
+                        }
+                        else {
+                            this.sitemapUrls.push(...result.parseResult.urls);
+                        }
+                    }
+                }
+                catch {
+                }
+            }
+            for (const url of this.sitemapUrls) {
+                this.sitemapUrlSet.add(normalizeUrl(url.loc));
+            }
+        }
+        catch (error) {
+        }
+    }
+    buildSitemapAnalysis() {
+        const crawledUrls = new Set(this.results.map(r => normalizeUrl(r.url)));
+        const crawledFromSitemap = this.sitemapUrls.filter(u => crawledUrls.has(normalizeUrl(u.loc))).length;
+        const linkedUrls = new Set();
+        for (const page of this.results) {
+            for (const link of page.links) {
+                if (link.href) {
+                    linkedUrls.add(normalizeUrl(link.href));
+                }
+            }
+        }
+        const orphanUrls = this.sitemapUrls
+            .filter(u => {
+            const normalized = normalizeUrl(u.loc);
+            return !linkedUrls.has(normalized) && crawledUrls.has(normalized);
+        })
+            .map(u => u.loc);
+        const missingFromSitemap = Array.from(crawledUrls)
+            .filter(url => !this.sitemapUrlSet.has(url));
+        const blockedBySitemapRobots = [];
+        if (this.robotsData) {
+            for (const sitemapUrl of this.sitemapUrls) {
+                try {
+                    const urlPath = new URL(sitemapUrl.loc).pathname;
+                    if (!isPathAllowed(this.robotsData, urlPath, this.options.userAgent)) {
+                        blockedBySitemapRobots.push(sitemapUrl.loc);
+                    }
+                }
+                catch {
+                }
+            }
+        }
+        return {
+            found: this.sitemapUrls.length > 0,
+            url: this.sitemapValidation?.parseResult ? undefined : undefined,
+            totalUrls: this.sitemapUrls.length,
+            crawledFromSitemap,
+            orphanUrls,
+            missingFromSitemap,
+            blockedBySitemapRobots,
+            validationIssues: this.sitemapValidation?.issues.map(i => ({
+                type: i.type,
+                message: i.message,
+            })) || [],
+            sitemapUrls: this.sitemapUrls,
+        };
+    }
+    buildRobotsAnalysis() {
+        return {
+            found: this.robotsValidation?.found ?? false,
+            sitemaps: this.robotsData?.sitemaps ?? [],
+            blocksAll: this.robotsData?.blocksAllRobots ?? false,
+            issues: this.robotsValidation?.issues ?? [],
         };
     }
     async crawlPage(item) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "recker",
-  "version": "1.0.33",
+  "version": "1.0.34-next.c0d9cdb",
   "description": "AI & DevX focused HTTP client for Node.js 18+",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",