recker 1.0.28-next.dea5ff5 → 1.0.29

package/dist/cli/tui/shell.d.ts

@@ -51,6 +51,7 @@ export declare class RekShell {
     private runRDAP;
     private runPing;
     private runScrap;
+    private runSpider;
     private runSelect;
     private runSelectText;
     private runSelectAttr;

package/dist/cli/tui/shell.js

@@ -10,11 +10,12 @@ import { inspectTLS } from '../../utils/tls-inspector.js';
 import { getSecurityRecords } from '../../utils/dns-toolkit.js';
 import { rdap } from '../../utils/rdap.js';
 import { ScrapeDocument } from '../../scrape/document.js';
+import { Spider } from '../../scrape/spider.js';
 import colors from '../../utils/colors.js';
 import { getShellSearch } from './shell-search.js';
 import { openSearchPanel } from './search-panel.js';
 import { ScrollBuffer, parseScrollKey, parseMouseScroll, disableMouseReporting } from './scroll-buffer.js';
-import { analyzeSeo } from '../../seo/index.js';
+import { analyzeSeo, SeoSpider } from '../../seo/index.js';
 let highlight;
 async function initDependencies() {
     if (!highlight) {
@@ -94,7 +95,7 @@ export class RekShell {
             'get', 'post', 'put', 'delete', 'patch', 'head', 'options',
             'ws', 'udp', 'load', 'chat', 'ai',
             'whois', 'tls', 'ssl', 'security', 'ip', 'dns', 'dns:propagate', 'dns:email', 'rdap', 'ping',
-            'scrap', '$', '$text', '$attr', '$html', '$links', '$images', '$scripts', '$css', '$sourcemaps', '$unmap', '$unmap:view', '$unmap:save', '$beautify', '$beautify:save', '$table',
+            'scrap', 'spider', '$', '$text', '$attr', '$html', '$links', '$images', '$scripts', '$css', '$sourcemaps', '$unmap', '$unmap:view', '$unmap:save', '$beautify', '$beautify:save', '$table',
             '?', 'search', 'suggest', 'example',
             'help', 'clear', 'exit', 'set', 'url', 'vars', 'env'
         ];
@@ -368,6 +369,9 @@ export class RekShell {
             case 'scrap':
                 await this.runScrap(parts[1]);
                 return;
+            case 'spider':
+                await this.runSpider(parts.slice(1));
+                return;
             case '$':
                 await this.runSelect(parts.slice(1).join(' '));
                 return;
@@ -972,11 +976,20 @@ ${colors.bold('Details:')}`);
             const html = await res.text();
             const duration = Math.round(performance.now() - startTime);
             const report = await analyzeSeo(html, { baseUrl: url });
+            const t = res.timings;
+            report.timing = {
+                ttfb: t?.firstByte ? Math.round(t.firstByte) : undefined,
+                total: t?.total ? Math.round(t.total) : duration,
+                dns: t?.dns ? Math.round(t.dns) : undefined,
+                tcp: t?.tcp ? Math.round(t.tcp) : undefined,
+                tls: t?.tls ? Math.round(t.tls) : undefined,
+                download: t?.content ? Math.round(t.content) : undefined,
+            };
             if (jsonOutput) {
                 const jsonResult = {
                     url,
                     analyzedAt: new Date().toISOString(),
-                    durationMs: duration,
+                    timing: report.timing,
                     score: report.score,
                     grade: report.grade,
                     title: report.title,
@@ -985,8 +998,9 @@ ${colors.bold('Details:')}`);
                     headings: report.headings,
                     links: report.links,
                     images: report.images,
-                    openGraph: report.social.openGraph,
-                    twitterCard: report.social.twitterCard,
+                    openGraph: report.openGraph,
+                    twitterCard: report.twitterCard,
+                    social: report.social,
                     jsonLd: report.jsonLd,
                     technical: report.technical,
                     checks: report.checks,
@@ -1024,6 +1038,50 @@ Grade: ${gradeColor(colors.bold(report.grade))}  (${report.score}/100)
                     : report.metaDescription.text;
                 console.log(colors.bold('Description:') + ` ${desc} ` + colors.gray(`(${report.metaDescription.length} chars)`));
             }
+            if (report.openGraph && Object.values(report.openGraph).some(v => v)) {
+                console.log('');
+                console.log(colors.bold(colors.cyan('OpenGraph:')));
+                if (report.openGraph.title) {
+                    const ogTitle = report.openGraph.title.length > 60
+                        ? report.openGraph.title.slice(0, 57) + '...'
+                        : report.openGraph.title;
+                    console.log(`  ${colors.gray('og:title:')} ${ogTitle}`);
+                }
+                if (report.openGraph.description) {
+                    const ogDesc = report.openGraph.description.length > 60
+                        ? report.openGraph.description.slice(0, 57) + '...'
+                        : report.openGraph.description;
+                    console.log(`  ${colors.gray('og:description:')} ${ogDesc}`);
+                }
+                if (report.openGraph.image) {
+                    const ogImg = report.openGraph.image.length > 50
+                        ? '...' + report.openGraph.image.slice(-47)
+                        : report.openGraph.image;
+                    console.log(`  ${colors.gray('og:image:')} ${colors.blue(ogImg)}`);
+                }
+                if (report.openGraph.type) {
+                    console.log(`  ${colors.gray('og:type:')} ${report.openGraph.type}`);
+                }
+            }
+            if (report.timing) {
+                const t = report.timing;
+                console.log('');
+                console.log(colors.bold('Timing:'));
+                const timings = [];
+                if (t.dns !== undefined)
+                    timings.push(`DNS ${t.dns}ms`);
+                if (t.tcp !== undefined)
+                    timings.push(`TCP ${t.tcp}ms`);
+                if (t.tls !== undefined)
+                    timings.push(`TLS ${t.tls}ms`);
+                if (t.ttfb !== undefined)
+                    timings.push(`TTFB ${t.ttfb}ms`);
+                if (t.download !== undefined)
+                    timings.push(`Download ${t.download}ms`);
+                if (t.total !== undefined)
+                    timings.push(`Total ${t.total}ms`);
+                console.log(`  ${timings.join(' → ')}`);
+            }
             if (report.content) {
                 console.log(colors.bold('Content:') + ` ${report.content.wordCount} words, ${report.content.paragraphCount} paragraphs, ~${report.content.readingTimeMinutes} min read`);
             }
@@ -1434,6 +1492,274 @@ ${colors.bold('Network:')}
         }
         console.log('');
     }
+    async runSpider(args) {
+        let url = '';
+        let maxDepth = 3;
+        let maxPages = 100;
+        let concurrency = 5;
+        let seoEnabled = false;
+        let outputFile = '';
+        for (let i = 0; i < args.length; i++) {
+            const arg = args[i];
+            if (arg.startsWith('depth=')) {
+                maxDepth = parseInt(arg.split('=')[1]) || 4;
+            }
+            else if (arg.startsWith('limit=')) {
+                maxPages = parseInt(arg.split('=')[1]) || 100;
+            }
+            else if (arg.startsWith('concurrency=')) {
+                concurrency = parseInt(arg.split('=')[1]) || 5;
+            }
+            else if (arg === 'seo') {
+                seoEnabled = true;
+            }
+            else if (arg.startsWith('output=')) {
+                outputFile = arg.split('=')[1] || '';
+            }
+            else if (!arg.includes('=')) {
+                url = arg;
+            }
+        }
+        if (!url) {
+            if (!this.baseUrl) {
+                console.log(colors.yellow('Usage: spider <url> [options]'));
+                console.log(colors.gray('  Options:'));
+                console.log(colors.gray('    depth=4           Max crawl depth'));
+                console.log(colors.gray('    limit=100         Max pages to crawl'));
+                console.log(colors.gray('    concurrency=5     Concurrent requests'));
+                console.log(colors.gray('    seo               Enable SEO analysis'));
+                console.log(colors.gray('    output=file.json  Save JSON report'));
+                console.log(colors.gray('  Examples:'));
+                console.log(colors.gray('    spider example.com'));
+                console.log(colors.gray('    spider example.com depth=2 limit=50'));
+                console.log(colors.gray('    spider example.com seo output=seo-report.json'));
+                return;
+            }
+            url = this.baseUrl;
+        }
+        else if (!url.startsWith('http')) {
+            url = `https://${url}`;
+        }
+        console.log(colors.cyan(`\nSpider starting: ${url}`));
+        const modeLabel = seoEnabled ? colors.magenta(' + SEO') : '';
+        console.log(colors.gray(`  Depth: ${maxDepth} | Limit: ${maxPages} | Concurrency: ${concurrency}${modeLabel}`));
+        if (outputFile) {
+            console.log(colors.gray(`  Output: ${outputFile}`));
+        }
+        console.log('');
+        if (seoEnabled) {
+            const seoSpider = new SeoSpider({
+                maxDepth,
+                maxPages,
+                concurrency,
+                sameDomain: true,
+                delay: 100,
+                seo: true,
+                output: outputFile || undefined,
+                onProgress: (progress) => {
+                    process.stdout.write(`\r${colors.gray('  Crawling:')} ${colors.cyan(progress.crawled.toString())} pages | ${colors.gray('Queue:')} ${progress.queued} | ${colors.gray('Depth:')} ${progress.depth}   `);
+                },
+            });
+            try {
+                const result = await seoSpider.crawl(url);
+                process.stdout.write('\r' + ' '.repeat(80) + '\r');
+                console.log(colors.green(`\n✔ SEO Spider complete`) + colors.gray(` (${(result.duration / 1000).toFixed(1)}s)`));
+                console.log(`  ${colors.cyan('Pages crawled')}: ${result.pages.length}`);
+                console.log(`  ${colors.cyan('Unique URLs')}: ${result.visited.size}`);
+                console.log(`  ${colors.cyan('Avg SEO Score')}: ${result.summary.avgScore}/100`);
+                const responseTimes = result.pages.filter(p => p.duration > 0).map(p => p.duration);
+                const avgResponseTime = responseTimes.length > 0
+                    ? Math.round(responseTimes.reduce((a, b) => a + b, 0) / responseTimes.length)
+                    : 0;
+                const minResponseTime = responseTimes.length > 0 ? Math.min(...responseTimes) : 0;
+                const maxResponseTime = responseTimes.length > 0 ? Math.max(...responseTimes) : 0;
+                const reqPerSec = result.duration > 0 ? (result.pages.length / (result.duration / 1000)).toFixed(1) : '0';
+                const statusCounts = new Map();
+                for (const page of result.pages) {
+                    const status = page.status || 0;
+                    statusCounts.set(status, (statusCounts.get(status) || 0) + 1);
+                }
+                let totalInternalLinks = 0;
+                let totalExternalLinks = 0;
+                let totalImages = 0;
+                let imagesWithoutAlt = 0;
+                let pagesWithoutTitle = 0;
+                let pagesWithoutDescription = 0;
+                for (const page of result.pages) {
+                    if (page.seoReport) {
+                        totalInternalLinks += page.seoReport.links?.internal || 0;
+                        totalExternalLinks += page.seoReport.links?.external || 0;
+                        totalImages += page.seoReport.images?.total || 0;
+                        imagesWithoutAlt += page.seoReport.images?.withoutAlt || 0;
+                        if (!page.seoReport.title?.text)
+                            pagesWithoutTitle++;
+                        if (!page.seoReport.metaDescription?.text)
+                            pagesWithoutDescription++;
+                    }
+                }
+                console.log(colors.bold('\n  Performance:'));
+                console.log(`    ${colors.gray('Avg Response:')}  ${avgResponseTime}ms`);
+                console.log(`    ${colors.gray('Min/Max:')}       ${minResponseTime}ms / ${maxResponseTime}ms`);
+                console.log(`    ${colors.gray('Throughput:')}    ${reqPerSec} req/s`);
+                console.log(colors.bold('\n  HTTP Status:'));
+                const sortedStatuses = Array.from(statusCounts.entries()).sort((a, b) => b[1] - a[1]);
+                for (const [status, count] of sortedStatuses.slice(0, 5)) {
+                    const statusLabel = status === 0 ? 'Error' : status.toString();
+                    const statusColor = status >= 400 || status === 0 ? colors.red :
+                        status >= 300 ? colors.yellow : colors.green;
+                    const pct = ((count / result.pages.length) * 100).toFixed(0);
+                    console.log(`    ${statusColor(statusLabel.padEnd(5))} ${count.toString().padStart(3)} (${pct}%)`);
+                }
+                console.log(colors.bold('\n  Content:'));
+                console.log(`    ${colors.gray('Internal links:')} ${totalInternalLinks.toLocaleString()}`);
+                console.log(`    ${colors.gray('External links:')} ${totalExternalLinks.toLocaleString()}`);
+                console.log(`    ${colors.gray('Images:')}         ${totalImages.toLocaleString()} (${imagesWithoutAlt} missing alt)`);
+                console.log(`    ${colors.gray('Missing title:')}  ${pagesWithoutTitle}`);
+                console.log(`    ${colors.gray('Missing desc:')}   ${pagesWithoutDescription}`);
+                console.log(colors.bold('\n  SEO Summary:'));
+                const { summary } = result;
+                console.log(`    ${colors.red('✗')} Pages with errors:     ${summary.pagesWithErrors}`);
+                console.log(`    ${colors.yellow('⚠')} Pages with warnings:   ${summary.pagesWithWarnings}`);
+                console.log(`    ${colors.magenta('⚐')} Duplicate titles:      ${summary.duplicateTitles}`);
+                console.log(`    ${colors.magenta('⚐')} Duplicate descriptions:${summary.duplicateDescriptions}`);
+                console.log(`    ${colors.magenta('⚐')} Duplicate H1s:         ${summary.duplicateH1s}`);
+                console.log(`    ${colors.gray('○')} Orphan pages:          ${summary.orphanPages}`);
+                if (result.siteWideIssues.length > 0) {
+                    console.log(colors.bold('\n  Site-Wide Issues:'));
+                    for (const issue of result.siteWideIssues.slice(0, 10)) {
+                        const icon = issue.severity === 'error' ? colors.red('✗') :
+                            issue.severity === 'warning' ? colors.yellow('⚠') : colors.gray('○');
+                        console.log(`    ${icon} ${issue.message}`);
+                        if (issue.value) {
+                            const truncatedValue = issue.value.length > 50 ? issue.value.slice(0, 47) + '...' : issue.value;
+                            console.log(`      ${colors.gray(`"${truncatedValue}"`)}`);
+                        }
+                        const uniquePaths = [...new Set(issue.affectedUrls.map(u => new URL(u).pathname))];
+                        if (uniquePaths.length <= 3) {
+                            for (const path of uniquePaths) {
+                                console.log(`      ${colors.gray('→')} ${path}`);
+                            }
+                        }
+                        else {
+                            console.log(`      ${colors.gray(`→ ${uniquePaths.length} pages affected`)}`);
+                        }
+                    }
+                    if (result.siteWideIssues.length > 10) {
+                        console.log(colors.gray(`    ... and ${result.siteWideIssues.length - 10} more issues`));
+                    }
+                }
+                const pagesWithScores = result.pages
+                    .filter(p => p.seoReport)
+                    .sort((a, b) => (a.seoReport?.score || 0) - (b.seoReport?.score || 0));
+                const seenPaths = new Set();
+                const uniquePages = pagesWithScores.filter(page => {
+                    const path = new URL(page.url).pathname;
+                    if (seenPaths.has(path))
+                        return false;
+                    seenPaths.add(path);
+                    return true;
+                });
+                if (uniquePages.length > 0) {
+                    console.log(colors.bold('\n  Pages by SEO Score:'));
+                    const worstPages = uniquePages.slice(0, 5);
+                    for (const page of worstPages) {
+                        const score = page.seoReport?.score || 0;
+                        const grade = page.seoReport?.grade || '?';
+                        const path = new URL(page.url).pathname;
+                        const scoreColor = score >= 80 ? colors.green : score >= 60 ? colors.yellow : colors.red;
+                        console.log(`    ${scoreColor(`${score.toString().padStart(3)}`)} ${colors.gray(`[${grade}]`)} ${path.slice(0, 50)}`);
+                    }
+                    if (uniquePages.length > 5) {
+                        console.log(colors.gray(`    ... and ${uniquePages.length - 5} more pages`));
+                    }
+                }
+                if (outputFile) {
+                    console.log(colors.green(`\n  Report saved to: ${outputFile}`));
+                }
+                this.lastResponse = result;
+                console.log(colors.gray('\n  Result stored in lastResponse.'));
+            }
+            catch (error) {
+                console.error(colors.red(`SEO Spider failed: ${error.message}`));
+            }
+        }
+        else {
+            const spider = new Spider({
+                maxDepth,
+                maxPages,
+                concurrency,
+                sameDomain: true,
+                delay: 100,
+                onProgress: (progress) => {
+                    process.stdout.write(`\r${colors.gray('  Crawling:')} ${colors.cyan(progress.crawled.toString())} pages | ${colors.gray('Queue:')} ${progress.queued} | ${colors.gray('Depth:')} ${progress.depth}   `);
+                },
+            });
+            try {
+                const result = await spider.crawl(url);
+                process.stdout.write('\r' + ' '.repeat(80) + '\r');
+                console.log(colors.green(`\n✔ Spider complete`) + colors.gray(` (${(result.duration / 1000).toFixed(1)}s)`));
+                console.log(`  ${colors.cyan('Pages crawled')}: ${result.pages.length}`);
+                console.log(`  ${colors.cyan('Unique URLs')}: ${result.visited.size}`);
+                console.log(`  ${colors.cyan('Errors')}: ${result.errors.length}`);
+                const byDepth = new Map();
+                for (const page of result.pages) {
+                    byDepth.set(page.depth, (byDepth.get(page.depth) || 0) + 1);
+                }
+                console.log(colors.bold('\n  Pages by depth:'));
+                for (const [depth, count] of Array.from(byDepth.entries()).sort((a, b) => a[0] - b[0])) {
+                    const bar = '█'.repeat(Math.min(count, 40));
+                    console.log(`    ${colors.gray(`d${depth}:`)} ${bar} ${count}`);
+                }
+                const topPages = [...result.pages]
+                    .filter(p => !p.error)
+                    .sort((a, b) => b.links.length - a.links.length)
+                    .slice(0, 10);
+                if (topPages.length > 0) {
+                    console.log(colors.bold('\n  Top pages by outgoing links:'));
+                    for (const page of topPages) {
+                        const title = page.title.slice(0, 40) || new URL(page.url).pathname;
+                        console.log(`    ${colors.cyan(page.links.length.toString().padStart(3))} ${title}`);
+                    }
+                }
+                const formatError = (error) => {
+                    const statusMatch = error.match(/status code (\d{3})/i);
+                    if (statusMatch) {
+                        return `HTTP ${statusMatch[1]}`;
+                    }
+                    return error.length > 50 ? error.slice(0, 47) + '...' : error;
+                };
+                if (result.errors.length > 0 && result.errors.length <= 10) {
+                    console.log(colors.bold('\n  Errors:'));
+                    for (const err of result.errors) {
+                        const path = new URL(err.url).pathname;
+                        console.log(`    ${colors.red('✗')} ${path.padEnd(25)} ${colors.gray('→')} ${formatError(err.error)}`);
+                    }
+                }
+                else if (result.errors.length > 10) {
+                    console.log(colors.yellow(`\n  ${result.errors.length} errors (showing first 10):`));
+                    for (const err of result.errors.slice(0, 10)) {
+                        const path = new URL(err.url).pathname;
+                        console.log(`    ${colors.red('✗')} ${path.padEnd(25)} ${colors.gray('→')} ${formatError(err.error)}`);
+                    }
+                }
+                if (outputFile) {
+                    const reportData = {
+                        ...result,
+                        visited: Array.from(result.visited),
+                        generatedAt: new Date().toISOString(),
+                    };
+                    await fs.writeFile(outputFile, JSON.stringify(reportData, null, 2), 'utf-8');
+                    console.log(colors.green(`\n  Report saved to: ${outputFile}`));
+                }
+                this.lastResponse = result;
+                console.log(colors.gray('\n  Result stored in lastResponse. Use $links to explore.'));
+            }
+            catch (error) {
+                console.error(colors.red(`Spider failed: ${error.message}`));
+            }
+        }
+        console.log('');
+    }
     async runSelect(selector) {
         if (!this.currentDoc) {
             console.log(colors.yellow('No document loaded. Use "scrap <url>" first.'));
@@ -2358,6 +2684,13 @@ ${colors.bold('Network:')}
     ${colors.green('$beautify:save [f]')}  Save beautified code to file.
     ${colors.green('$table <selector>')}   Extract table as data.
 
+  ${colors.bold('Web Crawler:')}
+    ${colors.green('spider <url>')}        Crawl website following internal links.
+                             ${colors.gray('Options:')}
+                             ${colors.white('--depth=4')}     ${colors.gray('Maximum depth to crawl')}
+                             ${colors.white('--limit=100')}   ${colors.gray('Maximum pages to crawl')}
+                             ${colors.white('--concurrency=5')} ${colors.gray('Parallel requests')}
+
   ${colors.bold('Documentation:')}
     ${colors.green('? <query>')}           Search Recker documentation.
     ${colors.green('search <query>')}      Alias for ? (hybrid fuzzy+semantic search).
@@ -2375,6 +2708,7 @@ ${colors.bold('Network:')}
     › post /post name="Neo" active:=true role:Admin
     › load /heavy-endpoint users=100 mode=stress
     › chat openai gpt-5.1
+    › spider example.com depth=2 limit=50
     `);
     }
 }

package/dist/scrape/index.d.ts

@@ -1,4 +1,6 @@
 export { ScrapeDocument } from './document.js';
 export { ScrapeElement } from './element.js';
+export { Spider, spider } from './spider.js';
+export type { SpiderOptions, SpiderPageResult, SpiderProgress, SpiderResult, } from './spider.js';
 export { extractLinks, extractImages, extractMeta, extractOpenGraph, extractTwitterCard, extractJsonLd, extractForms, extractTables, extractScripts, extractStyles, } from './extractors.js';
 export type { ExtractedLink, ExtractedImage, ExtractedMeta, OpenGraphData, TwitterCardData, JsonLdData, ExtractedForm, ExtractedFormField, ExtractedTable, ExtractedScript, ExtractedStyle, ExtractionSchema, ExtractionSchemaField, ScrapeOptions, LinkExtractionOptions, ImageExtractionOptions, } from './types.js';

package/dist/scrape/index.js

@@ -1,3 +1,4 @@
 export { ScrapeDocument } from './document.js';
 export { ScrapeElement } from './element.js';
+export { Spider, spider } from './spider.js';
 export { extractLinks, extractImages, extractMeta, extractOpenGraph, extractTwitterCard, extractJsonLd, extractForms, extractTables, extractScripts, extractStyles, } from './extractors.js';

package/dist/scrape/spider.d.ts

@@ -0,0 +1,61 @@
+import type { ExtractedLink } from './types.js';
+export interface SpiderOptions {
+    maxDepth?: number;
+    maxPages?: number;
+    sameDomain?: boolean;
+    concurrency?: number;
+    timeout?: number;
+    delay?: number;
+    exclude?: RegExp[];
+    include?: RegExp[];
+    userAgent?: string;
+    respectRobotsTxt?: boolean;
+    onPage?: (result: SpiderPageResult) => void;
+    onProgress?: (progress: SpiderProgress) => void;
+}
+export interface SpiderPageResult {
+    url: string;
+    status: number;
+    title: string;
+    depth: number;
+    links: ExtractedLink[];
+    duration: number;
+    error?: string;
+}
+export interface SpiderProgress {
+    crawled: number;
+    queued: number;
+    total: number;
+    currentUrl: string;
+    depth: number;
+}
+export interface SpiderResult {
+    startUrl: string;
+    pages: SpiderPageResult[];
+    visited: Set<string>;
+    duration: number;
+    errors: Array<{
+        url: string;
+        error: string;
+    }>;
+}
+export declare class Spider {
+    private options;
+    private client;
+    private pool;
+    private visited;
+    private queue;
+    private results;
+    private errors;
+    private baseHost;
+    private running;
+    private aborted;
+    private pendingCount;
+    constructor(options?: SpiderOptions);
+    crawl(startUrl: string): Promise<SpiderResult>;
+    private crawlPage;
+    abort(): void;
+    isRunning(): boolean;
+    getProgress(): SpiderProgress;
+}
+export declare function spider(url: string, options?: SpiderOptions): Promise<SpiderResult>;

package/dist/scrape/spider.js

@@ -0,0 +1,250 @@
+import { createClient } from '../core/client.js';
+import { ScrapeDocument } from './document.js';
+import { RequestPool } from '../utils/request-pool.js';
+const TRACKING_PARAMS = new Set([
+    'utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content',
+    'gclid', 'gclsrc', 'dclid',
+    'fbclid', 'fb_action_ids', 'fb_action_types', 'fb_source', 'fb_ref',
+    'msclkid',
+    'twclid',
+    'ref', 'referer', 'referrer', 'source',
+    '_ga', '_gl', '_hsenc', '_hsmi',
+    'mc_cid', 'mc_eid',
+    'yclid', 'ymclid',
+    'igshid',
+    '_t', 't', 'timestamp', 'ts', 'nocache', 'cache',
+]);
+function normalizeUrl(urlStr) {
+    try {
+        const url = new URL(urlStr);
+        url.hash = '';
+        const paramsToDelete = [];
+        url.searchParams.forEach((_, key) => {
+            if (TRACKING_PARAMS.has(key.toLowerCase())) {
+                paramsToDelete.push(key);
+            }
+        });
+        paramsToDelete.forEach(key => url.searchParams.delete(key));
+        url.searchParams.sort();
+        if (url.pathname !== '/' && url.pathname.endsWith('/')) {
+            url.pathname = url.pathname.slice(0, -1);
+        }
+        return url.toString();
+    }
+    catch {
+        return urlStr;
+    }
+}
+function shouldCrawl(url, baseHost, options) {
+    try {
+        const parsed = new URL(url);
+        if (!['http:', 'https:'].includes(parsed.protocol)) {
+            return false;
+        }
+        if (options.sameDomain !== false && parsed.hostname !== baseHost) {
+            return false;
+        }
+        const skipExtensions = [
+            '.jpg', '.jpeg', '.png', '.gif', '.webp', '.svg', '.ico',
+            '.pdf', '.zip', '.tar', '.gz', '.rar',
+            '.mp3', '.mp4', '.avi', '.mov', '.webm',
+            '.css', '.js', '.json', '.xml', '.rss',
+            '.woff', '.woff2', '.ttf', '.eot',
+        ];
+        const pathname = parsed.pathname.toLowerCase();
+        if (skipExtensions.some(ext => pathname.endsWith(ext))) {
+            return false;
+        }
+        if (options.exclude?.some(pattern => pattern.test(url))) {
+            return false;
+        }
+        if (options.include?.length) {
+            if (!options.include.some(pattern => pattern.test(url))) {
+                return false;
+            }
+        }
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+function sleep(ms) {
+    return new Promise(resolve => setTimeout(resolve, ms));
+}
+export class Spider {
+    options;
+    client;
+    pool;
+    visited = new Set();
+    queue = [];
+    results = [];
+    errors = [];
+    baseHost = '';
+    running = false;
+    aborted = false;
+    pendingCount = 0;
+    constructor(options = {}) {
+        this.options = {
+            maxDepth: options.maxDepth ?? 4,
+            maxPages: options.maxPages ?? 100,
+            sameDomain: options.sameDomain ?? true,
+            concurrency: options.concurrency ?? 5,
+            timeout: options.timeout ?? 10000,
+            delay: options.delay ?? 100,
+            userAgent: options.userAgent ?? 'Recker Spider/1.0',
+            respectRobotsTxt: options.respectRobotsTxt ?? true,
+            exclude: options.exclude,
+            include: options.include,
+            onPage: options.onPage,
+            onProgress: options.onProgress,
+        };
+        this.client = createClient({
+            baseUrl: 'http://localhost',
+            timeout: this.options.timeout,
+            headers: {
+                'User-Agent': this.options.userAgent,
+            },
+        });
+        this.pool = new RequestPool({
+            concurrency: this.options.concurrency,
+            ...(this.options.delay > 0 ? {
+                requestsPerInterval: 1,
+                interval: this.options.delay,
+            } : {}),
+        });
+    }
+    async crawl(startUrl) {
+        const startTime = performance.now();
+        const normalizedStart = normalizeUrl(startUrl);
+        this.baseHost = new URL(normalizedStart).hostname;
+        this.visited.clear();
+        this.queue = [];
+        this.results = [];
+        this.errors = [];
+        this.running = true;
+        this.aborted = false;
+        this.pendingCount = 0;
+        const pending = new Map();
+        const scheduleUrl = (item) => {
+            const normalized = normalizeUrl(item.url);
+            if (this.visited.has(normalized))
+                return;
+            if (pending.has(normalized))
+                return;
+            if (item.depth > this.options.maxDepth)
+                return;
+            if (this.results.length + pending.size >= this.options.maxPages)
+                return;
+            this.visited.add(normalized);
+            this.pendingCount++;
+            const promise = this.pool.run(() => this.crawlPage({ ...item, url: normalized }))
+                .finally(() => {
+                pending.delete(normalized);
+                this.pendingCount--;
+            });
+            pending.set(normalized, promise);
+        };
+        scheduleUrl({ url: normalizedStart, depth: 0 });
+        while ((pending.size > 0 || this.queue.length > 0) && !this.aborted) {
+            while (this.queue.length > 0 && !this.aborted) {
+                const item = this.queue.shift();
+                if (this.results.length + pending.size >= this.options.maxPages)
+                    break;
+                scheduleUrl(item);
+            }
+            if (pending.size > 0) {
+                await Promise.race(pending.values());
+            }
+        }
+        if (pending.size > 0) {
+            await Promise.all(pending.values());
+        }
+        this.running = false;
+        return {
+            startUrl: normalizedStart,
+            pages: this.results,
+            visited: this.visited,
+            duration: Math.round(performance.now() - startTime),
+            errors: this.errors,
+        };
+    }
+    async crawlPage(item) {
+        const startTime = performance.now();
+        this.options.onProgress?.({
+            crawled: this.results.length,
+            queued: this.queue.length,
+            total: this.visited.size,
+            currentUrl: item.url,
+            depth: item.depth,
+        });
+        try {
+            const response = await this.client.get(item.url);
+            const status = response.status;
+            const contentType = response.headers.get('content-type') || '';
+            if (!contentType.includes('text/html')) {
+                return;
+            }
+            const html = await response.text();
+            const doc = await ScrapeDocument.create(html, { baseUrl: item.url });
+            const title = doc.selectFirst('title').text() || '';
+            const links = doc.links({ absolute: true });
+            const result = {
+                url: item.url,
+                status,
+                title,
+                depth: item.depth,
+                links,
+                duration: Math.round(performance.now() - startTime),
+            };
+            this.results.push(result);
+            this.options.onPage?.(result);
+            for (const link of links) {
+                if (!link.href)
+                    continue;
+                const normalized = normalizeUrl(link.href);
+                if (this.visited.has(normalized))
+                    continue;
+                if (!shouldCrawl(normalized, this.baseHost, this.options))
+                    continue;
+                this.queue.push({
+                    url: normalized,
+                    depth: item.depth + 1,
+                });
+            }
+        }
+        catch (error) {
+            const errorResult = {
+                url: item.url,
+                status: 0,
+                title: '',
+                depth: item.depth,
+                links: [],
+                duration: Math.round(performance.now() - startTime),
+                error: error.message,
+            };
+            this.results.push(errorResult);
+            this.errors.push({ url: item.url, error: error.message });
+            this.options.onPage?.(errorResult);
+        }
+    }
+    abort() {
+        this.aborted = true;
+    }
+    isRunning() {
+        return this.running;
+    }
+    getProgress() {
+        return {
+            crawled: this.results.length,
+            queued: this.queue.length,
+            total: this.visited.size,
+            currentUrl: '',
+            depth: 0,
+        };
+    }
+}
+export async function spider(url, options) {
+    const s = new Spider(options);
+    return s.crawl(url);
+}

package/dist/seo/analyzer.js

@@ -57,6 +57,21 @@ export class SeoAnalyzer {
             checks,
             title: meta.title ? { text: meta.title, length: meta.title.length } : undefined,
             metaDescription: meta.description ? { text: meta.description, length: meta.description.length } : undefined,
+            openGraph: Object.keys(og).length > 0 ? {
+                title: og.title,
+                description: og.description,
+                image: Array.isArray(og.image) ? og.image[0] : og.image,
+                url: og.url,
+                type: og.type,
+                siteName: og.siteName,
+            } : undefined,
+            twitterCard: Object.keys(twitter).length > 0 ? {
+                card: twitter.card,
+                title: twitter.title,
+                description: twitter.description,
+                image: Array.isArray(twitter.image) ? twitter.image[0] : twitter.image,
+                site: twitter.site,
+            } : undefined,
             headings: headings,
             content,
             links: linkAnalysis,

package/dist/seo/index.d.ts

@@ -1,5 +1,7 @@
 export { SeoAnalyzer, analyzeSeo } from './analyzer.js';
+export { SeoSpider, seoSpider } from './seo-spider.js';
+export type { SeoSpiderOptions, SeoPageResult, SiteWideIssue, SeoSpiderResult, } from './seo-spider.js';
 export { SeoRulesEngine, createRulesEngine, SEO_THRESHOLDS, ALL_SEO_RULES, } from './rules/index.js';
-export type { SeoReport, SeoCheckResult, SeoStatus, HeadingAnalysis, HeadingInfo, ContentMetrics, LinkAnalysis, ImageAnalysis, SocialMetaAnalysis, TechnicalSeo, SeoAnalyzerOptions, } from './types.js';
+export type { SeoReport, SeoCheckResult, SeoStatus, SeoTiming, HeadingAnalysis, HeadingInfo, ContentMetrics, LinkAnalysis, ImageAnalysis, SocialMetaAnalysis, TechnicalSeo, SeoAnalyzerOptions, } from './types.js';
 export type { SeoRule, RuleContext, RuleResult, RuleEvidence, RuleCategory, RuleSeverity, RulesEngineOptions, } from './rules/index.js';
 export type { SeoAnalyzerFullOptions } from './analyzer.js';

package/dist/seo/index.js

@@ -1,2 +1,3 @@
 export { SeoAnalyzer, analyzeSeo } from './analyzer.js';
+export { SeoSpider, seoSpider } from './seo-spider.js';
 export { SeoRulesEngine, createRulesEngine, SEO_THRESHOLDS, ALL_SEO_RULES, } from './rules/index.js';

package/dist/seo/seo-spider.d.ts

@@ -0,0 +1,47 @@
+import { SpiderOptions, SpiderResult, SpiderPageResult } from '../scrape/spider.js';
+import type { SeoReport } from './types.js';
+export interface SeoSpiderOptions extends SpiderOptions {
+    seo?: boolean;
+    output?: string;
+    onSeoAnalysis?: (result: SeoPageResult) => void;
+}
+export interface SeoPageResult extends SpiderPageResult {
+    seoReport?: SeoReport;
+}
+export interface SiteWideIssue {
+    type: 'duplicate-title' | 'duplicate-description' | 'duplicate-h1' | 'missing-canonical' | 'orphan-page';
+    severity: 'error' | 'warning' | 'info';
+    message: string;
+    affectedUrls: string[];
+    value?: string;
+}
+export interface SeoSpiderResult extends Omit<SpiderResult, 'pages'> {
+    pages: SeoPageResult[];
+    siteWideIssues: SiteWideIssue[];
+    summary: {
+        totalPages: number;
+        pagesWithErrors: number;
+        pagesWithWarnings: number;
+        avgScore: number;
+        duplicateTitles: number;
+        duplicateDescriptions: number;
+        duplicateH1s: number;
+        orphanPages: number;
+    };
+}
+export declare class SeoSpider {
+    private spider;
+    private options;
+    private seoResults;
+    constructor(options?: SeoSpiderOptions);
+    crawl(startUrl: string): Promise<SeoSpiderResult>;
+    private analyzePages;
+    private createReportFromPageData;
+    private detectSiteWideIssues;
+    private calculateSummary;
+    private scoreToGrade;
+    private saveReport;
+    abort(): void;
+    isRunning(): boolean;
+}
+export declare function seoSpider(url: string, options?: SeoSpiderOptions): Promise<SeoSpiderResult>;

package/dist/seo/seo-spider.js

@@ -0,0 +1,362 @@
+import { Spider } from '../scrape/spider.js';
+import { analyzeSeo } from './analyzer.js';
+import { createClient } from '../core/client.js';
+import * as fs from 'fs/promises';
+export class SeoSpider {
+    spider;
+    options;
+    seoResults = new Map();
+    constructor(options = {}) {
+        this.options = options;
+        this.spider = new Spider(options);
+    }
+    async crawl(startUrl) {
+        const result = await this.spider.crawl(startUrl);
+        if (!this.options.seo) {
+            return {
+                ...result,
+                pages: result.pages,
+                siteWideIssues: [],
+                summary: {
+                    totalPages: result.pages.length,
+                    pagesWithErrors: 0,
+                    pagesWithWarnings: 0,
+                    avgScore: 0,
+                    duplicateTitles: 0,
+                    duplicateDescriptions: 0,
+                    duplicateH1s: 0,
+                    orphanPages: 0,
+                },
+            };
+        }
+        const seoPages = await this.analyzePages(result.pages);
+        const siteWideIssues = this.detectSiteWideIssues(seoPages);
+        const summary = this.calculateSummary(seoPages, siteWideIssues);
+        const seoResult = {
+            ...result,
+            pages: seoPages,
+            siteWideIssues,
+            summary,
+        };
+        if (this.options.output) {
+            await this.saveReport(seoResult);
+        }
+        return seoResult;
+    }
+    async analyzePages(pages) {
+        const results = [];
+        const client = createClient({
+            timeout: this.options.timeout || 10000,
+            headers: {
+                'User-Agent': this.options.userAgent || 'Recker Spider/1.0',
+            },
+        });
+        for (const page of pages) {
+            if (page.error || page.status >= 400) {
+                results.push({
+                    ...page,
+                    seoReport: undefined,
+                });
+                continue;
+            }
+            try {
+                const response = await client.get(page.url);
+                const html = await response.text();
+                const seoReport = await analyzeSeo(html, { baseUrl: page.url });
+                const seoPage = {
+                    ...page,
+                    seoReport,
+                };
+                results.push(seoPage);
+                this.seoResults.set(page.url, seoReport);
+                this.options.onSeoAnalysis?.(seoPage);
+            }
+            catch {
+                results.push({
+                    ...page,
+                    seoReport: undefined,
+                });
+            }
+        }
+        return results;
+    }
+    createReportFromPageData(page) {
+        const checks = [];
+        if (page.title) {
+            const titleLength = page.title.length;
+            if (titleLength < 30) {
+                checks.push({
+                    name: 'Title Length',
+                    status: 'warn',
+                    message: `Title is too short (${titleLength} chars)`,
+                    value: titleLength,
+                    recommendation: 'Title should be 50-60 characters',
+                });
+            }
+            else if (titleLength > 60) {
+                checks.push({
+                    name: 'Title Length',
+                    status: 'warn',
+                    message: `Title is too long (${titleLength} chars)`,
+                    value: titleLength,
+                    recommendation: 'Title should be 50-60 characters',
+                });
+            }
+            else {
+                checks.push({
+                    name: 'Title Length',
+                    status: 'pass',
+                    message: `Good title length (${titleLength} chars)`,
+                    value: titleLength,
+                });
+            }
+        }
+        else {
+            checks.push({
+                name: 'Title',
+                status: 'fail',
+                message: 'Page has no title',
+                recommendation: 'Add a descriptive <title> tag',
+            });
+        }
+        const internalLinks = page.links.filter(l => l.type === 'internal').length;
+        const externalLinks = page.links.filter(l => l.type === 'external').length;
+        if (internalLinks === 0) {
+            checks.push({
+                name: 'Internal Links',
+                status: 'warn',
+                message: 'No internal links found',
+                recommendation: 'Add internal links to improve site structure',
+            });
+        }
+        else {
+            checks.push({
+                name: 'Internal Links',
+                status: 'pass',
+                message: `${internalLinks} internal links found`,
+                value: internalLinks,
+            });
+        }
+        const scoreSum = checks.reduce((sum, c) => {
+            if (c.status === 'pass')
+                return sum + 100;
+            if (c.status === 'warn')
+                return sum + 50;
+            return sum;
+        }, 0);
+        const score = checks.length > 0 ? Math.round(scoreSum / checks.length) : 0;
+        return {
+            url: page.url,
+            timestamp: new Date(),
+            grade: this.scoreToGrade(score),
+            score,
+            checks,
+            title: page.title ? { text: page.title, length: page.title.length } : undefined,
+            headings: {
+                structure: [],
+                h1Count: 0,
+                hasProperHierarchy: false,
+                issues: [],
+            },
+            content: {
+                wordCount: 0,
+                characterCount: 0,
+                sentenceCount: 0,
+                paragraphCount: 0,
+                readingTimeMinutes: 0,
+                avgWordsPerSentence: 0,
+                avgParagraphLength: 0,
+                listCount: 0,
+                strongTagCount: 0,
+                emTagCount: 0,
+            },
+            links: {
+                total: page.links.length,
+                internal: internalLinks,
+                external: externalLinks,
+                nofollow: 0,
+                broken: 0,
+                withoutText: page.links.filter(l => !l.text?.trim()).length,
+                sponsoredLinks: 0,
+                ugcLinks: 0,
+            },
+            images: {
+                total: 0,
+                withAlt: 0,
+                withoutAlt: 0,
+                lazy: 0,
+                missingDimensions: 0,
+                modernFormats: 0,
+                altTextLengths: [],
+                imageFilenames: [],
+                imagesWithAsyncDecoding: 0,
+            },
+            social: {
+                openGraph: {
+                    present: false,
+                    hasTitle: false,
+                    hasDescription: false,
+                    hasImage: false,
+                    hasUrl: false,
+                    issues: [],
+                },
+                twitterCard: {
+                    present: false,
+                    hasCard: false,
+                    hasTitle: false,
+                    hasDescription: false,
+                    hasImage: false,
+                    issues: [],
+                },
+            },
+            technical: {
+                hasCanonical: false,
+                hasRobotsMeta: false,
+                hasViewport: false,
+                hasCharset: false,
+                hasLang: false,
+            },
+            jsonLd: {
+                count: 0,
+                types: [],
+            },
+        };
+    }
+    detectSiteWideIssues(pages) {
+        const issues = [];
+        const titleGroups = new Map();
+        const descriptionGroups = new Map();
+        const h1Groups = new Map();
+        for (const page of pages) {
+            if (!page.seoReport)
+                continue;
+            const title = page.seoReport.title?.text?.trim();
+            if (title) {
+                const urls = titleGroups.get(title) || [];
+                urls.push(page.url);
+                titleGroups.set(title, urls);
+            }
+            const desc = page.seoReport.metaDescription?.text?.trim();
+            if (desc) {
+                const urls = descriptionGroups.get(desc) || [];
+                urls.push(page.url);
+                descriptionGroups.set(desc, urls);
+            }
+            const h1 = page.seoReport.headings?.structure?.find(h => h.level === 1)?.text?.trim();
+            if (h1) {
+                const urls = h1Groups.get(h1) || [];
+                urls.push(page.url);
+                h1Groups.set(h1, urls);
+            }
+        }
+        for (const [title, urls] of titleGroups) {
+            if (urls.length > 1) {
+                issues.push({
+                    type: 'duplicate-title',
+                    severity: 'error',
+                    message: `${urls.length} pages share the same title`,
+                    affectedUrls: urls,
+                    value: title,
+                });
+            }
+        }
+        for (const [desc, urls] of descriptionGroups) {
+            if (urls.length > 1) {
+                issues.push({
+                    type: 'duplicate-description',
+                    severity: 'warning',
+                    message: `${urls.length} pages share the same meta description`,
+                    affectedUrls: urls,
+                    value: desc,
+                });
+            }
+        }
+        for (const [h1, urls] of h1Groups) {
+            if (urls.length > 1) {
+                issues.push({
+                    type: 'duplicate-h1',
+                    severity: 'warning',
+                    message: `${urls.length} pages share the same H1 heading`,
+                    affectedUrls: urls,
+                    value: h1,
+                });
+            }
+        }
+        const linkedUrls = new Set();
+        for (const page of pages) {
+            for (const link of page.links) {
+                if (link.type === 'internal' && link.href) {
+                    linkedUrls.add(link.href);
+                }
+            }
+        }
+        const orphanPages = pages
+            .filter(p => !linkedUrls.has(p.url) && p.depth > 0)
+            .map(p => p.url);
+        if (orphanPages.length > 0) {
+            issues.push({
+                type: 'orphan-page',
+                severity: 'warning',
+                message: `${orphanPages.length} page(s) have no internal links pointing to them`,
+                affectedUrls: orphanPages,
+            });
+        }
+        return issues;
+    }
+    calculateSummary(pages, issues) {
+        const pagesWithSeo = pages.filter(p => p.seoReport);
+        const scores = pagesWithSeo.map(p => p.seoReport.score);
+        const avgScore = scores.length > 0
+            ? Math.round(scores.reduce((a, b) => a + b, 0) / scores.length)
+            : 0;
+        const pagesWithErrors = pagesWithSeo.filter(p => p.seoReport.checks.some(c => c.status === 'fail')).length;
+        const pagesWithWarnings = pagesWithSeo.filter(p => p.seoReport.checks.some(c => c.status === 'warn')).length;
+        const duplicateTitles = issues.filter(i => i.type === 'duplicate-title').length;
+        const duplicateDescriptions = issues.filter(i => i.type === 'duplicate-description').length;
+        const duplicateH1s = issues.filter(i => i.type === 'duplicate-h1').length;
+        const orphanPages = issues
+            .filter(i => i.type === 'orphan-page')
+            .reduce((sum, i) => sum + i.affectedUrls.length, 0);
+        return {
+            totalPages: pages.length,
+            pagesWithErrors,
+            pagesWithWarnings,
+            avgScore,
+            duplicateTitles,
+            duplicateDescriptions,
+            duplicateH1s,
+            orphanPages,
+        };
+    }
+    scoreToGrade(score) {
+        if (score >= 90)
+            return 'A';
+        if (score >= 80)
+            return 'B';
+        if (score >= 70)
+            return 'C';
+        if (score >= 60)
+            return 'D';
+        return 'F';
+    }
+    async saveReport(result) {
+        if (!this.options.output)
+            return;
+        const reportData = {
+            ...result,
+            visited: Array.from(result.visited),
+            generatedAt: new Date().toISOString(),
+        };
+        await fs.writeFile(this.options.output, JSON.stringify(reportData, null, 2), 'utf-8');
+    }
+    abort() {
+        this.spider.abort();
+    }
+    isRunning() {
+        return this.spider.isRunning();
+    }
+}
+export async function seoSpider(url, options) {
+    const spider = new SeoSpider(options);
+    return spider.crawl(url);
+}

package/dist/seo/types.d.ts

@@ -86,11 +86,20 @@ export interface TechnicalSeo {
     hasLang: boolean;
     langValue?: string;
 }
+export interface SeoTiming {
+    ttfb?: number;
+    total?: number;
+    dns?: number;
+    tcp?: number;
+    tls?: number;
+    download?: number;
+}
 export interface SeoReport {
     url: string;
     timestamp: Date;
     grade: string;
     score: number;
+    timing?: SeoTiming;
     checks: SeoCheckResult[];
     title?: {
         text: string;
@@ -100,6 +109,21 @@ export interface SeoReport {
         text: string;
         length: number;
     };
+    openGraph?: {
+        title?: string;
+        description?: string;
+        image?: string;
+        url?: string;
+        type?: string;
+        siteName?: string;
+    };
+    twitterCard?: {
+        card?: string;
+        title?: string;
+        description?: string;
+        image?: string;
+        site?: string;
+    };
     headings: HeadingAnalysis;
     content: ContentMetrics;
     links: LinkAnalysis;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "recker",
-  "version": "1.0.28-next.dea5ff5",
+  "version": "1.0.29",
   "description": "AI & DevX focused HTTP client for Node.js 18+",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",