recker 1.0.28-next.4354f8c → 1.0.28-next.857660a

package/dist/cli/tui/shell.js

@@ -15,7 +15,7 @@ import colors from '../../utils/colors.js';
 import { getShellSearch } from './shell-search.js';
 import { openSearchPanel } from './search-panel.js';
 import { ScrollBuffer, parseScrollKey, parseMouseScroll, disableMouseReporting } from './scroll-buffer.js';
-import { analyzeSeo } from '../../seo/index.js';
+import { analyzeSeo, SeoSpider } from '../../seo/index.js';
 let highlight;
 async function initDependencies() {
     if (!highlight) {
@@ -972,15 +972,21 @@ ${colors.bold('Details:')}`);
         }
         const startTime = performance.now();
         try {
+            const ttfbStart = performance.now();
             const res = await this.client.get(url);
+            const ttfb = Math.round(performance.now() - ttfbStart);
             const html = await res.text();
             const duration = Math.round(performance.now() - startTime);
             const report = await analyzeSeo(html, { baseUrl: url });
+            report.timing = {
+                ttfb,
+                total: duration,
+            };
             if (jsonOutput) {
                 const jsonResult = {
                     url,
                     analyzedAt: new Date().toISOString(),
-                    durationMs: duration,
+                    timing: report.timing,
                     score: report.score,
                     grade: report.grade,
                     title: report.title,
@@ -989,8 +995,9 @@ ${colors.bold('Details:')}`);
                     headings: report.headings,
                     links: report.links,
                     images: report.images,
-                    openGraph: report.social.openGraph,
-                    twitterCard: report.social.twitterCard,
+                    openGraph: report.openGraph,
+                    twitterCard: report.twitterCard,
+                    social: report.social,
                     jsonLd: report.jsonLd,
                     technical: report.technical,
                     checks: report.checks,
@@ -1028,6 +1035,36 @@ Grade: ${gradeColor(colors.bold(report.grade))}  (${report.score}/100)
                     : report.metaDescription.text;
                 console.log(colors.bold('Description:') + ` ${desc} ` + colors.gray(`(${report.metaDescription.length} chars)`));
             }
+            if (report.openGraph && Object.values(report.openGraph).some(v => v)) {
+                console.log('');
+                console.log(colors.bold(colors.cyan('OpenGraph:')));
+                if (report.openGraph.title) {
+                    const ogTitle = report.openGraph.title.length > 60
+                        ? report.openGraph.title.slice(0, 57) + '...'
+                        : report.openGraph.title;
+                    console.log(`  ${colors.gray('og:title:')} ${ogTitle}`);
+                }
+                if (report.openGraph.description) {
+                    const ogDesc = report.openGraph.description.length > 60
+                        ? report.openGraph.description.slice(0, 57) + '...'
+                        : report.openGraph.description;
+                    console.log(`  ${colors.gray('og:description:')} ${ogDesc}`);
+                }
+                if (report.openGraph.image) {
+                    const ogImg = report.openGraph.image.length > 50
+                        ? '...' + report.openGraph.image.slice(-47)
+                        : report.openGraph.image;
+                    console.log(`  ${colors.gray('og:image:')} ${colors.blue(ogImg)}`);
+                }
+                if (report.openGraph.type) {
+                    console.log(`  ${colors.gray('og:type:')} ${report.openGraph.type}`);
+                }
+            }
+            if (report.timing?.ttfb !== undefined) {
+                console.log('');
+                console.log(colors.bold('Timing:') + ` TTFB ${report.timing.ttfb}ms` +
+                    (report.timing.total ? `, Total ${report.timing.total}ms` : ''));
+            }
             if (report.content) {
                 console.log(colors.bold('Content:') + ` ${report.content.wordCount} words, ${report.content.paragraphCount} paragraphs, ~${report.content.readingTimeMinutes} min read`);
             }
@@ -1443,18 +1480,26 @@ ${colors.bold('Network:')}
         let maxDepth = 3;
         let maxPages = 100;
         let concurrency = 5;
+        let seoEnabled = false;
+        let outputFile = '';
         for (let i = 0; i < args.length; i++) {
             const arg = args[i];
-            if (arg.startsWith('--depth=') || arg.startsWith('-d=')) {
-                maxDepth = parseInt(arg.split('=')[1]) || 3;
+            if (arg.startsWith('depth=')) {
+                maxDepth = parseInt(arg.split('=')[1]) || 4;
             }
-            else if (arg.startsWith('--limit=') || arg.startsWith('-l=')) {
+            else if (arg.startsWith('limit=')) {
                 maxPages = parseInt(arg.split('=')[1]) || 100;
             }
-            else if (arg.startsWith('--concurrency=') || arg.startsWith('-c=')) {
+            else if (arg.startsWith('concurrency=')) {
                 concurrency = parseInt(arg.split('=')[1]) || 5;
             }
-            else if (!arg.startsWith('-')) {
+            else if (arg === 'seo') {
+                seoEnabled = true;
+            }
+            else if (arg.startsWith('output=')) {
+                outputFile = arg.split('=')[1] || '';
+            }
+            else if (!arg.includes('=')) {
                 url = arg;
             }
         }
@@ -1462,12 +1507,15 @@ ${colors.bold('Network:')}
             if (!this.baseUrl) {
                 console.log(colors.yellow('Usage: spider <url> [options]'));
                 console.log(colors.gray('  Options:'));
-                console.log(colors.gray('    --depth=3      Max crawl depth'));
-                console.log(colors.gray('    --limit=100    Max pages to crawl'));
-                console.log(colors.gray('    --concurrency=5  Concurrent requests'));
+                console.log(colors.gray('    depth=4           Max crawl depth'));
+                console.log(colors.gray('    limit=100         Max pages to crawl'));
+                console.log(colors.gray('    concurrency=5     Concurrent requests'));
+                console.log(colors.gray('    seo               Enable SEO analysis'));
+                console.log(colors.gray('    output=file.json  Save JSON report'));
                 console.log(colors.gray('  Examples:'));
-                console.log(colors.gray('    spider https://example.com'));
-                console.log(colors.gray('    spider https://example.com --depth=2 --limit=50'));
+                console.log(colors.gray('    spider example.com'));
+                console.log(colors.gray('    spider example.com depth=2 limit=50'));
+                console.log(colors.gray('    spider example.com seo output=seo-report.json'));
                 return;
             }
             url = this.baseUrl;
@@ -1476,64 +1524,222 @@ ${colors.bold('Network:')}
             url = `https://${url}`;
         }
         console.log(colors.cyan(`\nSpider starting: ${url}`));
-        console.log(colors.gray(`  Depth: ${maxDepth} | Limit: ${maxPages} | Concurrency: ${concurrency}`));
+        const modeLabel = seoEnabled ? colors.magenta(' + SEO') : '';
+        console.log(colors.gray(`  Depth: ${maxDepth} | Limit: ${maxPages} | Concurrency: ${concurrency}${modeLabel}`));
+        if (outputFile) {
+            console.log(colors.gray(`  Output: ${outputFile}`));
+        }
         console.log('');
-        const spider = new Spider({
-            maxDepth,
-            maxPages,
-            concurrency,
-            sameDomain: true,
-            delay: 100,
-            onProgress: (progress) => {
-                process.stdout.write(`\r${colors.gray('  Crawling:')} ${colors.cyan(progress.crawled.toString())} pages | ${colors.gray('Queue:')} ${progress.queued} | ${colors.gray('Depth:')} ${progress.depth}   `);
-            },
-        });
-        try {
-            const result = await spider.crawl(url);
-            process.stdout.write('\r' + ' '.repeat(80) + '\r');
-            console.log(colors.green(`\n✔ Spider complete`) + colors.gray(` (${(result.duration / 1000).toFixed(1)}s)`));
-            console.log(`  ${colors.cyan('Pages crawled')}: ${result.pages.length}`);
-            console.log(`  ${colors.cyan('Unique URLs')}: ${result.visited.size}`);
-            console.log(`  ${colors.cyan('Errors')}: ${result.errors.length}`);
-            const byDepth = new Map();
-            for (const page of result.pages) {
-                byDepth.set(page.depth, (byDepth.get(page.depth) || 0) + 1);
-            }
-            console.log(colors.bold('\n  Pages by depth:'));
-            for (const [depth, count] of Array.from(byDepth.entries()).sort((a, b) => a[0] - b[0])) {
-                const bar = '█'.repeat(Math.min(count, 40));
-                console.log(`    ${colors.gray(`d${depth}:`)} ${bar} ${count}`);
-            }
-            const topPages = [...result.pages]
-                .filter(p => !p.error)
-                .sort((a, b) => b.links.length - a.links.length)
-                .slice(0, 10);
-            if (topPages.length > 0) {
-                console.log(colors.bold('\n  Top pages by outgoing links:'));
-                for (const page of topPages) {
-                    const title = page.title.slice(0, 40) || new URL(page.url).pathname;
-                    console.log(`    ${colors.cyan(page.links.length.toString().padStart(3))} ${title}`);
-                }
-            }
-            if (result.errors.length > 0 && result.errors.length <= 10) {
-                console.log(colors.bold('\n  Errors:'));
-                for (const err of result.errors) {
-                    const path = new URL(err.url).pathname;
-                    console.log(`    ${colors.red('✗')} ${path.slice(0, 40)} ${colors.gray('→')} ${err.error.slice(0, 30)}`);
-                }
-            }
-            else if (result.errors.length > 10) {
-                console.log(colors.yellow(`\n  ${result.errors.length} errors (showing first 10):`));
-                for (const err of result.errors.slice(0, 10)) {
-                    const path = new URL(err.url).pathname;
-                    console.log(`    ${colors.red('✗')} ${path.slice(0, 40)} ${colors.gray('→')} ${err.error.slice(0, 30)}`);
+        if (seoEnabled) {
+            const seoSpider = new SeoSpider({
+                maxDepth,
+                maxPages,
+                concurrency,
+                sameDomain: true,
+                delay: 100,
+                seo: true,
+                output: outputFile || undefined,
+                onProgress: (progress) => {
+                    process.stdout.write(`\r${colors.gray('  Crawling:')} ${colors.cyan(progress.crawled.toString())} pages | ${colors.gray('Queue:')} ${progress.queued} | ${colors.gray('Depth:')} ${progress.depth}   `);
+                },
+            });
+            try {
+                const result = await seoSpider.crawl(url);
+                process.stdout.write('\r' + ' '.repeat(80) + '\r');
+                console.log(colors.green(`\n✔ SEO Spider complete`) + colors.gray(` (${(result.duration / 1000).toFixed(1)}s)`));
+                console.log(`  ${colors.cyan('Pages crawled')}: ${result.pages.length}`);
+                console.log(`  ${colors.cyan('Unique URLs')}: ${result.visited.size}`);
+                console.log(`  ${colors.cyan('Avg SEO Score')}: ${result.summary.avgScore}/100`);
+                const responseTimes = result.pages.filter(p => p.duration > 0).map(p => p.duration);
+                const avgResponseTime = responseTimes.length > 0
+                    ? Math.round(responseTimes.reduce((a, b) => a + b, 0) / responseTimes.length)
+                    : 0;
+                const minResponseTime = responseTimes.length > 0 ? Math.min(...responseTimes) : 0;
+                const maxResponseTime = responseTimes.length > 0 ? Math.max(...responseTimes) : 0;
+                const reqPerSec = result.duration > 0 ? (result.pages.length / (result.duration / 1000)).toFixed(1) : '0';
+                const statusCounts = new Map();
+                for (const page of result.pages) {
+                    const status = page.status || 0;
+                    statusCounts.set(status, (statusCounts.get(status) || 0) + 1);
+                }
+                let totalInternalLinks = 0;
+                let totalExternalLinks = 0;
+                let totalImages = 0;
+                let imagesWithoutAlt = 0;
+                let pagesWithoutTitle = 0;
+                let pagesWithoutDescription = 0;
+                for (const page of result.pages) {
+                    if (page.seoReport) {
+                        totalInternalLinks += page.seoReport.links?.internal || 0;
+                        totalExternalLinks += page.seoReport.links?.external || 0;
+                        totalImages += page.seoReport.images?.total || 0;
+                        imagesWithoutAlt += page.seoReport.images?.withoutAlt || 0;
+                        if (!page.seoReport.title?.text)
+                            pagesWithoutTitle++;
+                        if (!page.seoReport.metaDescription?.text)
+                            pagesWithoutDescription++;
+                    }
+                }
+                console.log(colors.bold('\n  Performance:'));
+                console.log(`    ${colors.gray('Avg Response:')}  ${avgResponseTime}ms`);
+                console.log(`    ${colors.gray('Min/Max:')}       ${minResponseTime}ms / ${maxResponseTime}ms`);
+                console.log(`    ${colors.gray('Throughput:')}    ${reqPerSec} req/s`);
+                console.log(colors.bold('\n  HTTP Status:'));
+                const sortedStatuses = Array.from(statusCounts.entries()).sort((a, b) => b[1] - a[1]);
+                for (const [status, count] of sortedStatuses.slice(0, 5)) {
+                    const statusLabel = status === 0 ? 'Error' : status.toString();
+                    const statusColor = status >= 400 || status === 0 ? colors.red :
+                        status >= 300 ? colors.yellow : colors.green;
+                    const pct = ((count / result.pages.length) * 100).toFixed(0);
+                    console.log(`    ${statusColor(statusLabel.padEnd(5))} ${count.toString().padStart(3)} (${pct}%)`);
+                }
+                console.log(colors.bold('\n  Content:'));
+                console.log(`    ${colors.gray('Internal links:')} ${totalInternalLinks.toLocaleString()}`);
+                console.log(`    ${colors.gray('External links:')} ${totalExternalLinks.toLocaleString()}`);
+                console.log(`    ${colors.gray('Images:')}         ${totalImages.toLocaleString()} (${imagesWithoutAlt} missing alt)`);
+                console.log(`    ${colors.gray('Missing title:')}  ${pagesWithoutTitle}`);
+                console.log(`    ${colors.gray('Missing desc:')}   ${pagesWithoutDescription}`);
+                console.log(colors.bold('\n  SEO Summary:'));
+                const { summary } = result;
+                console.log(`    ${colors.red('✗')} Pages with errors:     ${summary.pagesWithErrors}`);
+                console.log(`    ${colors.yellow('⚠')} Pages with warnings:   ${summary.pagesWithWarnings}`);
+                console.log(`    ${colors.magenta('⚐')} Duplicate titles:      ${summary.duplicateTitles}`);
+                console.log(`    ${colors.magenta('⚐')} Duplicate descriptions:${summary.duplicateDescriptions}`);
+                console.log(`    ${colors.magenta('⚐')} Duplicate H1s:         ${summary.duplicateH1s}`);
+                console.log(`    ${colors.gray('○')} Orphan pages:          ${summary.orphanPages}`);
+                if (result.siteWideIssues.length > 0) {
+                    console.log(colors.bold('\n  Site-Wide Issues:'));
+                    for (const issue of result.siteWideIssues.slice(0, 10)) {
+                        const icon = issue.severity === 'error' ? colors.red('✗') :
+                            issue.severity === 'warning' ? colors.yellow('⚠') : colors.gray('○');
+                        console.log(`    ${icon} ${issue.message}`);
+                        if (issue.value) {
+                            const truncatedValue = issue.value.length > 50 ? issue.value.slice(0, 47) + '...' : issue.value;
+                            console.log(`      ${colors.gray(`"${truncatedValue}"`)}`);
+                        }
+                        const uniquePaths = [...new Set(issue.affectedUrls.map(u => new URL(u).pathname))];
+                        if (uniquePaths.length <= 3) {
+                            for (const path of uniquePaths) {
+                                console.log(`      ${colors.gray('→')} ${path}`);
+                            }
+                        }
+                        else {
+                            console.log(`      ${colors.gray(`→ ${uniquePaths.length} pages affected`)}`);
+                        }
+                    }
+                    if (result.siteWideIssues.length > 10) {
+                        console.log(colors.gray(`    ... and ${result.siteWideIssues.length - 10} more issues`));
+                    }
                 }
+                const pagesWithScores = result.pages
+                    .filter(p => p.seoReport)
+                    .sort((a, b) => (a.seoReport?.score || 0) - (b.seoReport?.score || 0));
+                const seenPaths = new Set();
+                const uniquePages = pagesWithScores.filter(page => {
+                    const path = new URL(page.url).pathname;
+                    if (seenPaths.has(path))
+                        return false;
+                    seenPaths.add(path);
+                    return true;
+                });
+                if (uniquePages.length > 0) {
+                    console.log(colors.bold('\n  Pages by SEO Score:'));
+                    const worstPages = uniquePages.slice(0, 5);
+                    for (const page of worstPages) {
+                        const score = page.seoReport?.score || 0;
+                        const grade = page.seoReport?.grade || '?';
+                        const path = new URL(page.url).pathname;
+                        const scoreColor = score >= 80 ? colors.green : score >= 60 ? colors.yellow : colors.red;
+                        console.log(`    ${scoreColor(`${score.toString().padStart(3)}`)} ${colors.gray(`[${grade}]`)} ${path.slice(0, 50)}`);
+                    }
+                    if (uniquePages.length > 5) {
+                        console.log(colors.gray(`    ... and ${uniquePages.length - 5} more pages`));
+                    }
+                }
+                if (outputFile) {
+                    console.log(colors.green(`\n  Report saved to: ${outputFile}`));
+                }
+                this.lastResponse = result;
+                console.log(colors.gray('\n  Result stored in lastResponse.'));
+            }
+            catch (error) {
+                console.error(colors.red(`SEO Spider failed: ${error.message}`));
             }
-            this.lastResponse = result;
-            console.log(colors.gray('\n  Result stored in lastResponse. Use $links to explore.'));
         }
-        catch (error) {
-            console.error(colors.red(`Spider failed: ${error.message}`));
+        else {
+            const spider = new Spider({
+                maxDepth,
+                maxPages,
+                concurrency,
+                sameDomain: true,
+                delay: 100,
+                onProgress: (progress) => {
+                    process.stdout.write(`\r${colors.gray('  Crawling:')} ${colors.cyan(progress.crawled.toString())} pages | ${colors.gray('Queue:')} ${progress.queued} | ${colors.gray('Depth:')} ${progress.depth}   `);
+                },
+            });
+            try {
+                const result = await spider.crawl(url);
+                process.stdout.write('\r' + ' '.repeat(80) + '\r');
+                console.log(colors.green(`\n✔ Spider complete`) + colors.gray(` (${(result.duration / 1000).toFixed(1)}s)`));
+                console.log(`  ${colors.cyan('Pages crawled')}: ${result.pages.length}`);
+                console.log(`  ${colors.cyan('Unique URLs')}: ${result.visited.size}`);
+                console.log(`  ${colors.cyan('Errors')}: ${result.errors.length}`);
+                const byDepth = new Map();
+                for (const page of result.pages) {
+                    byDepth.set(page.depth, (byDepth.get(page.depth) || 0) + 1);
+                }
+                console.log(colors.bold('\n  Pages by depth:'));
+                for (const [depth, count] of Array.from(byDepth.entries()).sort((a, b) => a[0] - b[0])) {
+                    const bar = '█'.repeat(Math.min(count, 40));
+                    console.log(`    ${colors.gray(`d${depth}:`)} ${bar} ${count}`);
+                }
+                const topPages = [...result.pages]
+                    .filter(p => !p.error)
+                    .sort((a, b) => b.links.length - a.links.length)
+                    .slice(0, 10);
+                if (topPages.length > 0) {
+                    console.log(colors.bold('\n  Top pages by outgoing links:'));
+                    for (const page of topPages) {
+                        const title = page.title.slice(0, 40) || new URL(page.url).pathname;
+                        console.log(`    ${colors.cyan(page.links.length.toString().padStart(3))} ${title}`);
+                    }
+                }
+                const formatError = (error) => {
+                    const statusMatch = error.match(/status code (\d{3})/i);
+                    if (statusMatch) {
+                        return `HTTP ${statusMatch[1]}`;
+                    }
+                    return error.length > 50 ? error.slice(0, 47) + '...' : error;
+                };
+                if (result.errors.length > 0 && result.errors.length <= 10) {
+                    console.log(colors.bold('\n  Errors:'));
+                    for (const err of result.errors) {
+                        const path = new URL(err.url).pathname;
+                        console.log(`    ${colors.red('✗')} ${path.padEnd(25)} ${colors.gray('→')} ${formatError(err.error)}`);
+                    }
+                }
+                else if (result.errors.length > 10) {
+                    console.log(colors.yellow(`\n  ${result.errors.length} errors (showing first 10):`));
+                    for (const err of result.errors.slice(0, 10)) {
+                        const path = new URL(err.url).pathname;
+                        console.log(`    ${colors.red('✗')} ${path.padEnd(25)} ${colors.gray('→')} ${formatError(err.error)}`);
+                    }
+                }
+                if (outputFile) {
+                    const reportData = {
+                        ...result,
+                        visited: Array.from(result.visited),
+                        generatedAt: new Date().toISOString(),
+                    };
+                    await fs.writeFile(outputFile, JSON.stringify(reportData, null, 2), 'utf-8');
+                    console.log(colors.green(`\n  Report saved to: ${outputFile}`));
+                }
+                this.lastResponse = result;
+                console.log(colors.gray('\n  Result stored in lastResponse. Use $links to explore.'));
+            }
+            catch (error) {
+                console.error(colors.red(`Spider failed: ${error.message}`));
+            }
         }
         console.log('');
     }
@@ -2464,7 +2670,7 @@ ${colors.bold('Network:')}
   ${colors.bold('Web Crawler:')}
     ${colors.green('spider <url>')}        Crawl website following internal links.
                              ${colors.gray('Options:')}
-                             ${colors.white('--depth=3')}     ${colors.gray('Maximum depth to crawl')}
+                             ${colors.white('--depth=4')}     ${colors.gray('Maximum depth to crawl')}
                              ${colors.white('--limit=100')}   ${colors.gray('Maximum pages to crawl')}
                              ${colors.white('--concurrency=5')} ${colors.gray('Parallel requests')}
 
@@ -2485,7 +2691,7 @@ ${colors.bold('Network:')}
     › post /post name="Neo" active:=true role:Admin
     › load /heavy-endpoint users=100 mode=stress
     › chat openai gpt-5.1
-    › spider https://example.com --depth=2 --limit=50
+    › spider example.com depth=2 limit=50
     `);
     }
 }

package/dist/scrape/spider.d.ts

@@ -42,6 +42,7 @@ export interface SpiderResult {
 export declare class Spider {
     private options;
     private client;
+    private pool;
     private visited;
     private queue;
     private results;
@@ -49,6 +50,7 @@ export declare class Spider {
     private baseHost;
     private running;
     private aborted;
+    private pendingCount;
     constructor(options?: SpiderOptions);
     crawl(startUrl: string): Promise<SpiderResult>;
     private crawlPage;

package/dist/scrape/spider.js

@@ -1,9 +1,30 @@
 import { createClient } from '../core/client.js';
 import { ScrapeDocument } from './document.js';
+import { RequestPool } from '../utils/request-pool.js';
+const TRACKING_PARAMS = new Set([
+    'utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content',
+    'gclid', 'gclsrc', 'dclid',
+    'fbclid', 'fb_action_ids', 'fb_action_types', 'fb_source', 'fb_ref',
+    'msclkid',
+    'twclid',
+    'ref', 'referer', 'referrer', 'source',
+    '_ga', '_gl', '_hsenc', '_hsmi',
+    'mc_cid', 'mc_eid',
+    'yclid', 'ymclid',
+    'igshid',
+    '_t', 't', 'timestamp', 'ts', 'nocache', 'cache',
+]);
 function normalizeUrl(urlStr) {
     try {
         const url = new URL(urlStr);
         url.hash = '';
+        const paramsToDelete = [];
+        url.searchParams.forEach((_, key) => {
+            if (TRACKING_PARAMS.has(key.toLowerCase())) {
+                paramsToDelete.push(key);
+            }
+        });
+        paramsToDelete.forEach(key => url.searchParams.delete(key));
         url.searchParams.sort();
         if (url.pathname !== '/' && url.pathname.endsWith('/')) {
             url.pathname = url.pathname.slice(0, -1);
@@ -54,6 +75,7 @@ function sleep(ms) {
 export class Spider {
     options;
     client;
+    pool;
     visited = new Set();
     queue = [];
     results = [];
@@ -61,9 +83,10 @@ export class Spider {
     baseHost = '';
     running = false;
     aborted = false;
+    pendingCount = 0;
     constructor(options = {}) {
         this.options = {
-            maxDepth: options.maxDepth ?? 3,
+            maxDepth: options.maxDepth ?? 4,
             maxPages: options.maxPages ?? 100,
             sameDomain: options.sameDomain ?? true,
             concurrency: options.concurrency ?? 5,
@@ -83,42 +106,60 @@ export class Spider {
                 'User-Agent': this.options.userAgent,
             },
         });
+        this.pool = new RequestPool({
+            concurrency: this.options.concurrency,
+            ...(this.options.delay > 0 ? {
+                requestsPerInterval: 1,
+                interval: this.options.delay,
+            } : {}),
+        });
     }
     async crawl(startUrl) {
         const startTime = performance.now();
         const normalizedStart = normalizeUrl(startUrl);
         this.baseHost = new URL(normalizedStart).hostname;
         this.visited.clear();
-        this.queue = [{ url: normalizedStart, depth: 0 }];
+        this.queue = [];
         this.results = [];
         this.errors = [];
         this.running = true;
         this.aborted = false;
-        while (this.queue.length > 0 && !this.aborted) {
-            if (this.results.length >= this.options.maxPages) {
-                break;
-            }
-            const batch = [];
-            while (batch.length < this.options.concurrency && this.queue.length > 0) {
+        this.pendingCount = 0;
+        const pending = new Map();
+        const scheduleUrl = (item) => {
+            const normalized = normalizeUrl(item.url);
+            if (this.visited.has(normalized))
+                return;
+            if (pending.has(normalized))
+                return;
+            if (item.depth > this.options.maxDepth)
+                return;
+            if (this.results.length + pending.size >= this.options.maxPages)
+                return;
+            this.visited.add(normalized);
+            this.pendingCount++;
+            const promise = this.pool.run(() => this.crawlPage({ ...item, url: normalized }))
+                .finally(() => {
+                pending.delete(normalized);
+                this.pendingCount--;
+            });
+            pending.set(normalized, promise);
+        };
+        scheduleUrl({ url: normalizedStart, depth: 0 });
+        while ((pending.size > 0 || this.queue.length > 0) && !this.aborted) {
+            while (this.queue.length > 0 && !this.aborted) {
                 const item = this.queue.shift();
-                const normalized = normalizeUrl(item.url);
-                if (this.visited.has(normalized)) {
-                    continue;
-                }
-                if (item.depth > this.options.maxDepth) {
-                    continue;
-                }
-                this.visited.add(normalized);
-                batch.push({ ...item, url: normalized });
-            }
-            if (batch.length === 0) {
-                continue;
+                if (this.results.length + pending.size >= this.options.maxPages)
+                    break;
+                scheduleUrl(item);
             }
-            await Promise.all(batch.map(item => this.crawlPage(item)));
-            if (this.options.delay > 0 && this.queue.length > 0) {
-                await sleep(this.options.delay);
+            if (pending.size > 0) {
+                await Promise.race(pending.values());
             }
         }
+        if (pending.size > 0) {
+            await Promise.all(pending.values());
+        }
         this.running = false;
         return {
             startUrl: normalizedStart,

package/dist/seo/analyzer.js

@@ -57,6 +57,21 @@ export class SeoAnalyzer {
             checks,
             title: meta.title ? { text: meta.title, length: meta.title.length } : undefined,
             metaDescription: meta.description ? { text: meta.description, length: meta.description.length } : undefined,
+            openGraph: Object.keys(og).length > 0 ? {
+                title: og.title,
+                description: og.description,
+                image: Array.isArray(og.image) ? og.image[0] : og.image,
+                url: og.url,
+                type: og.type,
+                siteName: og.siteName,
+            } : undefined,
+            twitterCard: Object.keys(twitter).length > 0 ? {
+                card: twitter.card,
+                title: twitter.title,
+                description: twitter.description,
+                image: Array.isArray(twitter.image) ? twitter.image[0] : twitter.image,
+                site: twitter.site,
+            } : undefined,
             headings: headings,
             content,
             links: linkAnalysis,

package/dist/seo/index.d.ts

@@ -1,5 +1,7 @@
 export { SeoAnalyzer, analyzeSeo } from './analyzer.js';
+export { SeoSpider, seoSpider } from './seo-spider.js';
+export type { SeoSpiderOptions, SeoPageResult, SiteWideIssue, SeoSpiderResult, } from './seo-spider.js';
 export { SeoRulesEngine, createRulesEngine, SEO_THRESHOLDS, ALL_SEO_RULES, } from './rules/index.js';
-export type { SeoReport, SeoCheckResult, SeoStatus, HeadingAnalysis, HeadingInfo, ContentMetrics, LinkAnalysis, ImageAnalysis, SocialMetaAnalysis, TechnicalSeo, SeoAnalyzerOptions, } from './types.js';
+export type { SeoReport, SeoCheckResult, SeoStatus, SeoTiming, HeadingAnalysis, HeadingInfo, ContentMetrics, LinkAnalysis, ImageAnalysis, SocialMetaAnalysis, TechnicalSeo, SeoAnalyzerOptions, } from './types.js';
 export type { SeoRule, RuleContext, RuleResult, RuleEvidence, RuleCategory, RuleSeverity, RulesEngineOptions, } from './rules/index.js';
 export type { SeoAnalyzerFullOptions } from './analyzer.js';

package/dist/seo/index.js

@@ -1,2 +1,3 @@
 export { SeoAnalyzer, analyzeSeo } from './analyzer.js';
+export { SeoSpider, seoSpider } from './seo-spider.js';
 export { SeoRulesEngine, createRulesEngine, SEO_THRESHOLDS, ALL_SEO_RULES, } from './rules/index.js';

package/dist/seo/seo-spider.d.ts

@@ -0,0 +1,47 @@
+import { SpiderOptions, SpiderResult, SpiderPageResult } from '../scrape/spider.js';
+import type { SeoReport } from './types.js';
+export interface SeoSpiderOptions extends SpiderOptions {
+    seo?: boolean;
+    output?: string;
+    onSeoAnalysis?: (result: SeoPageResult) => void;
+}
+export interface SeoPageResult extends SpiderPageResult {
+    seoReport?: SeoReport;
+}
+export interface SiteWideIssue {
+    type: 'duplicate-title' | 'duplicate-description' | 'duplicate-h1' | 'missing-canonical' | 'orphan-page';
+    severity: 'error' | 'warning' | 'info';
+    message: string;
+    affectedUrls: string[];
+    value?: string;
+}
+export interface SeoSpiderResult extends Omit<SpiderResult, 'pages'> {
+    pages: SeoPageResult[];
+    siteWideIssues: SiteWideIssue[];
+    summary: {
+        totalPages: number;
+        pagesWithErrors: number;
+        pagesWithWarnings: number;
+        avgScore: number;
+        duplicateTitles: number;
+        duplicateDescriptions: number;
+        duplicateH1s: number;
+        orphanPages: number;
+    };
+}
+export declare class SeoSpider {
+    private spider;
+    private options;
+    private seoResults;
+    constructor(options?: SeoSpiderOptions);
+    crawl(startUrl: string): Promise<SeoSpiderResult>;
+    private analyzePages;
+    private createReportFromPageData;
+    private detectSiteWideIssues;
+    private calculateSummary;
+    private scoreToGrade;
+    private saveReport;
+    abort(): void;
+    isRunning(): boolean;
+}
+export declare function seoSpider(url: string, options?: SeoSpiderOptions): Promise<SeoSpiderResult>;

package/dist/seo/seo-spider.js

@@ -0,0 +1,362 @@
+import { Spider } from '../scrape/spider.js';
+import { analyzeSeo } from './analyzer.js';
+import { createClient } from '../core/client.js';
+import * as fs from 'fs/promises';
+export class SeoSpider {
+    spider;
+    options;
+    seoResults = new Map();
+    constructor(options = {}) {
+        this.options = options;
+        this.spider = new Spider(options);
+    }
+    async crawl(startUrl) {
+        const result = await this.spider.crawl(startUrl);
+        if (!this.options.seo) {
+            return {
+                ...result,
+                pages: result.pages,
+                siteWideIssues: [],
+                summary: {
+                    totalPages: result.pages.length,
+                    pagesWithErrors: 0,
+                    pagesWithWarnings: 0,
+                    avgScore: 0,
+                    duplicateTitles: 0,
+                    duplicateDescriptions: 0,
+                    duplicateH1s: 0,
+                    orphanPages: 0,
+                },
+            };
+        }
+        const seoPages = await this.analyzePages(result.pages);
+        const siteWideIssues = this.detectSiteWideIssues(seoPages);
+        const summary = this.calculateSummary(seoPages, siteWideIssues);
+        const seoResult = {
+            ...result,
+            pages: seoPages,
+            siteWideIssues,
+            summary,
+        };
+        if (this.options.output) {
+            await this.saveReport(seoResult);
+        }
+        return seoResult;
+    }
+    async analyzePages(pages) {
+        const results = [];
+        const client = createClient({
+            timeout: this.options.timeout || 10000,
+            headers: {
+                'User-Agent': this.options.userAgent || 'Recker Spider/1.0',
+            },
+        });
+        for (const page of pages) {
+            if (page.error || page.status >= 400) {
+                results.push({
+                    ...page,
+                    seoReport: undefined,
+                });
+                continue;
+            }
+            try {
+                const response = await client.get(page.url);
+                const html = await response.text();
+                const seoReport = await analyzeSeo(html, { baseUrl: page.url });
+                const seoPage = {
+                    ...page,
+                    seoReport,
+                };
+                results.push(seoPage);
+                this.seoResults.set(page.url, seoReport);
+                this.options.onSeoAnalysis?.(seoPage);
+            }
+            catch {
+                results.push({
+                    ...page,
+                    seoReport: undefined,
+                });
+            }
+        }
+        return results;
+    }
+    createReportFromPageData(page) {
+        const checks = [];
+        if (page.title) {
+            const titleLength = page.title.length;
+            if (titleLength < 30) {
+                checks.push({
+                    name: 'Title Length',
+                    status: 'warn',
+                    message: `Title is too short (${titleLength} chars)`,
+                    value: titleLength,
+                    recommendation: 'Title should be 50-60 characters',
+                });
+            }
+            else if (titleLength > 60) {
+                checks.push({
+                    name: 'Title Length',
+                    status: 'warn',
+                    message: `Title is too long (${titleLength} chars)`,
+                    value: titleLength,
+                    recommendation: 'Title should be 50-60 characters',
+                });
+            }
+            else {
+                checks.push({
+                    name: 'Title Length',
+                    status: 'pass',
+                    message: `Good title length (${titleLength} chars)`,
+                    value: titleLength,
+                });
+            }
+        }
+        else {
+            checks.push({
+                name: 'Title',
+                status: 'fail',
+                message: 'Page has no title',
+                recommendation: 'Add a descriptive <title> tag',
+            });
+        }
+        const internalLinks = page.links.filter(l => l.type === 'internal').length;
+        const externalLinks = page.links.filter(l => l.type === 'external').length;
+        if (internalLinks === 0) {
+            checks.push({
+                name: 'Internal Links',
+                status: 'warn',
+                message: 'No internal links found',
+                recommendation: 'Add internal links to improve site structure',
+            });
+        }
+        else {
+            checks.push({
+                name: 'Internal Links',
+                status: 'pass',
+                message: `${internalLinks} internal links found`,
+                value: internalLinks,
+            });
+        }
+        const scoreSum = checks.reduce((sum, c) => {
+            if (c.status === 'pass')
+                return sum + 100;
+            if (c.status === 'warn')
+                return sum + 50;
+            return sum;
+        }, 0);
+        const score = checks.length > 0 ? Math.round(scoreSum / checks.length) : 0;
+        return {
+            url: page.url,
+            timestamp: new Date(),
+            grade: this.scoreToGrade(score),
+            score,
+            checks,
+            title: page.title ? { text: page.title, length: page.title.length } : undefined,
+            headings: {
+                structure: [],
+                h1Count: 0,
+                hasProperHierarchy: false,
+                issues: [],
+            },
+            content: {
+                wordCount: 0,
+                characterCount: 0,
+                sentenceCount: 0,
+                paragraphCount: 0,
+                readingTimeMinutes: 0,
+                avgWordsPerSentence: 0,
+                avgParagraphLength: 0,
+                listCount: 0,
+                strongTagCount: 0,
+                emTagCount: 0,
+            },
+            links: {
+                total: page.links.length,
+                internal: internalLinks,
+                external: externalLinks,
+                nofollow: 0,
+                broken: 0,
+                withoutText: page.links.filter(l => !l.text?.trim()).length,
+                sponsoredLinks: 0,
+                ugcLinks: 0,
+            },
+            images: {
+                total: 0,
+                withAlt: 0,
+                withoutAlt: 0,
+                lazy: 0,
+                missingDimensions: 0,
+                modernFormats: 0,
+                altTextLengths: [],
+                imageFilenames: [],
+                imagesWithAsyncDecoding: 0,
+            },
+            social: {
+                openGraph: {
+                    present: false,
+                    hasTitle: false,
+                    hasDescription: false,
+                    hasImage: false,
+                    hasUrl: false,
+                    issues: [],
+                },
+                twitterCard: {
+                    present: false,
+                    hasCard: false,
+                    hasTitle: false,
+                    hasDescription: false,
+                    hasImage: false,
+                    issues: [],
+                },
+            },
+            technical: {
+                hasCanonical: false,
+                hasRobotsMeta: false,
+                hasViewport: false,
+                hasCharset: false,
+                hasLang: false,
+            },
+            jsonLd: {
+                count: 0,
+                types: [],
+            },
+        };
+    }
+    detectSiteWideIssues(pages) {
+        const issues = [];
+        const titleGroups = new Map();
+        const descriptionGroups = new Map();
+        const h1Groups = new Map();
+        for (const page of pages) {
+            if (!page.seoReport)
+                continue;
+            const title = page.seoReport.title?.text?.trim();
+            if (title) {
+                const urls = titleGroups.get(title) || [];
+                urls.push(page.url);
+                titleGroups.set(title, urls);
+            }
+            const desc = page.seoReport.metaDescription?.text?.trim();
+            if (desc) {
+                const urls = descriptionGroups.get(desc) || [];
+                urls.push(page.url);
+                descriptionGroups.set(desc, urls);
+            }
+            const h1 = page.seoReport.headings?.structure?.find(h => h.level === 1)?.text?.trim();
+            if (h1) {
+                const urls = h1Groups.get(h1) || [];
+                urls.push(page.url);
+                h1Groups.set(h1, urls);
+            }
+        }
+        for (const [title, urls] of titleGroups) {
+            if (urls.length > 1) {
+                issues.push({
+                    type: 'duplicate-title',
+                    severity: 'error',
+                    message: `${urls.length} pages share the same title`,
+                    affectedUrls: urls,
+                    value: title,
+                });
+            }
+        }
+        for (const [desc, urls] of descriptionGroups) {
+            if (urls.length > 1) {
+                issues.push({
+                    type: 'duplicate-description',
+                    severity: 'warning',
+                    message: `${urls.length} pages share the same meta description`,
+                    affectedUrls: urls,
+                    value: desc,
+                });
+            }
+        }
+        for (const [h1, urls] of h1Groups) {
+            if (urls.length > 1) {
+                issues.push({
+                    type: 'duplicate-h1',
+                    severity: 'warning',
+                    message: `${urls.length} pages share the same H1 heading`,
+                    affectedUrls: urls,
+                    value: h1,
+                });
+            }
+        }
+        const linkedUrls = new Set();
+        for (const page of pages) {
+            for (const link of page.links) {
+                if (link.type === 'internal' && link.href) {
+                    linkedUrls.add(link.href);
+                }
+            }
+        }
+        const orphanPages = pages
+            .filter(p => !linkedUrls.has(p.url) && p.depth > 0)
+            .map(p => p.url);
+        if (orphanPages.length > 0) {
+            issues.push({
+                type: 'orphan-page',
+                severity: 'warning',
+                message: `${orphanPages.length} page(s) have no internal links pointing to them`,
+                affectedUrls: orphanPages,
+            });
+        }
+        return issues;
+    }
+    calculateSummary(pages, issues) {
+        const pagesWithSeo = pages.filter(p => p.seoReport);
+        const scores = pagesWithSeo.map(p => p.seoReport.score);
+        const avgScore = scores.length > 0
+            ? Math.round(scores.reduce((a, b) => a + b, 0) / scores.length)
+            : 0;
+        const pagesWithErrors = pagesWithSeo.filter(p => p.seoReport.checks.some(c => c.status === 'fail')).length;
+        const pagesWithWarnings = pagesWithSeo.filter(p => p.seoReport.checks.some(c => c.status === 'warn')).length;
+        const duplicateTitles = issues.filter(i => i.type === 'duplicate-title').length;
+        const duplicateDescriptions = issues.filter(i => i.type === 'duplicate-description').length;
+        const duplicateH1s = issues.filter(i => i.type === 'duplicate-h1').length;
+        const orphanPages = issues
+            .filter(i => i.type === 'orphan-page')
+            .reduce((sum, i) => sum + i.affectedUrls.length, 0);
+        return {
+            totalPages: pages.length,
+            pagesWithErrors,
+            pagesWithWarnings,
+            avgScore,
+            duplicateTitles,
+            duplicateDescriptions,
+            duplicateH1s,
+            orphanPages,
+        };
+    }
+    scoreToGrade(score) {
+        if (score >= 90)
+            return 'A';
+        if (score >= 80)
+            return 'B';
+        if (score >= 70)
+            return 'C';
+        if (score >= 60)
+            return 'D';
+        return 'F';
+    }
+    async saveReport(result) {
+        if (!this.options.output)
+            return;
+        const reportData = {
+            ...result,
+            visited: Array.from(result.visited),
+            generatedAt: new Date().toISOString(),
+        };
+        await fs.writeFile(this.options.output, JSON.stringify(reportData, null, 2), 'utf-8');
+    }
+    abort() {
+        this.spider.abort();
+    }
+    isRunning() {
+        return this.spider.isRunning();
+    }
+}
+export async function seoSpider(url, options) {
+    const spider = new SeoSpider(options);
+    return spider.crawl(url);
+}

package/dist/seo/types.d.ts

@@ -86,11 +86,20 @@ export interface TechnicalSeo {
     hasLang: boolean;
     langValue?: string;
 }
+export interface SeoTiming {
+    ttfb?: number;
+    total?: number;
+    dns?: number;
+    tcp?: number;
+    tls?: number;
+    download?: number;
+}
 export interface SeoReport {
     url: string;
     timestamp: Date;
     grade: string;
     score: number;
+    timing?: SeoTiming;
     checks: SeoCheckResult[];
     title?: {
         text: string;
@@ -100,6 +109,21 @@ export interface SeoReport {
         text: string;
         length: number;
     };
+    openGraph?: {
+        title?: string;
+        description?: string;
+        image?: string;
+        url?: string;
+        type?: string;
+        siteName?: string;
+    };
+    twitterCard?: {
+        card?: string;
+        title?: string;
+        description?: string;
+        image?: string;
+        site?: string;
+    };
     headings: HeadingAnalysis;
     content: ContentMetrics;
     links: LinkAnalysis;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "recker",
-  "version": "1.0.28-next.4354f8c",
+  "version": "1.0.28-next.857660a",
   "description": "AI & DevX focused HTTP client for Node.js 18+",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",