recker 1.0.28 → 1.0.29

package/dist/cli/tui/shell.d.ts

@@ -51,6 +51,7 @@ export declare class RekShell {
     private runRDAP;
     private runPing;
     private runScrap;
+    private runSpider;
     private runSelect;
     private runSelectText;
     private runSelectAttr;

package/dist/cli/tui/shell.js

@@ -10,11 +10,12 @@ import { inspectTLS } from '../../utils/tls-inspector.js';
 import { getSecurityRecords } from '../../utils/dns-toolkit.js';
 import { rdap } from '../../utils/rdap.js';
 import { ScrapeDocument } from '../../scrape/document.js';
+import { Spider } from '../../scrape/spider.js';
 import colors from '../../utils/colors.js';
 import { getShellSearch } from './shell-search.js';
 import { openSearchPanel } from './search-panel.js';
 import { ScrollBuffer, parseScrollKey, parseMouseScroll, disableMouseReporting } from './scroll-buffer.js';
-import { analyzeSeo } from '../../seo/index.js';
+import { analyzeSeo, SeoSpider } from '../../seo/index.js';
 let highlight;
 async function initDependencies() {
     if (!highlight) {
@@ -94,7 +95,7 @@ export class RekShell {
             'get', 'post', 'put', 'delete', 'patch', 'head', 'options',
             'ws', 'udp', 'load', 'chat', 'ai',
             'whois', 'tls', 'ssl', 'security', 'ip', 'dns', 'dns:propagate', 'dns:email', 'rdap', 'ping',
-            'scrap', '$', '$text', '$attr', '$html', '$links', '$images', '$scripts', '$css', '$sourcemaps', '$unmap', '$unmap:view', '$unmap:save', '$beautify', '$beautify:save', '$table',
+            'scrap', 'spider', '$', '$text', '$attr', '$html', '$links', '$images', '$scripts', '$css', '$sourcemaps', '$unmap', '$unmap:view', '$unmap:save', '$beautify', '$beautify:save', '$table',
             '?', 'search', 'suggest', 'example',
             'help', 'clear', 'exit', 'set', 'url', 'vars', 'env'
         ];
@@ -368,6 +369,9 @@ export class RekShell {
             case 'scrap':
                 await this.runScrap(parts[1]);
                 return;
+            case 'spider':
+                await this.runSpider(parts.slice(1));
+                return;
             case '$':
                 await this.runSelect(parts.slice(1).join(' '));
                 return;
@@ -972,11 +976,20 @@ ${colors.bold('Details:')}`);
             const html = await res.text();
             const duration = Math.round(performance.now() - startTime);
             const report = await analyzeSeo(html, { baseUrl: url });
+            const t = res.timings;
+            report.timing = {
+                ttfb: t?.firstByte ? Math.round(t.firstByte) : undefined,
+                total: t?.total ? Math.round(t.total) : duration,
+                dns: t?.dns ? Math.round(t.dns) : undefined,
+                tcp: t?.tcp ? Math.round(t.tcp) : undefined,
+                tls: t?.tls ? Math.round(t.tls) : undefined,
+                download: t?.content ? Math.round(t.content) : undefined,
+            };
             if (jsonOutput) {
                 const jsonResult = {
                     url,
                     analyzedAt: new Date().toISOString(),
-                    durationMs: duration,
+                    timing: report.timing,
                     score: report.score,
                     grade: report.grade,
                     title: report.title,
@@ -985,8 +998,9 @@ ${colors.bold('Details:')}`);
                     headings: report.headings,
                     links: report.links,
                     images: report.images,
-                    openGraph: report.social.openGraph,
-                    twitterCard: report.social.twitterCard,
+                    openGraph: report.openGraph,
+                    twitterCard: report.twitterCard,
+                    social: report.social,
                     jsonLd: report.jsonLd,
                     technical: report.technical,
                     checks: report.checks,
@@ -1024,6 +1038,50 @@ Grade: ${gradeColor(colors.bold(report.grade))}  (${report.score}/100)
                     : report.metaDescription.text;
                 console.log(colors.bold('Description:') + ` ${desc} ` + colors.gray(`(${report.metaDescription.length} chars)`));
             }
+            if (report.openGraph && Object.values(report.openGraph).some(v => v)) {
+                console.log('');
+                console.log(colors.bold(colors.cyan('OpenGraph:')));
+                if (report.openGraph.title) {
+                    const ogTitle = report.openGraph.title.length > 60
+                        ? report.openGraph.title.slice(0, 57) + '...'
+                        : report.openGraph.title;
+                    console.log(`  ${colors.gray('og:title:')} ${ogTitle}`);
+                }
+                if (report.openGraph.description) {
+                    const ogDesc = report.openGraph.description.length > 60
+                        ? report.openGraph.description.slice(0, 57) + '...'
+                        : report.openGraph.description;
+                    console.log(`  ${colors.gray('og:description:')} ${ogDesc}`);
+                }
+                if (report.openGraph.image) {
+                    const ogImg = report.openGraph.image.length > 50
+                        ? '...' + report.openGraph.image.slice(-47)
+                        : report.openGraph.image;
+                    console.log(`  ${colors.gray('og:image:')} ${colors.blue(ogImg)}`);
+                }
+                if (report.openGraph.type) {
+                    console.log(`  ${colors.gray('og:type:')} ${report.openGraph.type}`);
+                }
+            }
+            if (report.timing) {
+                const t = report.timing;
+                console.log('');
+                console.log(colors.bold('Timing:'));
+                const timings = [];
+                if (t.dns !== undefined)
+                    timings.push(`DNS ${t.dns}ms`);
+                if (t.tcp !== undefined)
+                    timings.push(`TCP ${t.tcp}ms`);
+                if (t.tls !== undefined)
+                    timings.push(`TLS ${t.tls}ms`);
+                if (t.ttfb !== undefined)
+                    timings.push(`TTFB ${t.ttfb}ms`);
+                if (t.download !== undefined)
+                    timings.push(`Download ${t.download}ms`);
+                if (t.total !== undefined)
+                    timings.push(`Total ${t.total}ms`);
+                console.log(`  ${timings.join(' → ')}`);
+            }
             if (report.content) {
                 console.log(colors.bold('Content:') + ` ${report.content.wordCount} words, ${report.content.paragraphCount} paragraphs, ~${report.content.readingTimeMinutes} min read`);
             }
@@ -1434,6 +1492,274 @@ ${colors.bold('Network:')}
         }
         console.log('');
     }
+    async runSpider(args) {
+        let url = '';
+        let maxDepth = 3;
+        let maxPages = 100;
+        let concurrency = 5;
+        let seoEnabled = false;
+        let outputFile = '';
+        for (let i = 0; i < args.length; i++) {
+            const arg = args[i];
+            if (arg.startsWith('depth=')) {
+                maxDepth = parseInt(arg.split('=')[1]) || 4;
+            }
+            else if (arg.startsWith('limit=')) {
+                maxPages = parseInt(arg.split('=')[1]) || 100;
+            }
+            else if (arg.startsWith('concurrency=')) {
+                concurrency = parseInt(arg.split('=')[1]) || 5;
+            }
+            else if (arg === 'seo') {
+                seoEnabled = true;
+            }
+            else if (arg.startsWith('output=')) {
+                outputFile = arg.split('=')[1] || '';
+            }
+            else if (!arg.includes('=')) {
+                url = arg;
+            }
+        }
+        if (!url) {
+            if (!this.baseUrl) {
+                console.log(colors.yellow('Usage: spider <url> [options]'));
+                console.log(colors.gray('  Options:'));
+                console.log(colors.gray('    depth=4           Max crawl depth'));
+                console.log(colors.gray('    limit=100         Max pages to crawl'));
+                console.log(colors.gray('    concurrency=5     Concurrent requests'));
+                console.log(colors.gray('    seo               Enable SEO analysis'));
+                console.log(colors.gray('    output=file.json  Save JSON report'));
+                console.log(colors.gray('  Examples:'));
+                console.log(colors.gray('    spider example.com'));
+                console.log(colors.gray('    spider example.com depth=2 limit=50'));
+                console.log(colors.gray('    spider example.com seo output=seo-report.json'));
+                return;
+            }
+            url = this.baseUrl;
+        }
+        else if (!url.startsWith('http')) {
+            url = `https://${url}`;
+        }
+        console.log(colors.cyan(`\nSpider starting: ${url}`));
+        const modeLabel = seoEnabled ? colors.magenta(' + SEO') : '';
+        console.log(colors.gray(`  Depth: ${maxDepth} | Limit: ${maxPages} | Concurrency: ${concurrency}${modeLabel}`));
+        if (outputFile) {
+            console.log(colors.gray(`  Output: ${outputFile}`));
+        }
+        console.log('');
+        if (seoEnabled) {
+            const seoSpider = new SeoSpider({
+                maxDepth,
+                maxPages,
+                concurrency,
+                sameDomain: true,
+                delay: 100,
+                seo: true,
+                output: outputFile || undefined,
+                onProgress: (progress) => {
+                    process.stdout.write(`\r${colors.gray('  Crawling:')} ${colors.cyan(progress.crawled.toString())} pages | ${colors.gray('Queue:')} ${progress.queued} | ${colors.gray('Depth:')} ${progress.depth}   `);
+                },
+            });
+            try {
+                const result = await seoSpider.crawl(url);
+                process.stdout.write('\r' + ' '.repeat(80) + '\r');
+                console.log(colors.green(`\n✔ SEO Spider complete`) + colors.gray(` (${(result.duration / 1000).toFixed(1)}s)`));
+                console.log(`  ${colors.cyan('Pages crawled')}: ${result.pages.length}`);
+                console.log(`  ${colors.cyan('Unique URLs')}: ${result.visited.size}`);
+                console.log(`  ${colors.cyan('Avg SEO Score')}: ${result.summary.avgScore}/100`);
+                const responseTimes = result.pages.filter(p => p.duration > 0).map(p => p.duration);
+                const avgResponseTime = responseTimes.length > 0
+                    ? Math.round(responseTimes.reduce((a, b) => a + b, 0) / responseTimes.length)
+                    : 0;
+                const minResponseTime = responseTimes.length > 0 ? Math.min(...responseTimes) : 0;
+                const maxResponseTime = responseTimes.length > 0 ? Math.max(...responseTimes) : 0;
+                const reqPerSec = result.duration > 0 ? (result.pages.length / (result.duration / 1000)).toFixed(1) : '0';
+                const statusCounts = new Map();
+                for (const page of result.pages) {
+                    const status = page.status || 0;
+                    statusCounts.set(status, (statusCounts.get(status) || 0) + 1);
+                }
+                let totalInternalLinks = 0;
+                let totalExternalLinks = 0;
+                let totalImages = 0;
+                let imagesWithoutAlt = 0;
+                let pagesWithoutTitle = 0;
+                let pagesWithoutDescription = 0;
+                for (const page of result.pages) {
+                    if (page.seoReport) {
+                        totalInternalLinks += page.seoReport.links?.internal || 0;
+                        totalExternalLinks += page.seoReport.links?.external || 0;
+                        totalImages += page.seoReport.images?.total || 0;
+                        imagesWithoutAlt += page.seoReport.images?.withoutAlt || 0;
+                        if (!page.seoReport.title?.text)
+                            pagesWithoutTitle++;
+                        if (!page.seoReport.metaDescription?.text)
+                            pagesWithoutDescription++;
+                    }
+                }
+                console.log(colors.bold('\n  Performance:'));
+                console.log(`    ${colors.gray('Avg Response:')}  ${avgResponseTime}ms`);
+                console.log(`    ${colors.gray('Min/Max:')}       ${minResponseTime}ms / ${maxResponseTime}ms`);
+                console.log(`    ${colors.gray('Throughput:')}    ${reqPerSec} req/s`);
+                console.log(colors.bold('\n  HTTP Status:'));
+                const sortedStatuses = Array.from(statusCounts.entries()).sort((a, b) => b[1] - a[1]);
+                for (const [status, count] of sortedStatuses.slice(0, 5)) {
+                    const statusLabel = status === 0 ? 'Error' : status.toString();
+                    const statusColor = status >= 400 || status === 0 ? colors.red :
+                        status >= 300 ? colors.yellow : colors.green;
+                    const pct = ((count / result.pages.length) * 100).toFixed(0);
+                    console.log(`    ${statusColor(statusLabel.padEnd(5))} ${count.toString().padStart(3)} (${pct}%)`);
+                }
+                console.log(colors.bold('\n  Content:'));
+                console.log(`    ${colors.gray('Internal links:')} ${totalInternalLinks.toLocaleString()}`);
+                console.log(`    ${colors.gray('External links:')} ${totalExternalLinks.toLocaleString()}`);
+                console.log(`    ${colors.gray('Images:')}         ${totalImages.toLocaleString()} (${imagesWithoutAlt} missing alt)`);
+                console.log(`    ${colors.gray('Missing title:')}  ${pagesWithoutTitle}`);
+                console.log(`    ${colors.gray('Missing desc:')}   ${pagesWithoutDescription}`);
+                console.log(colors.bold('\n  SEO Summary:'));
+                const { summary } = result;
+                console.log(`    ${colors.red('✗')} Pages with errors:     ${summary.pagesWithErrors}`);
+                console.log(`    ${colors.yellow('⚠')} Pages with warnings:   ${summary.pagesWithWarnings}`);
+                console.log(`    ${colors.magenta('⚐')} Duplicate titles:      ${summary.duplicateTitles}`);
+                console.log(`    ${colors.magenta('⚐')} Duplicate descriptions:${summary.duplicateDescriptions}`);
+                console.log(`    ${colors.magenta('⚐')} Duplicate H1s:         ${summary.duplicateH1s}`);
+                console.log(`    ${colors.gray('○')} Orphan pages:          ${summary.orphanPages}`);
+                if (result.siteWideIssues.length > 0) {
+                    console.log(colors.bold('\n  Site-Wide Issues:'));
+                    for (const issue of result.siteWideIssues.slice(0, 10)) {
+                        const icon = issue.severity === 'error' ? colors.red('✗') :
+                            issue.severity === 'warning' ? colors.yellow('⚠') : colors.gray('○');
+                        console.log(`    ${icon} ${issue.message}`);
+                        if (issue.value) {
+                            const truncatedValue = issue.value.length > 50 ? issue.value.slice(0, 47) + '...' : issue.value;
+                            console.log(`      ${colors.gray(`"${truncatedValue}"`)}`);
+                        }
+                        const uniquePaths = [...new Set(issue.affectedUrls.map(u => new URL(u).pathname))];
+                        if (uniquePaths.length <= 3) {
+                            for (const path of uniquePaths) {
+                                console.log(`      ${colors.gray('→')} ${path}`);
+                            }
+                        }
+                        else {
+                            console.log(`      ${colors.gray(`→ ${uniquePaths.length} pages affected`)}`);
+                        }
+                    }
+                    if (result.siteWideIssues.length > 10) {
+                        console.log(colors.gray(`    ... and ${result.siteWideIssues.length - 10} more issues`));
+                    }
+                }
+                const pagesWithScores = result.pages
+                    .filter(p => p.seoReport)
+                    .sort((a, b) => (a.seoReport?.score || 0) - (b.seoReport?.score || 0));
+                const seenPaths = new Set();
+                const uniquePages = pagesWithScores.filter(page => {
+                    const path = new URL(page.url).pathname;
+                    if (seenPaths.has(path))
+                        return false;
+                    seenPaths.add(path);
+                    return true;
+                });
+                if (uniquePages.length > 0) {
+                    console.log(colors.bold('\n  Pages by SEO Score:'));
+                    const worstPages = uniquePages.slice(0, 5);
+                    for (const page of worstPages) {
+                        const score = page.seoReport?.score || 0;
+                        const grade = page.seoReport?.grade || '?';
+                        const path = new URL(page.url).pathname;
+                        const scoreColor = score >= 80 ? colors.green : score >= 60 ? colors.yellow : colors.red;
+                        console.log(`    ${scoreColor(`${score.toString().padStart(3)}`)} ${colors.gray(`[${grade}]`)} ${path.slice(0, 50)}`);
+                    }
+                    if (uniquePages.length > 5) {
+                        console.log(colors.gray(`    ... and ${uniquePages.length - 5} more pages`));
+                    }
+                }
+                if (outputFile) {
+                    console.log(colors.green(`\n  Report saved to: ${outputFile}`));
+                }
+                this.lastResponse = result;
+                console.log(colors.gray('\n  Result stored in lastResponse.'));
+            }
+            catch (error) {
+                console.error(colors.red(`SEO Spider failed: ${error.message}`));
+            }
+        }
+        else {
+            const spider = new Spider({
+                maxDepth,
+                maxPages,
+                concurrency,
+                sameDomain: true,
+                delay: 100,
+                onProgress: (progress) => {
+                    process.stdout.write(`\r${colors.gray('  Crawling:')} ${colors.cyan(progress.crawled.toString())} pages | ${colors.gray('Queue:')} ${progress.queued} | ${colors.gray('Depth:')} ${progress.depth}   `);
+                },
+            });
+            try {
+                const result = await spider.crawl(url);
+                process.stdout.write('\r' + ' '.repeat(80) + '\r');
+                console.log(colors.green(`\n✔ Spider complete`) + colors.gray(` (${(result.duration / 1000).toFixed(1)}s)`));
+                console.log(`  ${colors.cyan('Pages crawled')}: ${result.pages.length}`);
+                console.log(`  ${colors.cyan('Unique URLs')}: ${result.visited.size}`);
+                console.log(`  ${colors.cyan('Errors')}: ${result.errors.length}`);
+                const byDepth = new Map();
+                for (const page of result.pages) {
+                    byDepth.set(page.depth, (byDepth.get(page.depth) || 0) + 1);
+                }
+                console.log(colors.bold('\n  Pages by depth:'));
+                for (const [depth, count] of Array.from(byDepth.entries()).sort((a, b) => a[0] - b[0])) {
+                    const bar = '█'.repeat(Math.min(count, 40));
+                    console.log(`    ${colors.gray(`d${depth}:`)} ${bar} ${count}`);
+                }
+                const topPages = [...result.pages]
+                    .filter(p => !p.error)
+                    .sort((a, b) => b.links.length - a.links.length)
+                    .slice(0, 10);
+                if (topPages.length > 0) {
+                    console.log(colors.bold('\n  Top pages by outgoing links:'));
+                    for (const page of topPages) {
+                        const title = page.title.slice(0, 40) || new URL(page.url).pathname;
+                        console.log(`    ${colors.cyan(page.links.length.toString().padStart(3))} ${title}`);
+                    }
+                }
+                const formatError = (error) => {
+                    const statusMatch = error.match(/status code (\d{3})/i);
+                    if (statusMatch) {
+                        return `HTTP ${statusMatch[1]}`;
+                    }
+                    return error.length > 50 ? error.slice(0, 47) + '...' : error;
+                };
+                if (result.errors.length > 0 && result.errors.length <= 10) {
+                    console.log(colors.bold('\n  Errors:'));
+                    for (const err of result.errors) {
+                        const path = new URL(err.url).pathname;
+                        console.log(`    ${colors.red('✗')} ${path.padEnd(25)} ${colors.gray('→')} ${formatError(err.error)}`);
+                    }
+                }
+                else if (result.errors.length > 10) {
+                    console.log(colors.yellow(`\n  ${result.errors.length} errors (showing first 10):`));
+                    for (const err of result.errors.slice(0, 10)) {
+                        const path = new URL(err.url).pathname;
+                        console.log(`    ${colors.red('✗')} ${path.padEnd(25)} ${colors.gray('→')} ${formatError(err.error)}`);
+                    }
+                }
+                if (outputFile) {
+                    const reportData = {
+                        ...result,
+                        visited: Array.from(result.visited),
+                        generatedAt: new Date().toISOString(),
+                    };
+                    await fs.writeFile(outputFile, JSON.stringify(reportData, null, 2), 'utf-8');
+                    console.log(colors.green(`\n  Report saved to: ${outputFile}`));
+                }
+                this.lastResponse = result;
+                console.log(colors.gray('\n  Result stored in lastResponse. Use $links to explore.'));
+            }
+            catch (error) {
+                console.error(colors.red(`Spider failed: ${error.message}`));
+            }
+        }
+        console.log('');
+    }
     async runSelect(selector) {
         if (!this.currentDoc) {
             console.log(colors.yellow('No document loaded. Use "scrap <url>" first.'));
@@ -2358,6 +2684,13 @@ ${colors.bold('Network:')}
     ${colors.green('$beautify:save [f]')}  Save beautified code to file.
     ${colors.green('$table <selector>')}   Extract table as data.
 
+  ${colors.bold('Web Crawler:')}
+    ${colors.green('spider <url>')}        Crawl website following internal links.
+                             ${colors.gray('Options:')}
+                             ${colors.white('--depth=4')}     ${colors.gray('Maximum depth to crawl')}
+                             ${colors.white('--limit=100')}   ${colors.gray('Maximum pages to crawl')}
+                             ${colors.white('--concurrency=5')} ${colors.gray('Parallel requests')}
+
   ${colors.bold('Documentation:')}
     ${colors.green('? <query>')}           Search Recker documentation.
     ${colors.green('search <query>')}      Alias for ? (hybrid fuzzy+semantic search).
@@ -2375,6 +2708,7 @@ ${colors.bold('Network:')}
     › post /post name="Neo" active:=true role:Admin
     › load /heavy-endpoint users=100 mode=stress
     › chat openai gpt-5.1
+    › spider example.com depth=2 limit=50
     `);
     }
 }

package/dist/scrape/index.d.ts

@@ -1,4 +1,6 @@
 export { ScrapeDocument } from './document.js';
 export { ScrapeElement } from './element.js';
+export { Spider, spider } from './spider.js';
+export type { SpiderOptions, SpiderPageResult, SpiderProgress, SpiderResult, } from './spider.js';
 export { extractLinks, extractImages, extractMeta, extractOpenGraph, extractTwitterCard, extractJsonLd, extractForms, extractTables, extractScripts, extractStyles, } from './extractors.js';
 export type { ExtractedLink, ExtractedImage, ExtractedMeta, OpenGraphData, TwitterCardData, JsonLdData, ExtractedForm, ExtractedFormField, ExtractedTable, ExtractedScript, ExtractedStyle, ExtractionSchema, ExtractionSchemaField, ScrapeOptions, LinkExtractionOptions, ImageExtractionOptions, } from './types.js';

package/dist/scrape/index.js

@@ -1,3 +1,4 @@
 export { ScrapeDocument } from './document.js';
 export { ScrapeElement } from './element.js';
+export { Spider, spider } from './spider.js';
 export { extractLinks, extractImages, extractMeta, extractOpenGraph, extractTwitterCard, extractJsonLd, extractForms, extractTables, extractScripts, extractStyles, } from './extractors.js';

package/dist/scrape/spider.d.ts

@@ -0,0 +1,61 @@
+import type { ExtractedLink } from './types.js';
+export interface SpiderOptions {
+    maxDepth?: number;
+    maxPages?: number;
+    sameDomain?: boolean;
+    concurrency?: number;
+    timeout?: number;
+    delay?: number;
+    exclude?: RegExp[];
+    include?: RegExp[];
+    userAgent?: string;
+    respectRobotsTxt?: boolean;
+    onPage?: (result: SpiderPageResult) => void;
+    onProgress?: (progress: SpiderProgress) => void;
+}
+export interface SpiderPageResult {
+    url: string;
+    status: number;
+    title: string;
+    depth: number;
+    links: ExtractedLink[];
+    duration: number;
+    error?: string;
+}
+export interface SpiderProgress {
+    crawled: number;
+    queued: number;
+    total: number;
+    currentUrl: string;
+    depth: number;
+}
+export interface SpiderResult {
+    startUrl: string;
+    pages: SpiderPageResult[];
+    visited: Set<string>;
+    duration: number;
+    errors: Array<{
+        url: string;
+        error: string;
+    }>;
+}
+export declare class Spider {
+    private options;
+    private client;
+    private pool;
+    private visited;
+    private queue;
+    private results;
+    private errors;
+    private baseHost;
+    private running;
+    private aborted;
+    private pendingCount;
+    constructor(options?: SpiderOptions);
+    crawl(startUrl: string): Promise<SpiderResult>;
+    private crawlPage;
+    abort(): void;
+    isRunning(): boolean;
+    getProgress(): SpiderProgress;
+}
+export declare function spider(url: string, options?: SpiderOptions): Promise<SpiderResult>;