recker 1.0.79 → 1.0.80-next.70ea84d

package/dist/cli/commands/server-runner.js

@@ -1,4 +1,5 @@
 import { CommandEmitter } from '../events/emitter.js';
+import { AbortError, ValidationError } from '../../core/errors.js';
 const DEFAULT_PORTS = {
     http: 3000,
     webhook: 3000,
@@ -56,7 +57,7 @@ export class ServerRunner extends CommandEmitter {
         this.emitStart(`${type} server`, { port, host });
         try {
             if (signal?.aborted) {
-                throw new Error('Aborted');
+                throw new AbortError('Server start aborted');
             }
             const abortHandler = () => {
                 this.aborted = true;
@@ -104,7 +105,7 @@ export class ServerRunner extends CommandEmitter {
             return result;
         }
         catch (err) {
-            if (signal?.aborted || this.aborted || err.message === 'Aborted') {
+            if (signal?.aborted || this.aborted || err instanceof AbortError) {
                 this.emitError('Server stopped by user', { code: 'ABORT' });
                 return {
                     type,
@@ -145,121 +146,99 @@ export class ServerRunner extends CommandEmitter {
     async createServer(type, options) {
         switch (type) {
             case 'http': {
-                const { MockHttpServer } = await import('../../testing/mock-http-server.js');
-                const server = await MockHttpServer.create({
+                const { createMockHttpServer } = await import('../../testing/index.js');
+                const server = await createMockHttpServer({
                     port: options.port,
                     host: options.host,
                     delay: options.delay,
                     cors: options.cors !== false,
                 });
                 if (options.echo) {
-                    server.any('/*', (req) => ({
+                    const echoHandler = (req) => ({
                         status: 200,
                         body: { method: req.method, path: req.path, query: req.query, headers: req.headers, body: req.body },
-                    }));
+                    });
+                    server.get('/*', echoHandler);
+                    server.post('/*', echoHandler);
+                    server.put('/*', echoHandler);
+                    server.patch('/*', echoHandler);
+                    server.delete('/*', echoHandler);
                 }
                 this.attachHttpEvents(server);
                 return server;
             }
             case 'webhook': {
-                const { createWebhookServer } = await import('../../testing/mock-http-server.js');
-                const server = await createWebhookServer({
-                    port: options.port,
-                    host: options.host,
-                    status: options.status || 204,
-                    log: false,
-                });
+                const { createMockHttpServer } = await import('../../testing/index.js');
+                const webhookStatus = options.status || 204;
+                const webhookHandler = () => ({ status: webhookStatus });
+                const server = await createMockHttpServer({ port: options.port, host: options.host });
+                server.get('/*', webhookHandler);
+                server.post('/*', webhookHandler);
+                server.put('/*', webhookHandler);
+                server.patch('/*', webhookHandler);
+                server.delete('/*', webhookHandler);
                 this.attachHttpEvents(server);
                 return server;
             }
             case 'websocket': {
-                const { MockWebSocketServer } = await import('../../testing/mock-websocket-server.js');
-                const server = await MockWebSocketServer.create({
+                const { createMockWebSocketServer } = await import('../../testing/index.js');
+                const server = await createMockWebSocketServer({
                     port: options.port,
                     host: options.host,
-                    echo: options.echo !== false,
-                    delay: options.delay,
                 });
+                if (options.echo === false) {
+                    server.setMessageHandler(() => undefined);
+                }
                 this.attachWebSocketEvents(server);
                 return server;
             }
             case 'sse': {
-                const { MockSSEServer } = await import('../../testing/mock-sse-server.js');
-                const server = await MockSSEServer.create({
+                const { createMockSSEServer } = await import('../../testing/index.js');
+                const server = await createMockSSEServer({
                     port: options.port,
                     host: options.host,
                     path: options.path || '/events',
                 });
                 if (options.heartbeat) {
-                    server.startPeriodicEvents('heartbeat', options.heartbeat);
+                    server.startPeriodicEvents('heartbeat', options.heartbeat, () => ({ data: 'heartbeat', event: 'heartbeat' }));
                 }
                 this.attachSSEEvents(server);
                 return server;
             }
             case 'hls': {
-                const { MockHlsServer } = await import('../../testing/mock-hls-server.js');
-                const http = await import('node:http');
-                const baseUrl = `http://${options.host}:${options.port}`;
-                const hlsServer = await MockHlsServer.create({
-                    baseUrl,
+                const { createMockHlsServer } = await import('../../testing/index.js');
+                const hlsServer = await createMockHlsServer({
+                    port: options.port,
+                    host: options.host,
                     mode: options.mode || 'vod',
                     segmentCount: options.segments || 10,
                     segmentDuration: options.duration || 6,
-                    multiQuality: true,
-                });
-                const httpServer = http.createServer(async (req, res) => {
-                    const url = `${baseUrl}${req.url}`;
-                    this.stats.totalRequests++;
-                    this.emit('data', {
-                        type: 'data',
-                        category: 'server:request',
-                        timestamp: Date.now(),
-                        command: 'server',
-                        level: 'debug',
-                        payload: { method: req.method, path: req.url },
-                    });
-                    try {
-                        const response = await hlsServer.transport.dispatch({ url, method: req.method || 'GET' });
-                        res.statusCode = response.status;
-                        response.headers.forEach((value, key) => res.setHeader(key, value));
-                        const body = await response.arrayBuffer();
-                        res.end(Buffer.from(body));
-                    }
-                    catch {
-                        res.statusCode = 404;
-                        res.end('Not Found');
-                    }
-                });
-                await new Promise((resolve) => {
-                    httpServer.listen(options.port, options.host, () => resolve());
                 });
                 return {
-                    port: options.port,
+                    port: hlsServer.port,
                     address: options.host,
-                    url: hlsServer.manifestUrl,
-                    isRunning: true,
-                    stop: async () => {
-                        httpServer.close();
-                        await hlsServer.stop();
-                    },
-                    on: () => this,
-                    emit: () => false,
+                    url: hlsServer.masterUrl,
+                    isRunning: hlsServer.isRunning,
+                    stop: () => hlsServer.stop(),
+                    on: hlsServer.on.bind(hlsServer),
+                    emit: hlsServer.emit.bind(hlsServer),
                 };
             }
             case 'udp': {
-                const { MockUDPServer } = await import('../../testing/mock-udp-server.js');
-                const server = new MockUDPServer({
+                const { createMockUdpServer } = await import('../../testing/index.js');
+                const server = await createMockUdpServer({
                     port: options.port,
                     host: options.host,
-                    echo: options.echo !== false,
                 });
-                await server.start();
+                if (options.echo === false) {
+                    server.setMessageHandler(() => undefined);
+                }
                 this.attachUDPEvents(server);
                 return server;
             }
             case 'dns': {
-                const { MockDnsServer } = await import('../../testing/mock-dns-server.js');
-                const server = await MockDnsServer.create({
+                const { createMockDnsServer } = await import('../../testing/index.js');
+                const server = await createMockDnsServer({
                     port: options.port,
                     host: options.host,
                     delay: options.delay,
@@ -268,41 +247,39 @@ export class ServerRunner extends CommandEmitter {
                 return server;
             }
             case 'whois': {
-                const { MockWhoisServer } = await import('../../testing/mock-whois-server.js');
-                const server = await MockWhoisServer.create({
+                const { createMockWhoisServer } = await import('../../testing/index.js');
+                const server = await createMockWhoisServer({
                     port: options.port,
                     host: options.host,
-                    delay: options.delay,
+                    responseDelay: options.delay,
                 });
                 this.attachWhoisEvents(server);
                 return server;
             }
             case 'telnet': {
-                const { MockTelnetServer } = await import('../../testing/mock-telnet-server.js');
-                const server = await MockTelnetServer.create({
+                const { createMockTelnetServer } = await import('../../testing/index.js');
+                const server = await createMockTelnetServer({
                     port: options.port,
                     host: options.host,
-                    echo: options.echo !== false,
-                    delay: options.delay,
+                    responseDelay: options.delay,
                 });
                 this.attachTelnetEvents(server);
                 return server;
             }
             case 'ftp': {
-                const { MockFtpServer } = await import('../../testing/mock-ftp-server.js');
-                const server = await MockFtpServer.create({
+                const { createMockFtpServer } = await import('../../testing/index.js');
+                const server = await createMockFtpServer({
                     port: options.port,
                     host: options.host,
-                    username: options.username || 'user',
-                    password: options.password || 'pass',
-                    anonymous: options.anonymous !== false,
-                    delay: options.delay,
                 });
                 this.attachFTPEvents(server);
                 return server;
             }
             default:
-                throw new Error(`Unknown server type: ${type}`);
+                throw new ValidationError(`Unknown server type: ${type}`, {
+                    field: 'serverType',
+                    value: type,
+                });
         }
     }
     async waitForStop(signal) {
@@ -441,7 +418,7 @@ export class ServerRunner extends CommandEmitter {
                 command: 'server',
                 level: 'debug',
                 payload: {
-                    domain: query.domain,
+                    domain: query.name,
                     recordType: query.type,
                 },
             });

package/dist/cli/commands/spider-runner.d.ts

@@ -1,4 +1,7 @@
 import { CommandEmitter } from '../events/emitter.js';
+import type { SeoReport } from '../../seo/types.js';
+import type { SpiderPageResult, SpiderTransport } from '../../scrape/spider.js';
+import type { SpiderFocusMode } from '../utils/option-helpers.js';
 export interface SpiderRunnerOptions {
     depth?: number;
     limit?: number;
@@ -6,7 +9,19 @@ export interface SpiderRunnerOptions {
     robots?: boolean;
     seo?: boolean;
     useSitemap?: boolean;
-    focus?: 'all' | 'links' | 'duplicates' | 'security' | 'ai' | 'resources';
+    transport?: SpiderTransport;
+    preferCurlFirst?: boolean;
+    timeout?: number;
+    delay?: number;
+    maxRetryAttempts?: number;
+    baseRetryDelayMs?: number;
+    maxRetryDelayMs?: number;
+    retryBackoffMultiplier?: number;
+    retryJitterMs?: number;
+    maxDomainBlockStrikes?: number;
+    rotateUserAgent?: boolean;
+    randomizeHeaders?: boolean;
+    focus?: SpiderFocusMode;
     signal?: AbortSignal;
     extract?: string[];
     include?: string[];
@@ -17,6 +32,19 @@ export interface SpiderRunnerResult {
     pagesVisited: number;
     duration: number;
     errors: number;
+    security?: {
+        pages: number;
+        blockedPages: number;
+        captchaPages: number;
+        captchaProviders: Record<string, number>;
+        attempts: number;
+        retries: number;
+        transportUsage: Record<Exclude<SpiderTransport, 'auto'>, number>;
+        avgAttempts: number;
+        avgTtfbMs?: number;
+        avgTotalMs?: number;
+        avgDownloadMs?: number;
+    };
     internalLinks: number;
     externalLinks: number;
     images: number;
@@ -29,10 +57,18 @@ export interface SpiderRunnerResult {
         depth: number;
         links: number;
         duration: number;
+        security?: SpiderPageResult['security'];
+        timings?: SpiderPageResult['timings'];
+        resources?: {
+            images: number;
+            scripts: number;
+            stylesheets: number;
+        };
         seoScore?: number;
         seoGrade?: string;
         seoErrors?: number;
         seoWarnings?: number;
+        seoReport?: SeoReport;
         extracted?: Record<string, unknown>;
     }>;
     extraction?: {

package/dist/cli/commands/spider-runner.js

@@ -1,10 +1,11 @@
 import { CommandEmitter } from '../events/emitter.js';
+import { AbortError } from '../../core/errors.js';
 export class SpiderRunner extends CommandEmitter {
     constructor() {
         super('spider');
     }
     async run(url, options = {}) {
-        const { depth = 5, limit = 100, concurrency = 5, robots = true, seo = false, focus = 'all', signal, extract, include, exclude, } = options;
+        const { depth = 5, limit = 100, concurrency = 5, robots = true, seo = false, focus = 'all', signal, extract, include, exclude, transport, preferCurlFirst, timeout, delay, maxRetryAttempts, baseRetryDelayMs, maxRetryDelayMs, retryBackoffMultiplier, retryJitterMs, maxDomainBlockStrikes, rotateUserAgent, randomizeHeaders, } = options;
         let startUrl = url;
         if (!startUrl.startsWith('http')) {
             startUrl = `https://${startUrl}`;
@@ -17,9 +18,67 @@ export class SpiderRunner extends CommandEmitter {
         let scripts = 0;
         let stylesheets = 0;
         let errorCount = 0;
+        let blockedPages = 0;
+        let captchaPages = 0;
+        let captchaProviders = {};
+        let attemptsTotal = 0;
+        let retriesTotal = 0;
+        let transportUsage = { undici: 0, curl: 0 };
+        let timingSum = { ttfb: 0, total: 0, download: 0 };
+        let timingCount = 0;
+        const collectPageMetrics = (page) => {
+            const resources = page.resources;
+            if (resources) {
+                images += resources.images || 0;
+                scripts += resources.scripts || 0;
+                stylesheets += resources.stylesheets || 0;
+            }
+            if (page.security) {
+                if (page.security.blocked)
+                    blockedPages += 1;
+                if (page.security.captchaDetected)
+                    captchaPages += 1;
+                if (page.security.captchaDetected) {
+                    const provider = (page.security.captchaProvider || '').trim().toLowerCase();
+                    const key = provider.length > 0 ? provider : 'unknown';
+                    captchaProviders[key] = (captchaProviders[key] || 0) + 1;
+                }
+                attemptsTotal += Number(page.security.attempts ?? 1);
+                retriesTotal += Number(page.security.retryCount ?? 0);
+                const transport = page.security.transport;
+                if (transport && transport !== 'auto') {
+                    transportUsage[transport] += 1;
+                }
+            }
+            else {
+                attemptsTotal += 1;
+            }
+            const ttfb = page.timings?.ttfb;
+            const total = page.timings?.total;
+            const download = page.timings?.download;
+            if (typeof ttfb === 'number' || typeof total === 'number' || typeof download === 'number') {
+                timingSum.ttfb += typeof ttfb === 'number' ? ttfb : 0;
+                timingSum.total += typeof total === 'number' ? total : 0;
+                timingSum.download += typeof download === 'number' ? download : 0;
+                timingCount += 1;
+            }
+        };
+        const getSecuritySummary = () => ({
+            pages: pages.length,
+            blockedPages,
+            captchaPages,
+            captchaProviders,
+            attempts: attemptsTotal,
+            retries: retriesTotal,
+            transportUsage,
+            avgAttempts: pages.length > 0 ? Number((attemptsTotal / pages.length).toFixed(2)) : 0,
+            avgTtfbMs: timingCount > 0 ? Math.round(timingSum.ttfb / timingCount) : undefined,
+            avgTotalMs: timingCount > 0 ? Math.round(timingSum.total / timingCount) : undefined,
+            avgDownloadMs: timingCount > 0 ? Math.round(timingSum.download / timingCount) : undefined,
+        });
         try {
             if (seo) {
-                return await this.runSeoSpider(startUrl, options, pages);
+                return await this.runSeoSpider(startUrl, options, pages, collectPageMetrics, getSecuritySummary);
             }
             const { Spider } = await import('../../scrape/spider.js');
             const extractionByPage = {};
@@ -28,10 +87,23 @@ export class SpiderRunner extends CommandEmitter {
                 maxPages: limit,
                 concurrency,
                 respectRobotsTxt: robots,
+                transport,
+                preferCurlFirst,
+                timeout,
+                delay,
+                maxRetryAttempts,
+                baseRetryDelayMs,
+                maxRetryDelayMs,
+                retryBackoffMultiplier,
+                retryJitterMs,
+                maxDomainBlockStrikes,
+                rotateUserAgent,
+                randomizeHeaders,
                 extract,
                 include: include?.map(p => new RegExp(p)),
                 exclude: exclude?.map(p => new RegExp(p)),
                 onPage: (page) => {
+                    collectPageMetrics(page);
                     pages.push({
                         url: page.url,
                         status: page.status,
@@ -39,6 +111,9 @@ export class SpiderRunner extends CommandEmitter {
                         depth: page.depth,
                         links: page.links?.length || 0,
                         duration: page.duration,
+                        security: page.security,
+                        timings: page.timings,
+                        resources: page.resources,
                         extracted: page.extracted,
                     });
                     if (page.extracted && Object.keys(page.extracted).length > 0) {
@@ -70,9 +145,12 @@ export class SpiderRunner extends CommandEmitter {
                         title: page.title,
                         links: page.links?.length || 0,
                         duration: page.duration,
+                        security: page.security,
+                        timings: page.timings,
+                        resources: page.resources,
                     });
                     if (signal?.aborted) {
-                        throw new Error('Aborted');
+                        throw new AbortError('Spider crawl aborted');
                     }
                 },
                 onProgress: (progress) => {
@@ -121,6 +199,7 @@ export class SpiderRunner extends CommandEmitter {
                 images,
                 scripts,
                 stylesheets,
+                security: getSecuritySummary(),
                 pages,
                 extraction,
             };
@@ -134,24 +213,25 @@ export class SpiderRunner extends CommandEmitter {
                 images,
                 scripts,
                 stylesheets,
+                security: getSecuritySummary(),
             });
             return summary;
         }
         catch (err) {
-            if (signal?.aborted || err.message === 'Aborted') {
+            if (signal?.aborted || err instanceof AbortError) {
                 this.emitError('Spider stopped by user', { code: 'ABORT' });
             }
             else {
-                this.emitError(err.message || 'Spider failed', {
-                    stack: err.stack,
+                this.emitError(err instanceof Error ? err.message : 'Spider failed', {
+                    stack: err instanceof Error ? err.stack : undefined,
                     context: startUrl,
                 });
             }
             throw err;
         }
     }
-    async runSeoSpider(url, options, pages) {
-        const { depth = 5, limit = 100, concurrency = 5, robots = true, focus = 'all', signal, useSitemap = true } = options;
+    async runSeoSpider(url, options, pages, collectPageMetrics, getSecuritySummary) {
+        const { depth = 5, limit = 100, concurrency = 5, robots = true, focus = 'all', signal, useSitemap = true, transport, preferCurlFirst, timeout, delay, maxRetryAttempts, baseRetryDelayMs, maxRetryDelayMs, retryBackoffMultiplier, retryJitterMs, maxDomainBlockStrikes, rotateUserAgent, randomizeHeaders, } = options;
         const focusCategories = {
             links: ['links'],
             duplicates: ['title', 'meta', 'content'],
@@ -171,7 +251,18 @@ export class SpiderRunner extends CommandEmitter {
             maxPages: limit,
             concurrency,
             sameDomain: true,
-            delay: 100,
+            delay,
+            timeout,
+            transport,
+            preferCurlFirst,
+            maxRetryAttempts,
+            baseRetryDelayMs,
+            maxRetryDelayMs,
+            retryBackoffMultiplier,
+            retryJitterMs,
+            maxDomainBlockStrikes,
+            rotateUserAgent,
+            randomizeHeaders,
             seo: true,
             respectRobotsTxt: robots,
             useSitemap,
@@ -186,7 +277,7 @@ export class SpiderRunner extends CommandEmitter {
                     depth: progress.depth,
                 });
                 if (signal?.aborted) {
-                    throw new Error('Aborted');
+                    throw new AbortError('Spider crawl aborted');
                 }
             },
         });
@@ -200,11 +291,16 @@ export class SpiderRunner extends CommandEmitter {
                 depth: page.depth,
                 links: page.links?.length || 0,
                 duration: page.duration,
+                security: page.security,
+                timings: page.timings,
+                resources: page.resources,
                 seoScore: seoReport?.score,
                 seoGrade: seoReport?.grade,
                 seoErrors: seoReport?.summary?.errors,
                 seoWarnings: seoReport?.summary?.warnings,
+                seoReport,
             });
+            collectPageMetrics(page);
             if (page.links) {
                 const pageHost = new URL(page.url).hostname;
                 for (const link of page.links) {
@@ -231,6 +327,9 @@ export class SpiderRunner extends CommandEmitter {
                 title: page.title,
                 links: page.links?.length || 0,
                 duration: page.duration,
+                security: page.security,
+                timings: page.timings,
+                resources: page.resources,
             });
         }
         const summary = {
@@ -243,6 +342,7 @@ export class SpiderRunner extends CommandEmitter {
             images,
             scripts,
             stylesheets,
+            security: getSecuritySummary(),
             pages,
             seo: result.summary ? {
                 avgScore: result.summary.avgScore,
@@ -273,6 +373,10 @@ export class SpiderRunner extends CommandEmitter {
             errors: result.errors?.length || 0,
             internalLinks,
             externalLinks,
+            images,
+            scripts,
+            stylesheets,
+            security: getSecuritySummary(),
         });
         return summary;
     }
@@ -340,6 +444,10 @@ export async function runSpiderWithEvents(url, options = {}) {
                 concurrency: spiderOptions.concurrency ?? 5,
                 seo: spiderOptions.seo ?? false,
                 extract: spiderOptions.extract,
+                transport: spiderOptions.transport,
+                preferCurlFirst: spiderOptions.preferCurlFirst,
+                timeout: spiderOptions.timeout,
+                delay: spiderOptions.delay,
             },
         });
         runner.on('spider:page', (event) => {
@@ -351,6 +459,12 @@ export async function runSpiderWithEvents(url, options = {}) {
                 title: event.title,
                 links: event.links,
                 duration: event.duration,
+                blocked: event.security?.blocked,
+                blockedByCaptcha: event.security?.blockedByCaptcha,
+                captchaDetected: event.security?.captchaDetected,
+                transport: event.security?.transport,
+                timing: event.timings,
+                resources: event.resources,
             });
         });
     }
@@ -369,10 +483,16 @@ export async function runSpiderWithEvents(url, options = {}) {
                 depth: page.depth,
                 links: page.links,
                 duration: page.duration,
+                security: page.security,
+                timings: page.timings,
+                resources: page.resources,
                 seoScore: page.seoScore,
                 seoGrade: page.seoGrade,
                 seoErrors: page.seoErrors,
                 seoWarnings: page.seoWarnings,
+                attempts: page.security?.attempts,
+                retryCount: page.security?.retryCount,
+                retryAfterMs: page.security?.retryAfterMs,
                 extracted: page.extracted,
             });
         }
@@ -386,6 +506,10 @@ export async function runSpiderWithEvents(url, options = {}) {
             internalLinks: result.internalLinks,
             externalLinks: result.externalLinks,
             seo: result.seo,
+            security: result.security,
+            images: result.images,
+            scripts: result.scripts,
+            stylesheets: result.stylesheets,
             extraction: result.extraction,
         });
         await jsonlWriter.close();

package/dist/cli/commands/spider.d.ts

@@ -1,11 +1,14 @@
 import { RekCommand as Command } from '../router.js';
+import { parseSpiderSerpConfig } from '../utils/serp-config.js';
+import { type SpiderTransport } from '../utils/option-helpers.js';
+type SpiderFocusMode = 'all' | 'links' | 'duplicates' | 'security' | 'ai' | 'resources';
 export interface SpiderOptions {
     url: string;
     depth?: number;
     limit?: number;
     concurrency?: number;
     output?: string;
-    focus?: string;
+    focus?: SpiderFocusMode;
     seo?: boolean;
     robots?: boolean;
     json?: boolean;
@@ -15,6 +18,20 @@ export interface SpiderOptions {
     exclude?: string | string[];
     noSitemap?: boolean;
     disableTui?: boolean;
+    transport?: SpiderTransport;
+    preferCurlFirst?: boolean;
+    timeout?: number;
+    delay?: number;
+    maxRetryAttempts?: number;
+    baseRetryDelayMs?: number;
+    maxRetryDelayMs?: number;
+    retryBackoffMultiplier?: number;
+    retryJitterMs?: number;
+    maxDomainBlockStrikes?: number;
+    rotateUserAgent?: boolean;
+    randomizeHeaders?: boolean;
+    serpConfig?: ReturnType<typeof parseSpiderSerpConfig>;
 }
 export declare function runSpider(opts: SpiderOptions): Promise<void>;
 export declare function registerSpiderCommand(program: Command): void;
+export {};