recker 1.0.72 → 1.0.75-next.2e5a94f

package/dist/browser/seo/seo-spider.js

@@ -7,9 +7,29 @@ import * as fs from 'fs/promises';
 export class SeoSpider {
     spider;
     options;
-    seoResults = new Map();
     seoPages = [];
     homeHtml = '';
+    normalizeUrl(url) {
+        try {
+            const parsed = new URL(url);
+            parsed.hash = '';
+            parsed.searchParams.sort();
+            if (parsed.pathname !== '/' && parsed.pathname.endsWith('/')) {
+                parsed.pathname = parsed.pathname.slice(0, -1);
+            }
+            return parsed.toString();
+        }
+        catch {
+            return url;
+        }
+    }
+    toHeaderRecord(headers) {
+        const headerRecord = {};
+        headers.forEach((value, key) => {
+            headerRecord[key] = value;
+        });
+        return headerRecord;
+    }
     constructor(options = {}) {
         this.options = options;
         this.spider = new Spider({
@@ -36,11 +56,12 @@ export class SeoSpider {
                 : undefined;
             const seoReport = await analyzeSeo(html, {
                 baseUrl: pageResult.url,
+                timings: pageResult.timings,
+                htmlSize: pageResult.metrics?.htmlSize,
                 rules: rulesOptions,
             });
             const seoPage = { ...pageResult, seoReport };
             this.seoPages.push(seoPage);
-            this.seoResults.set(pageResult.url, seoReport);
             this.options.onSeoAnalysis?.(seoPage);
         }
         catch {
@@ -50,7 +71,6 @@ export class SeoSpider {
     }
     async crawl(startUrl) {
         this.seoPages = [];
-        this.seoResults.clear();
         this.homeHtml = '';
         const result = await this.spider.crawl(startUrl);
         if (!this.options.seo) {
@@ -206,7 +226,7 @@ export class SeoSpider {
                 return {
                     status: res.status,
                     text,
-                    headers: Object.fromEntries([...res.headers.entries()]),
+                    headers: this.toHeaderRecord(res.headers),
                 };
             };
             const result = await fetchAndValidateSitemap(sitemapUrl, fetcher);
@@ -216,191 +236,6 @@ export class SeoSpider {
             return undefined;
         }
     }
-    createReportFromPageData(page) {
-        const checks = [];
-        if (page.title) {
-            const titleLength = page.title.length;
-            if (titleLength < 30) {
-                checks.push({
-                    id: 'title-length',
-                    name: 'Title Length',
-                    category: 'title',
-                    status: 'warn',
-                    message: `Title is ${titleLength} characters`,
-                    value: titleLength,
-                    recommendation: 'Title should be 50-60 characters',
-                });
-            }
-            else if (titleLength > 60) {
-                checks.push({
-                    id: 'title-length',
-                    name: 'Title Length',
-                    category: 'title',
-                    status: 'warn',
-                    message: `Title is too long (${titleLength} chars)`,
-                    value: titleLength,
-                    recommendation: 'Title should be 50-60 characters',
-                });
-            }
-            else {
-                checks.push({
-                    id: 'title-length',
-                    name: 'Title Length',
-                    category: 'title',
-                    status: 'pass',
-                    message: `Good title length (${titleLength} chars)`,
-                    value: titleLength,
-                });
-            }
-        }
-        else {
-            checks.push({
-                id: 'title-missing',
-                name: 'Title',
-                category: 'title',
-                status: 'fail',
-                message: 'Page has no title',
-                recommendation: 'Add a descriptive <title> tag',
-            });
-        }
-        const internalLinks = page.links.filter(l => l.type === 'internal').length;
-        const externalLinks = page.links.filter(l => l.type === 'external').length;
-        if (internalLinks === 0) {
-            checks.push({
-                id: 'internal-links',
-                name: 'Internal Links',
-                category: 'links',
-                status: 'warn',
-                message: 'No internal links found',
-                recommendation: 'Add internal links to improve site structure',
-            });
-        }
-        else {
-            checks.push({
-                id: 'internal-links',
-                name: 'Internal Links',
-                category: 'links',
-                status: 'pass',
-                message: `${internalLinks} internal links found`,
-                value: internalLinks,
-            });
-        }
-        const scoreSum = checks.reduce((sum, c) => {
-            if (c.status === 'pass')
-                return sum + 100;
-            if (c.status === 'warn')
-                return sum + 50;
-            return sum;
-        }, 0);
-        const score = checks.length > 0 ? Math.round(scoreSum / checks.length) : 0;
-        const passed = checks.filter(c => c.status === 'pass').length;
-        const warnings = checks.filter(c => c.status === 'warn').length;
-        const errors = checks.filter(c => c.status === 'fail').length;
-        const infos = checks.filter(c => c.status === 'info').length;
-        const passRate = checks.length > 0 ? Math.round((passed / checks.length) * 100) : 0;
-        return {
-            url: page.url,
-            timestamp: new Date(),
-            grade: this.scoreToGrade(score),
-            score,
-            summary: {
-                totalChecks: checks.length,
-                passed,
-                warnings,
-                errors,
-                infos,
-                passRate,
-                issuesByCategory: {},
-                topIssues: checks
-                    .filter(c => c.status === 'fail' || c.status === 'warn')
-                    .slice(0, 5)
-                    .map(c => ({
-                    name: c.name,
-                    message: c.message,
-                    category: 'general',
-                    severity: c.status === 'fail' ? 'error' : 'warning',
-                })),
-                quickWins: [],
-                vitals: {
-                    wordCount: 0,
-                    readingTime: 0,
-                    imageCount: 0,
-                    linkCount: page.links.length,
-                },
-                completeness: {
-                    meta: 0,
-                    social: 0,
-                    technical: 0,
-                    content: 0,
-                    images: 0,
-                    links: 0,
-                },
-            },
-            checks,
-            title: page.title ? { text: page.title, length: page.title.length } : undefined,
-            headings: {
-                structure: [],
-                h1Count: 0,
-                hasProperHierarchy: false,
-                issues: [],
-            },
-            content: {
-                wordCount: 0,
-                characterCount: 0,
-                sentenceCount: 0,
-                paragraphCount: 0,
-                readingTimeMinutes: 0,
-                avgWordsPerSentence: 0,
-                avgParagraphLength: 0,
-                listCount: 0,
-                strongTagCount: 0,
-                emTagCount: 0,
-            },
-            links: {
-                total: page.links.length,
-                internal: internalLinks,
-                external: externalLinks,
-                nofollow: 0,
-                broken: 0,
-                withoutText: page.links.filter(l => !l.text?.trim()).length,
-                sponsoredLinks: 0,
-                ugcLinks: 0,
-            },
-            images: {
-                total: 0,
-                withAlt: 0,
-                withoutAlt: 0,
-                lazy: 0,
-                missingDimensions: 0,
-                modernFormats: 0,
-                altTextLengths: [],
-                imageAltTexts: [],
-                imageFilenames: [],
-                imagesWithAsyncDecoding: 0,
-            },
-            social: {
-                openGraph: {
-                    present: false, hasTitle: false, hasDescription: false, hasImage: false, hasUrl: false, issues: []
-                },
-                twitterCard: {
-                    present: false, hasCard: false, hasTitle: false, hasDescription: false, hasImage: false, issues: []
-                },
-            },
-            keywords: { totalWords: 0, uniqueWords: 0, topKeywords: [] },
-            technical: {
-                hasCanonical: false,
-                hasRobotsMeta: false,
-                hasViewport: false,
-                hasCharset: false,
-                hasLang: false,
-            },
-            structuredData: {
-                count: 0,
-                types: [],
-                items: [],
-            },
-        };
-    }
     detectSiteWideIssues(pages) {
         const issues = [];
         const titleGroups = new Map();
@@ -465,12 +300,12 @@ export class SeoSpider {
         for (const page of pages) {
             for (const link of page.links) {
                 if (link.type === 'internal' && link.href) {
-                    linkedUrls.add(link.href);
+                    linkedUrls.add(this.normalizeUrl(link.href));
                 }
             }
         }
         const orphanPages = pages
-            .filter(p => !linkedUrls.has(p.url) && p.depth > 0)
+            .filter(p => p.depth > 0 && !linkedUrls.has(this.normalizeUrl(p.url)))
             .map(p => p.url);
         if (orphanPages.length > 0) {
             issues.push({
@@ -507,17 +342,6 @@ export class SeoSpider {
             orphanPages,
         };
     }
-    scoreToGrade(score) {
-        if (score >= 90)
-            return 'A';
-        if (score >= 80)
-            return 'B';
-        if (score >= 70)
-            return 'C';
-        if (score >= 60)
-            return 'D';
-        return 'F';
-    }
     async saveReport(result) {
         if (!this.options.output)
             return;

package/dist/browser/seo/types.d.ts

@@ -1,6 +1,7 @@
 import { KeywordCloud } from './keywords.js';
 export type { KeywordCloud, KeywordItem } from './keywords.js';
 export type SeoStatus = 'pass' | 'warn' | 'fail' | 'info';
+export type SeoPageType = 'homepage' | 'product' | 'article' | 'category' | 'search' | 'other';
 export interface SeoCheckEvidence {
     found?: string | number | string[];
     expected?: string | number | string[];
@@ -15,6 +16,7 @@ export interface SeoCheckResult {
     name: string;
     category: string;
     status: SeoStatus;
+    severity?: 'error' | 'warning' | 'info';
     message: string;
     value?: string | number;
     recommendation?: string;
@@ -107,6 +109,7 @@ export interface SeoTiming {
     download?: number;
 }
 export interface SeoSummary {
+    pageType?: SeoPageType;
     totalChecks: number;
     passed: number;
     warnings: number;
@@ -150,6 +153,7 @@ export interface SeoReport {
     timestamp: Date;
     grade: string;
     score: number;
+    pageType?: SeoPageType;
     summary: SeoSummary;
     timing?: SeoTiming;
     checks: SeoCheckResult[];

package/dist/browser/seo/validators/sitemap.js

@@ -2,6 +2,13 @@ import { parse } from '../../scrape/parser/index.js';
 const VALID_CHANGEFREQ = ['always', 'hourly', 'daily', 'weekly', 'monthly', 'yearly', 'never'];
 const MAX_URLS_PER_SITEMAP = 50000;
 const MAX_SITEMAP_SIZE = 50 * 1024 * 1024;
+function toHeaderRecord(headers) {
+    const headerRecord = {};
+    headers.forEach((value, key) => {
+        headerRecord[key] = value;
+    });
+    return headerRecord;
+}
 export function parseSitemap(content, compressed = false) {
     const errors = [];
     const warnings = [];
@@ -141,7 +148,7 @@ export function parseSitemap(content, compressed = false) {
         warnings,
         urls,
         sitemaps,
-        urlCount: type === 'urlset' ? urls.length : sitemaps.reduce((sum, s) => sum + 1, 0),
+        urlCount: type === 'urlset' ? urls.length : sitemaps.length,
         size: content.length,
         compressed,
     };
@@ -339,7 +346,7 @@ export async function fetchAndValidateSitemap(url, fetcher) {
             response = {
                 status: fetchResponse.status,
                 text: await fetchResponse.text(),
-                headers: Object.fromEntries(fetchResponse.headers.entries()),
+                headers: toHeaderRecord(fetchResponse.headers),
             };
         }
         if (response.status === 404) {

package/dist/browser/transport/fetch.js

@@ -1,4 +1,5 @@
 import { createProgressStream } from '../utils/progress.js';
+import { TimeoutError } from '../core/errors.js';
 function parseContentLength(headers) {
     const raw = headers.get('content-length');
     if (!raw)
@@ -6,6 +7,19 @@ function parseContentLength(headers) {
     const parsed = parseInt(raw, 10);
     return Number.isFinite(parsed) ? parsed : undefined;
 }
+function getAbortReason(error) {
+    return error?.cause ?? error?.reason;
+}
+function isTimeoutReason(reason) {
+    if (reason instanceof TimeoutError) {
+        return true;
+    }
+    if (!reason || typeof reason !== 'object') {
+        return false;
+    }
+    const timeoutReason = reason;
+    return timeoutReason.name === 'TimeoutError';
+}
 function wrapDownloadResponse(response, onProgress) {
     if (!onProgress || !response.body)
         return response;
@@ -66,10 +80,16 @@ export class FetchTransport {
             ? req.timeout
             : req.timeout?.request;
         let signal = req.signal;
+        const requestTimeoutError = timeoutMs ? new TimeoutError(req, {
+            phase: 'request',
+            timeout: timeoutMs
+        }) : undefined;
         if (timeoutMs && !signal) {
             abortController = new AbortController();
             signal = abortController.signal;
-            timeoutId = setTimeout(() => abortController.abort(), timeoutMs);
+            timeoutId = setTimeout(() => {
+                timeoutControllerAbort(abortController, requestTimeoutError);
+            }, timeoutMs);
         }
         const followRedirects = req.followRedirects !== false;
         const maxRedirects = req.maxRedirects ?? 20;
@@ -148,10 +168,13 @@ export class FetchTransport {
             }
         }
         catch (error) {
-            if (error.name === 'AbortError' && abortController) {
-                const timeoutError = new Error(`Request timeout after ${timeoutMs}ms`);
-                timeoutError.name = 'TimeoutError';
-                throw timeoutError;
+            if (timeoutMs && (error.name === 'AbortError' || error.name === 'TimeoutError')) {
+                const timeoutReason = getAbortReason(error);
+                if (isTimeoutReason(timeoutReason) ||
+                    isTimeoutReason(getAbortReason(signal)) ||
+                    (abortController && isTimeoutReason(requestTimeoutError))) {
+                    throw timeoutReason instanceof TimeoutError ? timeoutReason : requestTimeoutError ?? error;
+                }
             }
             throw error;
         }
@@ -161,6 +184,16 @@ export class FetchTransport {
         }
     }
 }
+function timeoutControllerAbort(controller, reason) {
+    if (!controller.signal.aborted) {
+        if (reason) {
+            controller.abort(reason);
+        }
+        else {
+            controller.abort();
+        }
+    }
+}
 class FetchResponseWrapper {
     raw;
     timings;

package/dist/browser/transport/undici.js

@@ -15,6 +15,40 @@ const undiciBodySentChannel = channel('undici:request:bodySent');
 const undiciHeadersChannel = channel('undici:request:headers');
 const undiciConnectChannel = channel('undici:client:connect');
 const requestStorage = new AsyncLocalStorage();
+function getAbortReason(error) {
+    return error?.cause ?? error?.reason;
+}
+function isTimeoutAbortReason(reason) {
+    if (reason instanceof TimeoutError) {
+        return true;
+    }
+    if (!reason || typeof reason !== 'object') {
+        return false;
+    }
+    const timeoutReason = reason;
+    return timeoutReason.name === 'TimeoutError';
+}
+function isTotalRequestTimeoutError(error, timeoutReason, timeoutMs) {
+    if (!timeoutMs) {
+        return false;
+    }
+    if (timeoutReason !== undefined) {
+        return true;
+    }
+    const causeOrReason = getAbortReason(error);
+    const message = error?.message;
+    const hasTimeoutMessage = typeof message === 'string' && (message.includes('Request timed out (total time exceeded)') ||
+        message.includes('timed out (total time exceeded)'));
+    const hasRequestAbortMessage = typeof message === 'string' && message.toLowerCase() === 'request was aborted';
+    return (error instanceof TimeoutError ||
+        isTimeoutAbortReason(causeOrReason) ||
+        isTimeoutAbortReason(error) ||
+        error instanceof undiciErrors.RequestAbortedError ||
+        hasTimeoutMessage ||
+        hasRequestAbortMessage ||
+        error?.code === 'ABORT_ERR' ||
+        error?.code === 'UND_ERR_REQUEST_TIMEOUT');
+}
 undiciRequestChannel.subscribe((message) => {
     const store = requestStorage.getStore();
     if (store) {
@@ -36,7 +70,7 @@ undiciRequestChannel.subscribe((message) => {
         }
     }
 });
-undiciBodySentChannel.subscribe((message) => {
+undiciBodySentChannel.subscribe((_message) => {
     const store = requestStorage.getStore();
     if (store?.hooks && store.hooks.onRequestSent) {
         store.hooks.onRequestSent();
@@ -317,6 +351,9 @@ export class UndiciTransport {
         });
         let timeoutController;
         let timeoutId;
+        let timeoutReason;
+        let timeoutError;
+        let cancelOriginalAbortListener;
         if (!this.observability) {
             return this.dispatchFast(req, headers, currentUrl, timeouts, handleRedirectsManually, maxRedirects, followRedirects, uploadTotal);
         }
@@ -340,24 +377,32 @@ export class UndiciTransport {
                 let effectiveSignal = req.signal;
                 if (timeouts.totalTimeout) {
                     timeoutController = new AbortController();
+                    timeoutError = new TimeoutError(req, {
+                        phase: 'request',
+                        timeout: timeouts.totalTimeout
+                    });
                     if (req.signal) {
                         const originalSignal = req.signal;
                         effectiveSignal = timeoutController.signal;
                         const onOriginalAbort = () => {
-                            timeoutController.abort();
+                            timeoutReason = originalSignal.reason ?? new Error('Request aborted by external signal');
+                            timeoutController.abort(timeoutReason);
                         };
                         if (originalSignal.aborted) {
-                            timeoutController.abort();
+                            timeoutReason = originalSignal.reason ?? new Error('Request aborted by external signal');
+                            timeoutController.abort(timeoutReason);
                         }
                         else {
                             originalSignal.addEventListener('abort', onOriginalAbort, { once: true });
+                            cancelOriginalAbortListener = () => originalSignal.removeEventListener('abort', onOriginalAbort);
                         }
                     }
                     else {
                         effectiveSignal = timeoutController.signal;
                     }
                     timeoutId = setTimeout(() => {
-                        timeoutController.abort();
+                        timeoutReason = timeoutError;
+                        timeoutController.abort(timeoutReason);
                     }, timeouts.totalTimeout);
                 }
                 while (true) {
@@ -593,8 +638,8 @@ export class UndiciTransport {
                         timeout: timeouts.bodyTimeout
                     });
                 }
-                if (error.name === 'AbortError' || error.code === 'ABORT_ERR') {
-                    throw new TimeoutError(req, {
+                if (isTotalRequestTimeoutError(error, timeoutReason, timeouts.totalTimeout)) {
+                    throw timeoutError ?? new TimeoutError(req, {
                         phase: 'request',
                         timeout: timeouts.totalTimeout
                     });
@@ -611,6 +656,9 @@ export class UndiciTransport {
                 throw new NetworkError(error.message, code, req);
             }
             finally {
+                if (typeof cancelOriginalAbortListener === 'function') {
+                    cancelOriginalAbortListener();
+                }
                 if (timeoutId) {
                     clearTimeout(timeoutId);
                 }
@@ -620,6 +668,9 @@ export class UndiciTransport {
     async dispatchFast(req, headers, currentUrl, timeouts, handleRedirectsManually, maxRedirects, followRedirects, uploadTotal) {
         let timeoutController;
         let timeoutId;
+        let timeoutReason;
+        let timeoutError;
+        let cancelOriginalAbortListener;
         try {
             let redirectCount = 0;
             let currentMethod = req.method;
@@ -628,24 +679,32 @@ export class UndiciTransport {
             let effectiveSignal = req.signal;
             if (timeouts.totalTimeout) {
                 timeoutController = new AbortController();
+                timeoutError = new TimeoutError(req, {
+                    phase: 'request',
+                    timeout: timeouts.totalTimeout
+                });
                 if (req.signal) {
                     const originalSignal = req.signal;
                     effectiveSignal = timeoutController.signal;
                     const onOriginalAbort = () => {
-                        timeoutController.abort();
+                        timeoutReason = originalSignal.reason ?? new Error('Request aborted by external signal');
+                        timeoutController.abort(timeoutReason);
                     };
                     if (originalSignal.aborted) {
-                        timeoutController.abort();
+                        timeoutReason = originalSignal.reason ?? new Error('Request aborted by external signal');
+                        timeoutController.abort(timeoutReason);
                     }
                     else {
                         originalSignal.addEventListener('abort', onOriginalAbort, { once: true });
+                        cancelOriginalAbortListener = () => originalSignal.removeEventListener('abort', onOriginalAbort);
                     }
                 }
                 else {
                     effectiveSignal = timeoutController.signal;
                 }
                 timeoutId = setTimeout(() => {
-                    timeoutController.abort();
+                    timeoutReason = timeoutError;
+                    timeoutController.abort(timeoutReason);
                 }, timeouts.totalTimeout);
             }
             while (true) {
@@ -822,8 +881,8 @@ export class UndiciTransport {
                     timeout: timeouts.bodyTimeout
                 });
             }
-            if (error.name === 'AbortError' || error.code === 'ABORT_ERR') {
-                throw new TimeoutError(req, {
+            if (isTotalRequestTimeoutError(error, timeoutReason, timeouts.totalTimeout)) {
+                throw timeoutError ?? new TimeoutError(req, {
                     phase: 'request',
                     timeout: timeouts.totalTimeout
                 });
@@ -840,6 +899,9 @@ export class UndiciTransport {
             throw new NetworkError(error.message, code, req);
         }
         finally {
+            if (typeof cancelOriginalAbortListener === 'function') {
+                cancelOriginalAbortListener();
+            }
             if (timeoutId) {
                 clearTimeout(timeoutId);
             }

package/dist/browser/transport/worker.d.ts

@@ -3,7 +3,6 @@ export interface WorkerTransportOptions {
     poolSize?: number;
 }
 export declare class WorkerTransport implements Transport {
-    private options;
     private workers;
     private workerIndex;
     private pendingRequests;

package/dist/browser/transport/worker.js

@@ -58,13 +58,11 @@ self.onmessage = async (event) => {
 };
 `;
 export class WorkerTransport {
-    options;
     workers = [];
     workerIndex = 0;
     pendingRequests = new Map();
     workerUrl;
     constructor(options = {}) {
-        this.options = options;
         const poolSize = options.poolSize ?? (typeof navigator !== 'undefined' ? navigator.hardwareConcurrency : 4) ?? 4;
         const blob = new Blob([WORKER_SCRIPT], { type: 'application/javascript' });
         this.workerUrl = URL.createObjectURL(blob);
@@ -166,7 +164,7 @@ export class WorkerTransport {
         }
         this.workers = [];
         URL.revokeObjectURL(this.workerUrl);
-        for (const [id, pending] of this.pendingRequests) {
+        for (const pending of this.pendingRequests.values()) {
             pending.reject(new Error('Transport terminated'));
         }
         this.pendingRequests.clear();