recker 1.0.72 → 1.0.75-next.2e5a94f

package/dist/seo/analyzer.js

@@ -1,7 +1,7 @@
 import { parse } from '../scrape/parser/index.js';
 import { extractMeta, extractOpenGraph, extractTwitterCard, extractJsonLd, extractLinks, extractImages, } from '../scrape/extractors.js';
 import { generateKeywordCloud } from './keywords.js';
-import { createRulesEngine, SEO_THRESHOLDS, } from './rules/index.js';
+import { createRulesEngine, SEO_THRESHOLDS, calculateWeightedScore, } from './rules/index.js';
 export class SeoAnalyzer {
     root;
     options;
@@ -47,7 +47,9 @@ export class SeoAnalyzer {
         const analytics = this.analyzeAnalytics();
         const feeds = this.analyzeFeeds();
         const conversion = this.analyzeConversionElements(links, visibleText);
+        const pageType = this.detectPageType(jsonLd);
         const context = this.buildRuleContext({
+            pageType,
             meta,
             og,
             twitter,
@@ -68,7 +70,7 @@ export class SeoAnalyzer {
         });
         const ruleResults = this.rulesEngine.evaluate(context);
         const checks = this.convertToCheckResults(ruleResults);
-        const { score, grade } = this.calculateScore(checks);
+        const { score, grade } = this.calculateScore(ruleResults);
         const summary = this.buildSummary(ruleResults, checks, {
             content,
             imageAnalysis,
@@ -77,13 +79,17 @@ export class SeoAnalyzer {
             og,
             twitter,
             technical,
+            pageType,
+            timings: this.options.timings,
         });
         return {
             url,
             timestamp: new Date(),
             grade,
             score,
+            timing: this.options.timings,
             summary,
+            pageType,
             checks,
             title: meta.title
                 ? { text: meta.title, length: meta.title.length }
@@ -134,6 +140,47 @@ export class SeoAnalyzer {
             return bodies[0];
         return bodies.reduce((prev, curr) => curr.text.length > prev.text.length ? curr : prev);
     }
+    detectPageType(jsonLd) {
+        if (!this.options.baseUrl) {
+            return 'other';
+        }
+        try {
+            const parsed = new URL(this.options.baseUrl);
+            const pathname = parsed.pathname.toLowerCase();
+            const hasQueryKeyword = (value) => parsed.searchParams.has(value);
+            if (pathname === '/' || pathname === '') {
+                return 'homepage';
+            }
+            if (/(^|\/)(search|busca|s|results|query)\b/.test(pathname) ||
+                hasQueryKeyword('q') ||
+                hasQueryKeyword('query') ||
+                hasQueryKeyword('search')) {
+                return 'search';
+            }
+            const productSignals = ['product', 'produto', 'item', 'sku', 'shop'];
+            if (productSignals.some((segment) => pathname.includes(`/${segment}/`))) {
+                return 'product';
+            }
+            const articleSignals = ['article', 'post', 'blog', 'noticia', 'news'];
+            if (articleSignals.some((segment) => pathname.includes(`/${segment}/`)) ||
+                this.root.querySelectorAll('article').length > 0) {
+                return 'article';
+            }
+            if (/(^|\/)(categoria|category|tag|section|topic)\b/.test(pathname)) {
+                return 'category';
+            }
+            const hasProductJsonLd = jsonLd
+                .map((node) => node['@type'])
+                .some((type) => typeof type === 'string' && type.toLowerCase() === 'product');
+            if (hasProductJsonLd) {
+                return 'product';
+            }
+            return 'other';
+        }
+        catch {
+            return 'other';
+        }
+    }
     getVisibleText() {
         const body = this.getMainBody();
         if (!body)
@@ -159,7 +206,7 @@ export class SeoAnalyzer {
         return clone.text.replace(/\s+/g, ' ').trim();
     }
     buildRuleContext(data) {
-        const { meta, og, twitter, jsonLd, headings, content, linkAnalysis, imageAnalysis, links, keywords, resources, emailsFound, socialLinksFound, socialLinkDetails, analytics, feeds, conversion, } = data;
+        const { meta, og, twitter, jsonLd, headings, content, pageType, linkAnalysis, imageAnalysis, links, keywords, resources, emailsFound, socialLinksFound, socialLinkDetails, analytics, feeds, conversion, } = data;
         const html = this.root.querySelector('html');
         const htmlLang = html ? html.getAttribute('lang') : undefined;
         const hreflangTags = [];
@@ -198,7 +245,6 @@ export class SeoAnalyzer {
         const hasMixedContent = this.checkMixedContent();
         const h1Elements = this.root.querySelectorAll('h1');
         const h1Text = h1Elements.length > 0 ? h1Elements[0].text.trim() : '';
-        const iframeCount = this.root.querySelectorAll('iframe').length;
         const topKeywords = keywords.topKeywords.slice(0, 5).map(k => k.word);
         const mainKeyword = topKeywords.length > 0 ? topKeywords[0] : undefined;
         const keywordsInTitle = topKeywords.some(kw => meta.title?.toLowerCase().includes(kw));
@@ -234,7 +280,6 @@ export class SeoAnalyzer {
         const structuralHtml = this.analyzeStructuralHtml();
         const breadcrumbs = this.analyzeBreadcrumbs(jsonLd.map((j) => j['@type']).filter(Boolean));
         const multimedia = this.analyzeMultimedia();
-        const advancedImages = this.analyzeAdvancedImages();
         const responsiveImages = this.analyzeResponsiveImages();
         const inlineImages = this.analyzeInlineImages();
         const trustSignals = this.analyzeTrustSignals(links);
@@ -245,6 +290,7 @@ export class SeoAnalyzer {
             : 0;
         const textHtmlRatio = this.calculateTextHtmlRatio(content.characterCount);
         return {
+            pageType,
             jsFilesCount: resources.jsFilesCount,
             cssFilesCount: resources.cssFilesCount,
             unminifiedResources: resources.unminifiedResources,
@@ -774,7 +820,6 @@ export class SeoAnalyzer {
     }
     analyzeAnalytics() {
         const providers = [];
-        const html = this.root.innerHTML || '';
         const scripts = this.root.querySelectorAll('script');
         const scriptSources = [];
         const scriptContents = [];
@@ -899,21 +944,6 @@ export class SeoAnalyzer {
             hasPhoneOnPage,
         };
     }
-    analyzeAdvancedImages() {
-        let imagesWithSrcset = 0;
-        let largeBase64ImagesCount = 0;
-        const imgs = this.root.querySelectorAll('img');
-        imgs.forEach((img) => {
-            if (img.getAttribute('srcset') || (img.parentNode && img.parentNode.tagName === 'PICTURE')) {
-                imagesWithSrcset++;
-            }
-            const src = img.getAttribute('src') || '';
-            if (src.startsWith('data:image') && src.length > 5 * 1024) {
-                largeBase64ImagesCount++;
-            }
-        });
-        return { imagesWithSrcset, largeBase64ImagesCount };
-    }
     calculateTextHtmlRatio(bodyTextLength) {
         const htmlSize = this.root.innerHTML?.length;
         if (htmlSize && htmlSize > 0) {
@@ -926,6 +956,7 @@ export class SeoAnalyzer {
             id: r.id,
             name: r.name,
             category: r.category,
+            severity: r.severity,
             status: r.status,
             message: r.message,
             value: r.value,
@@ -934,6 +965,8 @@ export class SeoAnalyzer {
         }));
     }
     buildSummary(ruleResults, checks, data) {
+        const pageType = data.pageType;
+        const timings = data.timings;
         const passed = checks.filter((c) => c.status === 'pass').length;
         const warnings = checks.filter((c) => c.status === 'warn').length;
         const errors = checks.filter((c) => c.status === 'fail').length;
@@ -954,21 +987,25 @@ export class SeoAnalyzer {
             else if (result.status === 'fail')
                 issuesByCategory[cat].errors++;
         }
-        const topIssues = ruleResults
-            .filter((r) => r.status === 'fail' || r.status === 'warn')
+        const topIssues = checks
+            .filter((c) => c.status === 'fail' || c.status === 'warn')
             .sort((a, b) => {
-            if (a.status === 'fail' && b.status !== 'fail')
-                return -1;
-            if (a.status !== 'fail' && b.status === 'fail')
-                return 1;
-            return 0;
+            const severityOrder = (status) => status === 'fail' ? 2 : 1;
+            const statusDiff = severityOrder(b.status) - severityOrder(a.status);
+            if (statusDiff !== 0)
+                return statusDiff;
+            const aSeverity = a.severity || (a.status === 'fail' ? 'error' : 'warning');
+            const bSeverity = b.severity || (b.status === 'fail' ? 'error' : 'warning');
+            if (aSeverity === bSeverity)
+                return 0;
+            return aSeverity === 'error' ? -1 : 1;
         })
             .slice(0, 5)
             .map((r) => ({
             name: r.name,
             message: r.message,
             category: r.category,
-            severity: (r.status === 'fail' ? 'error' : 'warning'),
+            severity: (r.severity || (r.status === 'fail' ? 'error' : 'warning')),
         }));
         const quickWins = [];
         if (!data.meta.title)
@@ -993,8 +1030,8 @@ export class SeoAnalyzer {
         const vitals = {
             htmlSize,
             domElements,
-            ttfb: this.options.responseHeaders ? undefined : undefined,
-            totalTime: undefined,
+            ttfb: timings?.ttfb,
+            totalTime: timings?.total,
             wordCount: data.content.wordCount,
             totalWordCount: data.content.totalWordCount,
             readingTime: data.content.readingTimeMinutes,
@@ -1017,6 +1054,7 @@ export class SeoAnalyzer {
             infos,
             passRate,
             issuesByCategory,
+            pageType: pageType,
             topIssues,
             quickWins: limitedQuickWins,
             vitals,
@@ -1387,18 +1425,11 @@ export class SeoAnalyzer {
             unminifiedResourceUrls: unminified
         };
     }
-    calculateScore(checks) {
-        const weights = {
-            pass: 100,
-            warn: 50,
-            fail: 0,
-            info: 100,
-        };
-        const scoringChecks = checks.filter((c) => c.status !== 'info');
-        if (scoringChecks.length === 0)
+    calculateScore(results) {
+        if (results.length === 0)
             return { score: 100, grade: 'A' };
-        const totalWeight = scoringChecks.reduce((sum, check) => sum + weights[check.status], 0);
-        const score = Math.round(totalWeight / scoringChecks.length);
+        const { score: weightedScore } = calculateWeightedScore(results);
+        const score = weightedScore;
         let grade;
         if (score >= 90)
             grade = 'A';

package/dist/seo/index.d.ts

@@ -6,7 +6,7 @@ export type { SeoSpiderOptions, SeoPageResult, SiteWideIssue, SeoSpiderResult, }
 export { SeoRulesEngine, createRulesEngine, SEO_THRESHOLDS, ALL_SEO_RULES, } from './rules/index.js';
 export { generateSeoFilename, resolveOutputPath, writeReport, formatReportForJson, } from './output.js';
 export type { SeoOutputType, OutputOptions, WriteOptions } from './output.js';
-export type { SeoReport, SeoCheckResult, SeoStatus, SeoTiming, HeadingAnalysis, HeadingInfo, ContentMetrics, LinkAnalysis, ImageAnalysis, SocialMetaAnalysis, TechnicalSeo, SeoAnalyzerOptions, } from './types.js';
+export type { SeoReport, SeoCheckResult, SeoStatus, SeoPageType, SeoTiming, HeadingAnalysis, HeadingInfo, ContentMetrics, LinkAnalysis, ImageAnalysis, SocialMetaAnalysis, TechnicalSeo, SeoAnalyzerOptions, } from './types.js';
 export type { SeoRule, RuleContext, RuleResult, RuleEvidence, RuleCategory, RuleSeverity, RulesEngineOptions, } from './rules/index.js';
 export type { SeoAnalyzerFullOptions } from './analyzer.js';
 export { parseRobotsTxt, validateRobotsTxt, isPathAllowed, fetchAndValidateRobotsTxt, } from './validators/robots.js';

package/dist/seo/rules/types.d.ts

@@ -1,8 +1,10 @@
 import type { SeoStatus } from '../types.js';
 import type { ExtractedLink } from '../../scrape/types.js';
+import type { SeoPageType } from '../types.js';
 export type RuleSeverity = 'error' | 'warning' | 'info';
 export type RuleCategory = 'title' | 'meta' | 'og' | 'twitter' | 'headings' | 'images' | 'links' | 'content' | 'technical' | 'security' | 'mobile' | 'structured-data' | 'performance' | 'accessibility' | 'i18n' | 'ai-search' | 'resources' | 'crawlability' | 'canonicalization';
 export interface RuleContext {
+    pageType?: SeoPageType;
     keywordsInTitle?: boolean;
     keywordsInDescription?: boolean;
     keywordsInH1?: boolean;

package/dist/seo/seo-spider.d.ts

@@ -66,19 +66,18 @@ export interface SeoSpiderResult extends Omit<SpiderResult, 'pages'> {
 export declare class SeoSpider {
     private spider;
     private options;
-    private seoResults;
     private seoPages;
     private homeHtml;
+    private normalizeUrl;
+    private toHeaderRecord;
     constructor(options?: SeoSpiderOptions);
     private analyzePageDuringCrawl;
     crawl(startUrl: string): Promise<SeoSpiderResult>;
     private checkSiteFiles;
     private validateManifest;
     private validateSitemap;
-    private createReportFromPageData;
     private detectSiteWideIssues;
     private calculateSummary;
-    private scoreToGrade;
     private saveReport;
     abort(): void;
     isRunning(): boolean;

package/dist/seo/seo-spider.js

@@ -7,9 +7,29 @@ import * as fs from 'fs/promises';
 export class SeoSpider {
     spider;
     options;
-    seoResults = new Map();
     seoPages = [];
     homeHtml = '';
+    normalizeUrl(url) {
+        try {
+            const parsed = new URL(url);
+            parsed.hash = '';
+            parsed.searchParams.sort();
+            if (parsed.pathname !== '/' && parsed.pathname.endsWith('/')) {
+                parsed.pathname = parsed.pathname.slice(0, -1);
+            }
+            return parsed.toString();
+        }
+        catch {
+            return url;
+        }
+    }
+    toHeaderRecord(headers) {
+        const headerRecord = {};
+        headers.forEach((value, key) => {
+            headerRecord[key] = value;
+        });
+        return headerRecord;
+    }
     constructor(options = {}) {
         this.options = options;
         this.spider = new Spider({
@@ -36,11 +56,12 @@ export class SeoSpider {
                 : undefined;
             const seoReport = await analyzeSeo(html, {
                 baseUrl: pageResult.url,
+                timings: pageResult.timings,
+                htmlSize: pageResult.metrics?.htmlSize,
                 rules: rulesOptions,
             });
             const seoPage = { ...pageResult, seoReport };
             this.seoPages.push(seoPage);
-            this.seoResults.set(pageResult.url, seoReport);
             this.options.onSeoAnalysis?.(seoPage);
         }
         catch {
@@ -50,7 +71,6 @@ export class SeoSpider {
     }
     async crawl(startUrl) {
         this.seoPages = [];
-        this.seoResults.clear();
         this.homeHtml = '';
         const result = await this.spider.crawl(startUrl);
         if (!this.options.seo) {
@@ -206,7 +226,7 @@ export class SeoSpider {
                 return {
                     status: res.status,
                     text,
-                    headers: Object.fromEntries([...res.headers.entries()]),
+                    headers: this.toHeaderRecord(res.headers),
                 };
             };
             const result = await fetchAndValidateSitemap(sitemapUrl, fetcher);
@@ -216,191 +236,6 @@ export class SeoSpider {
             return undefined;
         }
     }
-    createReportFromPageData(page) {
-        const checks = [];
-        if (page.title) {
-            const titleLength = page.title.length;
-            if (titleLength < 30) {
-                checks.push({
-                    id: 'title-length',
-                    name: 'Title Length',
-                    category: 'title',
-                    status: 'warn',
-                    message: `Title is ${titleLength} characters`,
-                    value: titleLength,
-                    recommendation: 'Title should be 50-60 characters',
-                });
-            }
-            else if (titleLength > 60) {
-                checks.push({
-                    id: 'title-length',
-                    name: 'Title Length',
-                    category: 'title',
-                    status: 'warn',
-                    message: `Title is too long (${titleLength} chars)`,
-                    value: titleLength,
-                    recommendation: 'Title should be 50-60 characters',
-                });
-            }
-            else {
-                checks.push({
-                    id: 'title-length',
-                    name: 'Title Length',
-                    category: 'title',
-                    status: 'pass',
-                    message: `Good title length (${titleLength} chars)`,
-                    value: titleLength,
-                });
-            }
-        }
-        else {
-            checks.push({
-                id: 'title-missing',
-                name: 'Title',
-                category: 'title',
-                status: 'fail',
-                message: 'Page has no title',
-                recommendation: 'Add a descriptive <title> tag',
-            });
-        }
-        const internalLinks = page.links.filter(l => l.type === 'internal').length;
-        const externalLinks = page.links.filter(l => l.type === 'external').length;
-        if (internalLinks === 0) {
-            checks.push({
-                id: 'internal-links',
-                name: 'Internal Links',
-                category: 'links',
-                status: 'warn',
-                message: 'No internal links found',
-                recommendation: 'Add internal links to improve site structure',
-            });
-        }
-        else {
-            checks.push({
-                id: 'internal-links',
-                name: 'Internal Links',
-                category: 'links',
-                status: 'pass',
-                message: `${internalLinks} internal links found`,
-                value: internalLinks,
-            });
-        }
-        const scoreSum = checks.reduce((sum, c) => {
-            if (c.status === 'pass')
-                return sum + 100;
-            if (c.status === 'warn')
-                return sum + 50;
-            return sum;
-        }, 0);
-        const score = checks.length > 0 ? Math.round(scoreSum / checks.length) : 0;
-        const passed = checks.filter(c => c.status === 'pass').length;
-        const warnings = checks.filter(c => c.status === 'warn').length;
-        const errors = checks.filter(c => c.status === 'fail').length;
-        const infos = checks.filter(c => c.status === 'info').length;
-        const passRate = checks.length > 0 ? Math.round((passed / checks.length) * 100) : 0;
-        return {
-            url: page.url,
-            timestamp: new Date(),
-            grade: this.scoreToGrade(score),
-            score,
-            summary: {
-                totalChecks: checks.length,
-                passed,
-                warnings,
-                errors,
-                infos,
-                passRate,
-                issuesByCategory: {},
-                topIssues: checks
-                    .filter(c => c.status === 'fail' || c.status === 'warn')
-                    .slice(0, 5)
-                    .map(c => ({
-                    name: c.name,
-                    message: c.message,
-                    category: 'general',
-                    severity: c.status === 'fail' ? 'error' : 'warning',
-                })),
-                quickWins: [],
-                vitals: {
-                    wordCount: 0,
-                    readingTime: 0,
-                    imageCount: 0,
-                    linkCount: page.links.length,
-                },
-                completeness: {
-                    meta: 0,
-                    social: 0,
-                    technical: 0,
-                    content: 0,
-                    images: 0,
-                    links: 0,
-                },
-            },
-            checks,
-            title: page.title ? { text: page.title, length: page.title.length } : undefined,
-            headings: {
-                structure: [],
-                h1Count: 0,
-                hasProperHierarchy: false,
-                issues: [],
-            },
-            content: {
-                wordCount: 0,
-                characterCount: 0,
-                sentenceCount: 0,
-                paragraphCount: 0,
-                readingTimeMinutes: 0,
-                avgWordsPerSentence: 0,
-                avgParagraphLength: 0,
-                listCount: 0,
-                strongTagCount: 0,
-                emTagCount: 0,
-            },
-            links: {
-                total: page.links.length,
-                internal: internalLinks,
-                external: externalLinks,
-                nofollow: 0,
-                broken: 0,
-                withoutText: page.links.filter(l => !l.text?.trim()).length,
-                sponsoredLinks: 0,
-                ugcLinks: 0,
-            },
-            images: {
-                total: 0,
-                withAlt: 0,
-                withoutAlt: 0,
-                lazy: 0,
-                missingDimensions: 0,
-                modernFormats: 0,
-                altTextLengths: [],
-                imageAltTexts: [],
-                imageFilenames: [],
-                imagesWithAsyncDecoding: 0,
-            },
-            social: {
-                openGraph: {
-                    present: false, hasTitle: false, hasDescription: false, hasImage: false, hasUrl: false, issues: []
-                },
-                twitterCard: {
-                    present: false, hasCard: false, hasTitle: false, hasDescription: false, hasImage: false, issues: []
-                },
-            },
-            keywords: { totalWords: 0, uniqueWords: 0, topKeywords: [] },
-            technical: {
-                hasCanonical: false,
-                hasRobotsMeta: false,
-                hasViewport: false,
-                hasCharset: false,
-                hasLang: false,
-            },
-            structuredData: {
-                count: 0,
-                types: [],
-                items: [],
-            },
-        };
-    }
     detectSiteWideIssues(pages) {
         const issues = [];
         const titleGroups = new Map();
@@ -465,12 +300,12 @@ export class SeoSpider {
         for (const page of pages) {
             for (const link of page.links) {
                 if (link.type === 'internal' && link.href) {
-                    linkedUrls.add(link.href);
+                    linkedUrls.add(this.normalizeUrl(link.href));
                 }
             }
         }
         const orphanPages = pages
-            .filter(p => !linkedUrls.has(p.url) && p.depth > 0)
+            .filter(p => p.depth > 0 && !linkedUrls.has(this.normalizeUrl(p.url)))
             .map(p => p.url);
         if (orphanPages.length > 0) {
             issues.push({
@@ -507,17 +342,6 @@ export class SeoSpider {
             orphanPages,
         };
     }
-    scoreToGrade(score) {
-        if (score >= 90)
-            return 'A';
-        if (score >= 80)
-            return 'B';
-        if (score >= 70)
-            return 'C';
-        if (score >= 60)
-            return 'D';
-        return 'F';
-    }
     async saveReport(result) {
         if (!this.options.output)
             return;

package/dist/seo/types.d.ts

@@ -1,6 +1,7 @@
 import { KeywordCloud } from './keywords.js';
 export type { KeywordCloud, KeywordItem } from './keywords.js';
 export type SeoStatus = 'pass' | 'warn' | 'fail' | 'info';
+export type SeoPageType = 'homepage' | 'product' | 'article' | 'category' | 'search' | 'other';
 export interface SeoCheckEvidence {
     found?: string | number | string[];
     expected?: string | number | string[];
@@ -15,6 +16,7 @@ export interface SeoCheckResult {
     name: string;
     category: string;
     status: SeoStatus;
+    severity?: 'error' | 'warning' | 'info';
     message: string;
     value?: string | number;
     recommendation?: string;
@@ -107,6 +109,7 @@ export interface SeoTiming {
     download?: number;
 }
 export interface SeoSummary {
+    pageType?: SeoPageType;
     totalChecks: number;
     passed: number;
     warnings: number;
@@ -150,6 +153,7 @@ export interface SeoReport {
     timestamp: Date;
     grade: string;
     score: number;
+    pageType?: SeoPageType;
     summary: SeoSummary;
     timing?: SeoTiming;
     checks: SeoCheckResult[];

package/dist/seo/validators/sitemap.js

@@ -2,6 +2,13 @@ import { parse } from '../../scrape/parser/index.js';
 const VALID_CHANGEFREQ = ['always', 'hourly', 'daily', 'weekly', 'monthly', 'yearly', 'never'];
 const MAX_URLS_PER_SITEMAP = 50000;
 const MAX_SITEMAP_SIZE = 50 * 1024 * 1024;
+function toHeaderRecord(headers) {
+    const headerRecord = {};
+    headers.forEach((value, key) => {
+        headerRecord[key] = value;
+    });
+    return headerRecord;
+}
 export function parseSitemap(content, compressed = false) {
     const errors = [];
     const warnings = [];
@@ -141,7 +148,7 @@ export function parseSitemap(content, compressed = false) {
         warnings,
         urls,
         sitemaps,
-        urlCount: type === 'urlset' ? urls.length : sitemaps.reduce((sum, s) => sum + 1, 0),
+        urlCount: type === 'urlset' ? urls.length : sitemaps.length,
         size: content.length,
         compressed,
     };
@@ -339,7 +346,7 @@ export async function fetchAndValidateSitemap(url, fetcher) {
             response = {
                 status: fetchResponse.status,
                 text: await fetchResponse.text(),
-                headers: Object.fromEntries(fetchResponse.headers.entries()),
+                headers: toHeaderRecord(fetchResponse.headers),
             };
         }
         if (response.status === 404) {