recker 1.0.30 → 1.0.32-next.02f2bae

package/dist/seo/rules/technical.js

@@ -1,5 +1,4 @@
 import { createResult } from './types.js';
-import { SEO_THRESHOLDS } from './thresholds.js';
 export const technicalRules = [
     {
         id: 'canonical-exists',
@@ -71,23 +70,6 @@ export const technicalRules = [
             return createResult({ id: 'favicon-exists', name: 'Favicon', category: 'technical', severity: 'warning' }, 'pass', 'Favicon is defined', { value: ctx.faviconUrl });
         },
     },
-    {
-        id: 'url-length',
-        name: 'URL Length',
-        category: 'technical',
-        severity: 'info',
-        description: 'URL should be under 75 characters',
-        check: (ctx) => {
-            if (!ctx.url)
-                return null;
-            const len = ctx.urlLength ?? ctx.url.length;
-            const max = SEO_THRESHOLDS.url.maxLength;
-            if (len > max) {
-                return createResult({ id: 'url-length', name: 'URL Length', category: 'technical', severity: 'info' }, 'info', `URL is long (${len} chars)`, { value: len, recommendation: `Keep URLs under ${max} characters when possible` });
-            }
-            return null;
-        },
-    },
     {
         id: 'url-lowercase',
         name: 'URL Lowercase',
@@ -220,4 +202,78 @@ export const technicalRules = [
             return createResult({ id: 'technical-robots-txt-hint', name: 'Robots.txt Hint', category: 'technical', severity: 'info' }, 'info', 'Robots.txt existence cannot be verified from HTML alone.', { recommendation: 'Ensure a valid `robots.txt` file is present at your domain root (e.g., `https://example.com/robots.txt`) to guide search engine crawlers and define your sitemap location.' });
         },
     },
+    {
+        id: 'url-many-parameters',
+        name: 'Too Many URL Parameters',
+        category: 'technical',
+        severity: 'warning',
+        description: 'URLs should not have more than 3 query parameters',
+        check: (ctx) => {
+            if (!ctx.url)
+                return null;
+            try {
+                const url = new URL(ctx.url);
+                const paramCount = Array.from(url.searchParams.keys()).length;
+                if (paramCount > 3) {
+                    return createResult({ id: 'url-many-parameters', name: 'Too Many URL Parameters', category: 'technical', severity: 'warning' }, 'warn', `URL has ${paramCount} query parameters`, {
+                        value: paramCount,
+                        recommendation: 'Reduce URL parameters to 3 or fewer for better crawlability',
+                        evidence: {
+                            found: url.search,
+                            expected: '3 or fewer parameters',
+                            impact: 'Multiple parameters make URLs less enticing and may cause indexing issues'
+                        }
+                    });
+                }
+            }
+            catch {
+            }
+            return null;
+        },
+    },
+    {
+        id: 'deprecated-plugins',
+        name: 'Deprecated Plugins',
+        category: 'technical',
+        severity: 'error',
+        description: 'Pages should not use Flash, Java Applets, or Silverlight',
+        check: (ctx) => {
+            if (ctx.hasDeprecatedPlugins === undefined)
+                return null;
+            if (ctx.hasDeprecatedPlugins) {
+                return createResult({ id: 'deprecated-plugins', name: 'Deprecated Plugins', category: 'technical', severity: 'error' }, 'fail', 'Page uses deprecated plugins (Flash, Java, or Silverlight)', {
+                    recommendation: 'Convert plugin content to HTML5',
+                    evidence: {
+                        found: ctx.deprecatedPluginTypes?.join(', ') || 'Flash/Java/Silverlight detected',
+                        expected: 'No deprecated plugins',
+                        impact: 'These plugins do not work on mobile devices and cannot be crawled properly',
+                        learnMore: 'https://developers.google.com/search/docs/crawling-indexing/mobile/mobile-sites-mobile-first-indexing'
+                    }
+                });
+            }
+            return null;
+        },
+    },
+    {
+        id: 'frame-tags',
+        name: 'Frame Tags',
+        category: 'technical',
+        severity: 'error',
+        description: 'Pages should not use <frame> or <frameset> tags',
+        check: (ctx) => {
+            if (ctx.hasFrameTags === undefined)
+                return null;
+            if (ctx.hasFrameTags) {
+                return createResult({ id: 'frame-tags', name: 'Frame Tags', category: 'technical', severity: 'error' }, 'fail', 'Page uses <frame> or <frameset> tags', {
+                    recommendation: 'Remove frame tags and restructure using modern HTML',
+                    evidence: {
+                        found: '<frame> or <frameset> tags detected',
+                        expected: 'No frame tags',
+                        impact: 'Search engines have difficulty indexing content within frames, affecting rankings'
+                    }
+                });
+            }
+            return null;
+        },
+    },
 ];

package/dist/seo/rules/types.d.ts

@@ -1,7 +1,7 @@
 import type { SeoStatus } from '../types.js';
 import type { ExtractedLink } from '../../scrape/types.js';
 export type RuleSeverity = 'error' | 'warning' | 'info';
-export type RuleCategory = 'title' | 'meta' | 'og' | 'twitter' | 'headings' | 'images' | 'links' | 'content' | 'technical' | 'security' | 'mobile' | 'structured-data' | 'performance' | 'accessibility';
+export type RuleCategory = 'title' | 'meta' | 'og' | 'twitter' | 'headings' | 'images' | 'links' | 'content' | 'technical' | 'security' | 'mobile' | 'structured-data' | 'performance' | 'accessibility' | 'ai-search' | 'resources' | 'crawlability' | 'canonicalization';
 export interface RuleContext {
     title?: string;
     titleLength?: number;
@@ -39,6 +39,8 @@ export interface RuleContext {
     altTextLengths?: number[];
     imageFilenames?: string[];
     imagesWithAsyncDecoding?: number;
+    brokenExternalImages?: number;
+    brokenExternalImageUrls?: string[];
     buttonsWithoutAriaLabel?: number;
     linksWithoutAriaLabel?: number;
     inputsWithoutLabel?: number;
@@ -76,6 +78,10 @@ export interface RuleContext {
     linksWithGenericText?: number;
     externalLinksWithoutNoopener?: number;
     externalLinksWithoutNoreferrer?: number;
+    linksToResources?: number;
+    resourceLinkUrls?: string[];
+    forbidden403Links?: number;
+    forbidden403LinkUrls?: string[];
     problematicLinks?: {
         withoutText?: ExtractedLink[];
         genericText?: ExtractedLink[];
@@ -126,6 +132,9 @@ export interface RuleContext {
     hasMixedContent?: boolean;
     responseHeaders?: Record<string, string | string[]>;
     textHtmlRatio?: number;
+    hasDeprecatedPlugins?: boolean;
+    deprecatedPluginTypes?: string[];
+    hasFrameTags?: boolean;
     hasFavicon?: boolean;
     faviconUrl?: string;
     hasPreconnect?: boolean;
@@ -156,6 +165,7 @@ export interface RuleContext {
     }>;
     ogLocale?: string;
     alternateLanguages?: string[];
+    detectedLanguage?: string;
     titleMatchesH1?: boolean;
     urlHasUppercase?: boolean;
     urlHasSpecialChars?: boolean;
@@ -257,9 +267,16 @@ export interface RuleContext {
     hasSitemapLink?: boolean;
     sitemapUrl?: string;
     robotsHasSitemap?: boolean;
+    robotsTxtExists?: boolean;
+    robotsTxtHasSitemap?: boolean;
     isPaginatedPage?: boolean;
     hasRelPrev?: boolean;
     hasRelNext?: boolean;
+    blockedResources?: number;
+    blockedResourceUrls?: string[];
+    blockedExternalResources?: number;
+    blockedExternalResourceUrls?: string[];
+    xRobotsTag?: string;
     hasDoctype?: boolean;
     httpStatusCode?: number;
     uncrawlableLinksCount?: number;
@@ -268,6 +285,10 @@ export interface RuleContext {
     structuredDataErrors?: number;
     isIndexable?: boolean;
     httpRedirectsToHttps?: boolean;
+    sniSupported?: boolean;
+    sitemapHttpUrls?: number;
+    sitemapHttpUrlsList?: string[];
+    hasHsts?: boolean;
     passiveVoicePercentage?: number;
     transitionWordPercentage?: number;
     consecutiveSentenceStarts?: number;
@@ -305,9 +326,88 @@ export interface RuleContext {
     contextualLinkCount?: number;
     incomingInternalLinks?: number;
     selfReferencingLinks?: number;
-    brokenInternalLinks?: number;
-    redirectChainLinks?: number;
+    brokenInternalLinks?: string[];
+    brokenExternalLinks?: string[];
+    redirectChainLinks?: Array<{
+        from: string;
+        to: string;
+        hops: number;
+    }>;
     pageClickDepth?: number;
+    clickDepth?: number;
+    isStartPage?: boolean;
+    nofollowInternalLinks?: number;
+    llmsTxt?: {
+        exists: boolean;
+        valid: boolean;
+        issues?: Array<{
+            message: string;
+        }>;
+        parseResult?: {
+            siteName?: string;
+            siteDescription?: string;
+            sections: Array<{
+                title: string;
+            }>;
+            links: Array<{
+                text: string;
+                url: string;
+            }>;
+        };
+    };
+    robotsTxt?: {
+        parseResult?: {
+            userAgentBlocks: Array<{
+                userAgents: string[];
+                rules: Array<{
+                    type: string;
+                    path: string;
+                }>;
+            }>;
+        };
+    };
+    headings?: {
+        structure: Array<{
+            level: number;
+            text: string;
+        }>;
+        h1Count?: number;
+        hasProperHierarchy?: boolean;
+    };
+    lastModified?: string;
+    semanticHtmlRatio?: number;
+    jsFilesCount?: number;
+    jsTotalSize?: number;
+    renderBlockingJs?: number;
+    cssFilesCount?: number;
+    cssTotalSize?: number;
+    hasCriticalCss?: boolean;
+    largeImages?: string[];
+    imagesTotal?: number;
+    modernFormatImages?: number;
+    fontFilesCount?: number;
+    hasFontDisplaySwap?: boolean;
+    totalRequests?: number;
+    totalPageSize?: number;
+    uncompressedResources?: number;
+    resourcesWithoutCaching?: number;
+    brokenExternalResources?: number;
+    brokenExternalResourceUrls?: string[];
+    unminifiedResources?: number;
+    unminifiedResourceUrls?: string[];
+    sslCertificate?: {
+        valid: boolean;
+        error?: string;
+        expiryDate?: string;
+        nameMismatch?: boolean;
+        commonName?: string;
+        expectedDomain?: string;
+        issuer?: string;
+        selfSigned?: boolean;
+    };
+    tlsVersion?: string;
+    hasPasswordField?: boolean;
+    formsOnHttp?: number;
 }
 export interface RuleEvidence {
     found?: string | number | string[];

package/dist/seo/seo-spider.d.ts

@@ -4,6 +4,8 @@ export interface SeoSpiderOptions extends SpiderOptions {
     seo?: boolean;
     output?: string;
     onSeoAnalysis?: (result: SeoPageResult) => void;
+    focusCategories?: string[];
+    focusMode?: 'all' | 'links' | 'duplicates' | 'security' | 'ai' | 'resources';
 }
 export interface SeoPageResult extends SpiderPageResult {
     seoReport?: SeoReport;

package/dist/seo/seo-spider.js

@@ -62,7 +62,13 @@ export class SeoSpider {
             try {
                 const response = await client.get(page.url);
                 const html = await response.text();
-                const seoReport = await analyzeSeo(html, { baseUrl: page.url });
+                const rulesOptions = this.options.focusCategories?.length
+                    ? { categories: this.options.focusCategories }
+                    : undefined;
+                const seoReport = await analyzeSeo(html, {
+                    baseUrl: page.url,
+                    rules: rulesOptions,
+                });
                 const seoPage = {
                     ...page,
                     seoReport,
@@ -145,11 +151,49 @@ export class SeoSpider {
             return sum;
         }, 0);
         const score = checks.length > 0 ? Math.round(scoreSum / checks.length) : 0;
+        const passed = checks.filter(c => c.status === 'pass').length;
+        const warnings = checks.filter(c => c.status === 'warn').length;
+        const errors = checks.filter(c => c.status === 'fail').length;
+        const infos = checks.filter(c => c.status === 'info').length;
+        const passRate = checks.length > 0 ? Math.round((passed / checks.length) * 100) : 0;
         return {
             url: page.url,
             timestamp: new Date(),
             grade: this.scoreToGrade(score),
             score,
+            summary: {
+                totalChecks: checks.length,
+                passed,
+                warnings,
+                errors,
+                infos,
+                passRate,
+                issuesByCategory: {},
+                topIssues: checks
+                    .filter(c => c.status === 'fail' || c.status === 'warn')
+                    .slice(0, 5)
+                    .map(c => ({
+                    name: c.name,
+                    message: c.message,
+                    category: 'general',
+                    severity: c.status === 'fail' ? 'error' : 'warning',
+                })),
+                quickWins: [],
+                vitals: {
+                    wordCount: 0,
+                    readingTime: 0,
+                    imageCount: 0,
+                    linkCount: page.links.length,
+                },
+                completeness: {
+                    meta: 0,
+                    social: 0,
+                    technical: 0,
+                    content: 0,
+                    images: 0,
+                    links: 0,
+                },
+            },
             checks,
             title: page.title ? { text: page.title, length: page.title.length } : undefined,
             headings: {
@@ -216,9 +260,10 @@ export class SeoSpider {
                 hasCharset: false,
                 hasLang: false,
             },
-            jsonLd: {
+            structuredData: {
                 count: 0,
                 types: [],
+                items: [],
             },
         };
     }

package/dist/seo/types.d.ts

@@ -38,6 +38,8 @@ export interface ContentMetrics {
     listCount: number;
     strongTagCount: number;
     emTagCount: number;
+    fleschReadingEase?: number;
+    hasQuestionHeadings?: boolean;
 }
 export interface LinkAnalysis {
     total: number;
@@ -46,6 +48,8 @@ export interface LinkAnalysis {
     nofollow: number;
     broken: number;
     withoutText: number;
+    sponsoredLinks: number;
+    ugcLinks: number;
 }
 export interface ImageAnalysis {
     total: number;
@@ -94,11 +98,50 @@ export interface SeoTiming {
     tls?: number;
     download?: number;
 }
+export interface SeoSummary {
+    totalChecks: number;
+    passed: number;
+    warnings: number;
+    errors: number;
+    infos: number;
+    passRate: number;
+    issuesByCategory: Record<string, {
+        passed: number;
+        warnings: number;
+        errors: number;
+    }>;
+    topIssues: Array<{
+        name: string;
+        message: string;
+        category: string;
+        severity: 'error' | 'warning';
+    }>;
+    quickWins: string[];
+    vitals: {
+        htmlSize?: number;
+        domElements?: number;
+        ttfb?: number;
+        totalTime?: number;
+        wordCount: number;
+        readingTime: number;
+        imageCount: number;
+        linkCount: number;
+    };
+    completeness: {
+        meta: number;
+        social: number;
+        technical: number;
+        content: number;
+        images: number;
+        links: number;
+    };
+}
 export interface SeoReport {
     url: string;
     timestamp: Date;
     grade: string;
     score: number;
+    summary: SeoSummary;
     timing?: SeoTiming;
     checks: SeoCheckResult[];
     title?: {
@@ -124,16 +167,17 @@ export interface SeoReport {
         image?: string;
         site?: string;
     };
+    structuredData: {
+        count: number;
+        types: string[];
+        items: Record<string, unknown>[];
+    };
     headings: HeadingAnalysis;
     content: ContentMetrics;
     links: LinkAnalysis;
     images: ImageAnalysis;
     social: SocialMetaAnalysis;
     technical: TechnicalSeo;
-    jsonLd: {
-        count: number;
-        types: string[];
-    };
 }
 export interface SeoAnalyzerOptions {
     baseUrl?: string;
@@ -158,27 +202,3 @@ export interface ExtractedImage {
     srcset?: string;
     loading?: 'lazy' | 'eager';
 }
-export interface LinkAnalysis {
-    total: number;
-    internal: number;
-    external: number;
-    nofollow: number;
-    broken: number;
-    withoutText: number;
-    sponsoredLinks: number;
-    ugcLinks: number;
-}
-export interface ContentMetrics {
-    wordCount: number;
-    characterCount: number;
-    sentenceCount: number;
-    paragraphCount: number;
-    readingTimeMinutes: number;
-    avgWordsPerSentence: number;
-    avgParagraphLength: number;
-    listCount: number;
-    strongTagCount: number;
-    emTagCount: number;
-    fleschReadingEase?: number;
-    hasQuestionHeadings?: boolean;
-}

package/dist/seo/utils/index.d.ts

@@ -0,0 +1 @@
+export * from './similarity.js';

package/dist/seo/utils/index.js

@@ -0,0 +1 @@
+export * from './similarity.js';

package/dist/seo/utils/similarity.d.ts

@@ -0,0 +1,47 @@
+export interface SimilarityResult {
+    urlA: string;
+    urlB: string;
+    similarity: number;
+    type: 'exact' | 'near-duplicate' | 'similar' | 'different';
+    simhashDistance?: number;
+    jaccardIndex?: number;
+}
+export interface DuplicateGroup {
+    type: 'title' | 'description' | 'h1' | 'content';
+    value: string;
+    urls: string[];
+    similarity?: number;
+}
+export interface ContentFingerprint {
+    url: string;
+    simhash: bigint;
+    shingles: Set<number>;
+    wordCount: number;
+    normalizedText: string;
+}
+export declare const DEFAULT_SIMILARITY_THRESHOLD = 85;
+export declare function normalizeText(text: string, removeStopWords?: boolean): string;
+export declare function tokenize(text: string): string[];
+export declare function createShingles(tokens: string[], size?: number): Set<number>;
+export declare function calculateSimHash(text: string): bigint;
+export declare function hammingDistance(hash1: bigint, hash2: bigint): number;
+export declare function simhashSimilarity(hash1: bigint, hash2: bigint): number;
+export declare function jaccardSimilarity(setA: Set<number>, setB: Set<number>): number;
+export declare function createFingerprint(url: string, text: string): ContentFingerprint;
+export declare function compareFingerprints(fpA: ContentFingerprint, fpB: ContentFingerprint): SimilarityResult;
+export declare function findDuplicateContent(pages: Array<{
+    url: string;
+    content: string;
+}>, threshold?: number): SimilarityResult[];
+export declare function findDuplicateMetadata(pages: Array<{
+    url: string;
+    title?: string;
+    description?: string;
+    h1?: string;
+}>, threshold?: number): DuplicateGroup[];
+export declare function calculateStringSimilarity(a: string, b: string): number;
+export declare function calculateTextToHtmlRatio(html: string, text: string): number;
+export declare function isThinContent(wordCount: number, textToHtmlRatio?: number, minWords?: number, minRatio?: number): {
+    isThin: boolean;
+    reason?: string;
+};