recker 1.0.43 → 1.0.44-next.084c2ef

package/dist/seo/rules/types.d.ts

@@ -1,8 +1,25 @@
 import type { SeoStatus } from '../types.js';
 import type { ExtractedLink } from '../../scrape/types.js';
 export type RuleSeverity = 'error' | 'warning' | 'info';
-export type RuleCategory = 'title' | 'meta' | 'og' | 'twitter' | 'headings' | 'images' | 'links' | 'content' | 'technical' | 'security' | 'mobile' | 'structured-data' | 'performance' | 'accessibility' | 'ai-search' | 'resources' | 'crawlability' | 'canonicalization';
+export type RuleCategory = 'title' | 'meta' | 'og' | 'twitter' | 'headings' | 'images' | 'links' | 'content' | 'technical' | 'security' | 'mobile' | 'structured-data' | 'performance' | 'accessibility' | 'i18n' | 'ai-search' | 'resources' | 'crawlability' | 'canonicalization';
 export interface RuleContext {
+    keywordsInTitle?: boolean;
+    keywordsInDescription?: boolean;
+    keywordsInH1?: boolean;
+    keywordsInUrl?: boolean;
+    keywordsInFirstParagraph?: boolean;
+    keywordsInAltText?: boolean;
+    keywordConsistencyScore?: number;
+    keywordConsistencyDetails?: {
+        inTitle: boolean;
+        inDescription: boolean;
+        inH1: boolean;
+        inUrl: boolean;
+        inFirstParagraph: boolean;
+        inAltText: boolean;
+    };
+    topKeywords?: string[];
+    mainKeyword?: string;
     title?: string;
     titleLength?: number;
     metaDescription?: string;
@@ -34,9 +51,12 @@ export interface RuleContext {
     imagesWithDimensions?: number;
     imagesMissingDimensions?: number;
     imagesWithEmptyAlt?: number;
+    imagesWithSrcset?: number;
+    largeBase64ImagesCount?: number;
     imagesDecorativeCount?: number;
     imagesUsingModernFormats?: number;
     altTextLengths?: number[];
+    imageAltTexts?: string[];
     imageFilenames?: string[];
     imagesWithAsyncDecoding?: number;
     brokenExternalImages?: number;
@@ -70,6 +90,8 @@ export interface RuleContext {
     totalLinks?: number;
     internalLinks?: number;
     externalLinks?: number;
+    internalHttpLinks?: number;
+    internalHttpLinkUrls?: string[];
     linksWithoutText?: number;
     nofollowLinks?: number;
     sponsoredLinks?: number;
@@ -89,6 +111,8 @@ export interface RuleContext {
         missingNoreferrer?: ExtractedLink[];
     };
     wordCount?: number;
+    emailsFound?: string[];
+    socialLinksFound?: string[];
     characterCount?: number;
     sentenceCount?: number;
     paragraphCount?: number;
@@ -102,7 +126,6 @@ export interface RuleContext {
     avgSentenceLength?: number;
     faqCount?: number;
     imagePerWordRatio?: number;
-    mainKeyword?: string;
     keywordDensity?: number;
     fleschReadingEase?: number;
     hasQuestionHeadings?: boolean;
@@ -120,6 +143,7 @@ export interface RuleContext {
     hasBreadcrumbsSchema?: boolean;
     videoCount?: number;
     audioCount?: number;
+    hasAutoplay?: boolean;
     hasCanonical?: boolean;
     canonicalUrl?: string;
     hasViewport?: boolean;
@@ -134,7 +158,10 @@ export interface RuleContext {
     textHtmlRatio?: number;
     hasDeprecatedPlugins?: boolean;
     deprecatedPluginTypes?: string[];
+    deprecatedTagsCount?: number;
+    deprecatedTagsFound?: string[];
     hasFrameTags?: boolean;
+    iframeCount?: number;
     hasFavicon?: boolean;
     faviconUrl?: string;
     hasPreconnect?: boolean;
@@ -307,6 +334,26 @@ export interface RuleContext {
     manifestShortName?: string;
     manifestName?: string;
     manifestBackgroundColor?: string;
+    manifestThemeColor?: string;
+    manifestDescription?: string;
+    manifestOrientation?: string;
+    manifestLang?: string;
+    manifestDir?: string;
+    manifestCategories?: string[];
+    manifestId?: string;
+    manifestScreenshots?: Array<{
+        src: string;
+        sizes?: string;
+        type?: string;
+        form_factor?: string;
+        label?: string;
+    }>;
+    manifestShortcuts?: Array<{
+        name: string;
+        url: string;
+        description?: string;
+    }>;
+    hasServiceWorker?: boolean;
     ogImageDimensions?: {
         width: number;
         height: number;
@@ -321,6 +368,21 @@ export interface RuleContext {
     pinterestRichPinSupport?: boolean;
     hasPinterestNopin?: boolean;
     fbAppId?: string;
+    totalSocialLinks?: number;
+    socialLinksInHeader?: number;
+    socialLinksInFooter?: number;
+    socialLinksWithoutAccessibility?: number;
+    socialLinksWithoutNewTab?: number;
+    socialLinksWithoutNoopener?: number;
+    platformsFound?: string[];
+    socialLinkDetails?: Array<{
+        href: string;
+        platform: string;
+        hasAccessibility: boolean;
+        hasNewTab: boolean;
+        hasNoopener: boolean;
+        location: 'header' | 'footer' | 'body';
+    }>;
     navLinkCount?: number;
     footerLinkCount?: number;
     contextualLinkCount?: number;
@@ -408,6 +470,16 @@ export interface RuleContext {
     tlsVersion?: string;
     hasPasswordField?: boolean;
     formsOnHttp?: number;
+    analyticsDetected?: boolean;
+    analyticsProviders?: string[];
+    hasRssFeed?: boolean;
+    rssFeedUrl?: string;
+    hasAtomFeed?: boolean;
+    atomFeedUrl?: string;
+    ctaButtonsCount?: number;
+    formCount?: number;
+    hasWhatsAppLink?: boolean;
+    pageInSitemap?: boolean;
 }
 export interface RuleEvidence {
     found?: string | number | string[];

package/dist/seo/seo-spider.d.ts

@@ -1,5 +1,6 @@
 import { SpiderOptions, SpiderResult, SpiderPageResult } from '../scrape/spider.js';
 import type { SeoReport } from './types.js';
+import { type SitemapValidationResult } from './validators/sitemap.js';
 export interface SeoSpiderOptions extends SpiderOptions {
     seo?: boolean;
     output?: string;
@@ -20,6 +21,37 @@ export interface SiteWideIssue {
 export interface SeoSpiderResult extends Omit<SpiderResult, 'pages'> {
     pages: SeoPageResult[];
     siteWideIssues: SiteWideIssue[];
+    discovery?: {
+        humans: {
+            found: boolean;
+            content?: string;
+            url: string;
+        };
+        llms: {
+            found: boolean;
+            content?: string;
+            url: string;
+        };
+        sitemap: {
+            found: boolean;
+            url: string;
+            urlCount?: number;
+        };
+        manifest: {
+            found: boolean;
+            url: string;
+            content?: Record<string, unknown>;
+            valid?: boolean;
+            issues?: string[];
+        };
+    };
+    rssFeeds?: Array<{
+        url: string;
+        type: 'rss' | 'atom' | 'unknown';
+        title?: string;
+        itemCount: number;
+    }>;
+    sitemapValidation?: SitemapValidationResult;
     summary: {
         totalPages: number;
         pagesWithErrors: number;
@@ -37,6 +69,9 @@ export declare class SeoSpider {
     private seoResults;
     constructor(options?: SeoSpiderOptions);
     crawl(startUrl: string): Promise<SeoSpiderResult>;
+    private checkSiteFiles;
+    private validateManifest;
+    private validateSitemap;
     private analyzePages;
     private createReportFromPageData;
     private detectSiteWideIssues;

package/dist/seo/seo-spider.js

@@ -1,6 +1,8 @@
 import { Spider } from '../scrape/spider.js';
 import { analyzeSeo } from './analyzer.js';
 import { createClient } from '../core/client.js';
+import { discoverFeeds } from './validators/rss.js';
+import { fetchAndValidateSitemap } from './validators/sitemap.js';
 import * as fs from 'fs/promises';
 export class SeoSpider {
     spider;
@@ -32,10 +34,23 @@ export class SeoSpider {
         const seoPages = await this.analyzePages(result.pages);
         const siteWideIssues = this.detectSiteWideIssues(seoPages);
         const summary = this.calculateSummary(seoPages, siteWideIssues);
+        const discovery = await this.checkSiteFiles(startUrl);
+        let homeHtml = '';
+        try {
+            const client = createClient({ timeout: 10000 });
+            const res = await client.get(startUrl);
+            homeHtml = await res.text();
+        }
+        catch { }
+        const rssFeeds = await discoverFeeds(new URL(startUrl).origin, homeHtml);
+        const sitemapValidation = await this.validateSitemap(startUrl);
         const seoResult = {
             ...result,
             pages: seoPages,
             siteWideIssues,
+            discovery,
+            rssFeeds,
+            sitemapValidation,
             summary,
         };
         if (this.options.output) {
@@ -43,6 +58,132 @@ export class SeoSpider {
         }
         return seoResult;
     }
+    async checkSiteFiles(startUrl) {
+        try {
+            const baseUrl = new URL(startUrl).origin;
+            const client = createClient({ timeout: 5000 });
+            const results = {
+                humans: { found: false, content: undefined, url: `${baseUrl}/humans.txt` },
+                llms: { found: false, content: undefined, url: `${baseUrl}/llms.txt` },
+                sitemap: { found: false, url: `${baseUrl}/sitemap.xml`, urlCount: undefined },
+                manifest: { found: false, url: `${baseUrl}/manifest.json`, content: undefined, valid: undefined, issues: undefined },
+            };
+            try {
+                const res = await client.get(results.humans.url);
+                if (res.status === 200) {
+                    results.humans.found = true;
+                    results.humans.content = await res.text();
+                }
+            }
+            catch { }
+            try {
+                const res = await client.get(results.llms.url);
+                if (res.status === 200) {
+                    results.llms.found = true;
+                    results.llms.content = await res.text();
+                }
+            }
+            catch { }
+            try {
+                const res = await client.get(results.sitemap.url);
+                if (res.status === 200) {
+                    results.sitemap.found = true;
+                    const content = await res.text();
+                    const urlMatches = content.match(/<loc>/g);
+                    results.sitemap.urlCount = urlMatches ? urlMatches.length : 0;
+                }
+            }
+            catch { }
+            try {
+                let res = await client.get(results.manifest.url);
+                if (res.status !== 200) {
+                    results.manifest.url = `${baseUrl}/site.webmanifest`;
+                    res = await client.get(results.manifest.url);
+                }
+                if (res.status === 200) {
+                    results.manifest.found = true;
+                    const text = await res.text();
+                    try {
+                        const manifest = JSON.parse(text);
+                        results.manifest.content = manifest;
+                        results.manifest.valid = true;
+                        results.manifest.issues = this.validateManifest(manifest);
+                    }
+                    catch {
+                        results.manifest.valid = false;
+                        results.manifest.issues = ['Invalid JSON format'];
+                    }
+                }
+            }
+            catch { }
+            return results;
+        }
+        catch {
+            return undefined;
+        }
+    }
+    validateManifest(manifest) {
+        const issues = [];
+        if (!manifest.name && !manifest.short_name) {
+            issues.push('Missing required field: name or short_name');
+        }
+        if (!manifest.icons || !Array.isArray(manifest.icons) || manifest.icons.length === 0) {
+            issues.push('Missing icons array');
+        }
+        else {
+            const icons = manifest.icons;
+            const sizes = icons.map(i => i.sizes).filter(Boolean);
+            const has192 = sizes.some(s => s?.includes('192'));
+            const has512 = sizes.some(s => s?.includes('512'));
+            if (!has192)
+                issues.push('Missing 192x192 icon (required for Add to Home Screen)');
+            if (!has512)
+                issues.push('Missing 512x512 icon (required for splash screen)');
+            const hasMaskable = icons.some(i => i.purpose?.includes('maskable'));
+            if (!hasMaskable)
+                issues.push('No maskable icon found (recommended for Android adaptive icons)');
+        }
+        if (!manifest.display) {
+            issues.push('Missing display mode (recommended: standalone)');
+        }
+        if (!manifest.start_url) {
+            issues.push('Missing start_url');
+        }
+        if (!manifest.theme_color) {
+            issues.push('Missing theme_color');
+        }
+        if (!manifest.background_color) {
+            issues.push('Missing background_color');
+        }
+        if (!manifest.scope) {
+            issues.push('Missing scope (defines navigation boundaries)');
+        }
+        if (manifest.short_name && typeof manifest.short_name === 'string' && manifest.short_name.length > 12) {
+            issues.push(`short_name too long (${manifest.short_name.length} chars, max 12)`);
+        }
+        return issues;
+    }
+    async validateSitemap(startUrl) {
+        try {
+            const baseUrl = new URL(startUrl).origin;
+            const sitemapUrl = `${baseUrl}/sitemap.xml`;
+            const client = createClient({ timeout: this.options.timeout || 15000 });
+            const fetcher = async (url) => {
+                const res = await client.get(url);
+                const text = await res.text();
+                return {
+                    status: res.status,
+                    text,
+                    headers: Object.fromEntries([...res.headers.entries()]),
+                };
+            };
+            const result = await fetchAndValidateSitemap(sitemapUrl, fetcher);
+            return result;
+        }
+        catch {
+            return undefined;
+        }
+    }
     async analyzePages(pages) {
         const results = [];
         const client = createClient({
@@ -93,8 +234,9 @@ export class SeoSpider {
             if (titleLength < 30) {
                 checks.push({
                     name: 'Title Length',
+                    category: 'title',
                     status: 'warn',
-                    message: `Title is too short (${titleLength} chars)`,
+                    message: `Title is ${titleLength} characters`,
                     value: titleLength,
                     recommendation: 'Title should be 50-60 characters',
                 });
@@ -102,6 +244,7 @@ export class SeoSpider {
             else if (titleLength > 60) {
                 checks.push({
                     name: 'Title Length',
+                    category: 'title',
                     status: 'warn',
                     message: `Title is too long (${titleLength} chars)`,
                     value: titleLength,
@@ -111,6 +254,7 @@ export class SeoSpider {
             else {
                 checks.push({
                     name: 'Title Length',
+                    category: 'title',
                     status: 'pass',
                     message: `Good title length (${titleLength} chars)`,
                     value: titleLength,
@@ -120,6 +264,7 @@ export class SeoSpider {
         else {
             checks.push({
                 name: 'Title',
+                category: 'title',
                 status: 'fail',
                 message: 'Page has no title',
                 recommendation: 'Add a descriptive <title> tag',
@@ -130,6 +275,7 @@ export class SeoSpider {
         if (internalLinks === 0) {
             checks.push({
                 name: 'Internal Links',
+                category: 'links',
                 status: 'warn',
                 message: 'No internal links found',
                 recommendation: 'Add internal links to improve site structure',
@@ -138,6 +284,7 @@ export class SeoSpider {
         else {
             checks.push({
                 name: 'Internal Links',
+                category: 'links',
                 status: 'pass',
                 message: `${internalLinks} internal links found`,
                 value: internalLinks,
@@ -232,27 +379,19 @@ export class SeoSpider {
                 missingDimensions: 0,
                 modernFormats: 0,
                 altTextLengths: [],
+                imageAltTexts: [],
                 imageFilenames: [],
                 imagesWithAsyncDecoding: 0,
             },
             social: {
                 openGraph: {
-                    present: false,
-                    hasTitle: false,
-                    hasDescription: false,
-                    hasImage: false,
-                    hasUrl: false,
-                    issues: [],
+                    present: false, hasTitle: false, hasDescription: false, hasImage: false, hasUrl: false, issues: []
                 },
                 twitterCard: {
-                    present: false,
-                    hasCard: false,
-                    hasTitle: false,
-                    hasDescription: false,
-                    hasImage: false,
-                    issues: [],
+                    present: false, hasCard: false, hasTitle: false, hasDescription: false, hasImage: false, issues: []
                 },
             },
+            keywords: { totalWords: 0, uniqueWords: 0, topKeywords: [] },
             technical: {
                 hasCanonical: false,
                 hasRobotsMeta: false,

package/dist/seo/types.d.ts

@@ -1,3 +1,5 @@
+import { KeywordCloud } from './keywords.js';
+export type { KeywordCloud, KeywordItem } from './keywords.js';
 export type SeoStatus = 'pass' | 'warn' | 'fail' | 'info';
 export interface SeoCheckEvidence {
     found?: string | number | string[];
@@ -10,6 +12,7 @@ export interface SeoCheckEvidence {
 }
 export interface SeoCheckResult {
     name: string;
+    category: string;
     status: SeoStatus;
     message: string;
     value?: string | number;
@@ -50,6 +53,8 @@ export interface LinkAnalysis {
     withoutText: number;
     sponsoredLinks: number;
     ugcLinks: number;
+    internalHttpLinks?: number;
+    internalHttpLinkUrls?: string[];
 }
 export interface ImageAnalysis {
     total: number;
@@ -59,6 +64,7 @@ export interface ImageAnalysis {
     missingDimensions: number;
     modernFormats: number;
     altTextLengths: number[];
+    imageAltTexts: string[];
     imageFilenames: string[];
     imagesWithAsyncDecoding: number;
 }
@@ -172,8 +178,9 @@ export interface SeoReport {
         types: string[];
         items: Record<string, unknown>[];
     };
-    headings: HeadingAnalysis;
     content: ContentMetrics;
+    headings: HeadingAnalysis;
+    keywords: KeywordCloud;
     links: LinkAnalysis;
     images: ImageAnalysis;
     social: SocialMetaAnalysis;

package/dist/seo/validators/llms-txt.js

@@ -1,6 +1,8 @@
 const MAX_FILE_SIZE = 100 * 1024;
 const MIN_DESCRIPTION_LENGTH = 50;
 const MAX_DESCRIPTION_LENGTH = 500;
+const OPTIMAL_MIN_LINKS = 10;
+const OPTIMAL_MAX_LINKS = 30;
 export function parseLlmsTxt(content) {
     const errors = [];
     const warnings = [];
@@ -181,6 +183,23 @@ export function validateLlmsTxt(content, baseUrl) {
             seenUrls.add(normalized);
         }
     }
+    const linkCount = parseResult.links.length;
+    if (linkCount > 0 && linkCount < OPTIMAL_MIN_LINKS) {
+        issues.push({
+            type: 'info',
+            code: 'FEW_LINKS',
+            message: `Only ${linkCount} link(s) found in llms.txt`,
+            recommendation: `Consider adding ${OPTIMAL_MIN_LINKS}-${OPTIMAL_MAX_LINKS} of your most valuable pages for better AI coverage`,
+        });
+    }
+    else if (linkCount > OPTIMAL_MAX_LINKS) {
+        issues.push({
+            type: 'info',
+            code: 'MANY_LINKS',
+            message: `${linkCount} links found in llms.txt`,
+            recommendation: `Focus on quality over quantity. ${OPTIMAL_MIN_LINKS}-${OPTIMAL_MAX_LINKS} high-value links are recommended to help AI systems identify your truly important content`,
+        });
+    }
     return {
         valid: issues.filter(i => i.type === 'error').length === 0,
         issues,

package/dist/seo/validators/rss.d.ts

@@ -0,0 +1,11 @@
+export interface RssFeed {
+    url: string;
+    type: 'rss' | 'atom' | 'unknown';
+    title?: string;
+    description?: string;
+    itemCount: number;
+    lastBuildDate?: string;
+    isValid: boolean;
+    error?: string;
+}
+export declare function discoverFeeds(baseUrl: string, html?: string): Promise<RssFeed[]>;

package/dist/seo/validators/rss.js

@@ -0,0 +1,93 @@
+import { createClient } from '../../core/client.js';
+const COMMON_PATHS = [
+    '/rss.xml',
+    '/feed.xml',
+    '/rss',
+    '/feed',
+    '/atom.xml',
+    '/feeds/posts/default',
+    '/index.xml'
+];
+export async function discoverFeeds(baseUrl, html) {
+    const candidateUrls = new Set();
+    const feeds = [];
+    if (html) {
+        const linkRegex = /<link[^>]+?type=["']application\/(rss\+xml|atom\+xml)["'][^>]*?>/gi;
+        const hrefRegex = /href=["']([^"']+)["']/;
+        const titleRegex = /title=["']([^"']+)["']/;
+        let match;
+        while ((match = linkRegex.exec(html)) !== null) {
+            const tag = match[0];
+            const hrefMatch = hrefRegex.exec(tag);
+            if (hrefMatch) {
+                let href = hrefMatch[1];
+                try {
+                    href = new URL(href, baseUrl).toString();
+                    candidateUrls.add(href);
+                }
+                catch { }
+            }
+        }
+    }
+    if (candidateUrls.size === 0) {
+        for (const path of COMMON_PATHS) {
+            try {
+                const url = new URL(path, baseUrl).toString();
+                candidateUrls.add(url);
+            }
+            catch { }
+        }
+    }
+    const client = createClient({ timeout: 8000 });
+    await Promise.all(Array.from(candidateUrls).map(async (url) => {
+        try {
+            const response = await client.get(url);
+            if (response.status !== 200)
+                return;
+            const contentType = response.headers.get('content-type') || '';
+            const text = await response.text();
+            let type = 'unknown';
+            let isValid = false;
+            let itemCount = 0;
+            let title;
+            let description;
+            let lastBuildDate;
+            if (text.includes('<rss') && text.includes('version="2.0"')) {
+                type = 'rss';
+                isValid = true;
+                itemCount = (text.match(/<item>/g) || []).length;
+                const titleMatch = text.match(/<channel>[\s\S]*?<title>(.*?)<\/title>/);
+                if (titleMatch)
+                    title = titleMatch[1].replace(/<!\[CDATA\[(.*?)\]\]>/g, '$1').trim();
+                const descMatch = text.match(/<channel>[\s\S]*?<description>(.*?)<\/description>/);
+                if (descMatch)
+                    description = descMatch[1].replace(/<!\[CDATA\[(.*?)\]\]>/g, '$1').trim();
+                const dateMatch = text.match(/<lastBuildDate>(.*?)<\/lastBuildDate>/);
+                if (dateMatch)
+                    lastBuildDate = dateMatch[1];
+            }
+            else if (text.includes('<feed') && text.includes('xmlns="http://www.w3.org/2005/Atom"')) {
+                type = 'atom';
+                isValid = true;
+                itemCount = (text.match(/<entry>/g) || []).length;
+                const titleMatch = text.match(/<title>(.*?)<\/title>/);
+                if (titleMatch)
+                    title = titleMatch[1].trim();
+            }
+            if (isValid) {
+                feeds.push({
+                    url,
+                    type,
+                    isValid,
+                    title,
+                    description,
+                    itemCount,
+                    lastBuildDate
+                });
+            }
+        }
+        catch (err) {
+        }
+    }));
+    return feeds;
+}