recker 1.0.28 → 1.0.29

package/dist/seo/seo-spider.js

@@ -0,0 +1,362 @@
+import { Spider } from '../scrape/spider.js';
+import { analyzeSeo } from './analyzer.js';
+import { createClient } from '../core/client.js';
+import * as fs from 'fs/promises';
+export class SeoSpider {
+    spider;
+    options;
+    seoResults = new Map();
+    constructor(options = {}) {
+        this.options = options;
+        this.spider = new Spider(options);
+    }
+    async crawl(startUrl) {
+        const result = await this.spider.crawl(startUrl);
+        if (!this.options.seo) {
+            return {
+                ...result,
+                pages: result.pages,
+                siteWideIssues: [],
+                summary: {
+                    totalPages: result.pages.length,
+                    pagesWithErrors: 0,
+                    pagesWithWarnings: 0,
+                    avgScore: 0,
+                    duplicateTitles: 0,
+                    duplicateDescriptions: 0,
+                    duplicateH1s: 0,
+                    orphanPages: 0,
+                },
+            };
+        }
+        const seoPages = await this.analyzePages(result.pages);
+        const siteWideIssues = this.detectSiteWideIssues(seoPages);
+        const summary = this.calculateSummary(seoPages, siteWideIssues);
+        const seoResult = {
+            ...result,
+            pages: seoPages,
+            siteWideIssues,
+            summary,
+        };
+        if (this.options.output) {
+            await this.saveReport(seoResult);
+        }
+        return seoResult;
+    }
+    async analyzePages(pages) {
+        const results = [];
+        const client = createClient({
+            timeout: this.options.timeout || 10000,
+            headers: {
+                'User-Agent': this.options.userAgent || 'Recker Spider/1.0',
+            },
+        });
+        for (const page of pages) {
+            if (page.error || page.status >= 400) {
+                results.push({
+                    ...page,
+                    seoReport: undefined,
+                });
+                continue;
+            }
+            try {
+                const response = await client.get(page.url);
+                const html = await response.text();
+                const seoReport = await analyzeSeo(html, { baseUrl: page.url });
+                const seoPage = {
+                    ...page,
+                    seoReport,
+                };
+                results.push(seoPage);
+                this.seoResults.set(page.url, seoReport);
+                this.options.onSeoAnalysis?.(seoPage);
+            }
+            catch {
+                results.push({
+                    ...page,
+                    seoReport: undefined,
+                });
+            }
+        }
+        return results;
+    }
+    createReportFromPageData(page) {
+        const checks = [];
+        if (page.title) {
+            const titleLength = page.title.length;
+            if (titleLength < 30) {
+                checks.push({
+                    name: 'Title Length',
+                    status: 'warn',
+                    message: `Title is too short (${titleLength} chars)`,
+                    value: titleLength,
+                    recommendation: 'Title should be 50-60 characters',
+                });
+            }
+            else if (titleLength > 60) {
+                checks.push({
+                    name: 'Title Length',
+                    status: 'warn',
+                    message: `Title is too long (${titleLength} chars)`,
+                    value: titleLength,
+                    recommendation: 'Title should be 50-60 characters',
+                });
+            }
+            else {
+                checks.push({
+                    name: 'Title Length',
+                    status: 'pass',
+                    message: `Good title length (${titleLength} chars)`,
+                    value: titleLength,
+                });
+            }
+        }
+        else {
+            checks.push({
+                name: 'Title',
+                status: 'fail',
+                message: 'Page has no title',
+                recommendation: 'Add a descriptive <title> tag',
+            });
+        }
+        const internalLinks = page.links.filter(l => l.type === 'internal').length;
+        const externalLinks = page.links.filter(l => l.type === 'external').length;
+        if (internalLinks === 0) {
+            checks.push({
+                name: 'Internal Links',
+                status: 'warn',
+                message: 'No internal links found',
+                recommendation: 'Add internal links to improve site structure',
+            });
+        }
+        else {
+            checks.push({
+                name: 'Internal Links',
+                status: 'pass',
+                message: `${internalLinks} internal links found`,
+                value: internalLinks,
+            });
+        }
+        const scoreSum = checks.reduce((sum, c) => {
+            if (c.status === 'pass')
+                return sum + 100;
+            if (c.status === 'warn')
+                return sum + 50;
+            return sum;
+        }, 0);
+        const score = checks.length > 0 ? Math.round(scoreSum / checks.length) : 0;
+        return {
+            url: page.url,
+            timestamp: new Date(),
+            grade: this.scoreToGrade(score),
+            score,
+            checks,
+            title: page.title ? { text: page.title, length: page.title.length } : undefined,
+            headings: {
+                structure: [],
+                h1Count: 0,
+                hasProperHierarchy: false,
+                issues: [],
+            },
+            content: {
+                wordCount: 0,
+                characterCount: 0,
+                sentenceCount: 0,
+                paragraphCount: 0,
+                readingTimeMinutes: 0,
+                avgWordsPerSentence: 0,
+                avgParagraphLength: 0,
+                listCount: 0,
+                strongTagCount: 0,
+                emTagCount: 0,
+            },
+            links: {
+                total: page.links.length,
+                internal: internalLinks,
+                external: externalLinks,
+                nofollow: 0,
+                broken: 0,
+                withoutText: page.links.filter(l => !l.text?.trim()).length,
+                sponsoredLinks: 0,
+                ugcLinks: 0,
+            },
+            images: {
+                total: 0,
+                withAlt: 0,
+                withoutAlt: 0,
+                lazy: 0,
+                missingDimensions: 0,
+                modernFormats: 0,
+                altTextLengths: [],
+                imageFilenames: [],
+                imagesWithAsyncDecoding: 0,
+            },
+            social: {
+                openGraph: {
+                    present: false,
+                    hasTitle: false,
+                    hasDescription: false,
+                    hasImage: false,
+                    hasUrl: false,
+                    issues: [],
+                },
+                twitterCard: {
+                    present: false,
+                    hasCard: false,
+                    hasTitle: false,
+                    hasDescription: false,
+                    hasImage: false,
+                    issues: [],
+                },
+            },
+            technical: {
+                hasCanonical: false,
+                hasRobotsMeta: false,
+                hasViewport: false,
+                hasCharset: false,
+                hasLang: false,
+            },
+            jsonLd: {
+                count: 0,
+                types: [],
+            },
+        };
+    }
+    detectSiteWideIssues(pages) {
+        const issues = [];
+        const titleGroups = new Map();
+        const descriptionGroups = new Map();
+        const h1Groups = new Map();
+        for (const page of pages) {
+            if (!page.seoReport)
+                continue;
+            const title = page.seoReport.title?.text?.trim();
+            if (title) {
+                const urls = titleGroups.get(title) || [];
+                urls.push(page.url);
+                titleGroups.set(title, urls);
+            }
+            const desc = page.seoReport.metaDescription?.text?.trim();
+            if (desc) {
+                const urls = descriptionGroups.get(desc) || [];
+                urls.push(page.url);
+                descriptionGroups.set(desc, urls);
+            }
+            const h1 = page.seoReport.headings?.structure?.find(h => h.level === 1)?.text?.trim();
+            if (h1) {
+                const urls = h1Groups.get(h1) || [];
+                urls.push(page.url);
+                h1Groups.set(h1, urls);
+            }
+        }
+        for (const [title, urls] of titleGroups) {
+            if (urls.length > 1) {
+                issues.push({
+                    type: 'duplicate-title',
+                    severity: 'error',
+                    message: `${urls.length} pages share the same title`,
+                    affectedUrls: urls,
+                    value: title,
+                });
+            }
+        }
+        for (const [desc, urls] of descriptionGroups) {
+            if (urls.length > 1) {
+                issues.push({
+                    type: 'duplicate-description',
+                    severity: 'warning',
+                    message: `${urls.length} pages share the same meta description`,
+                    affectedUrls: urls,
+                    value: desc,
+                });
+            }
+        }
+        for (const [h1, urls] of h1Groups) {
+            if (urls.length > 1) {
+                issues.push({
+                    type: 'duplicate-h1',
+                    severity: 'warning',
+                    message: `${urls.length} pages share the same H1 heading`,
+                    affectedUrls: urls,
+                    value: h1,
+                });
+            }
+        }
+        const linkedUrls = new Set();
+        for (const page of pages) {
+            for (const link of page.links) {
+                if (link.type === 'internal' && link.href) {
+                    linkedUrls.add(link.href);
+                }
+            }
+        }
+        const orphanPages = pages
+            .filter(p => !linkedUrls.has(p.url) && p.depth > 0)
+            .map(p => p.url);
+        if (orphanPages.length > 0) {
+            issues.push({
+                type: 'orphan-page',
+                severity: 'warning',
+                message: `${orphanPages.length} page(s) have no internal links pointing to them`,
+                affectedUrls: orphanPages,
+            });
+        }
+        return issues;
+    }
+    calculateSummary(pages, issues) {
+        const pagesWithSeo = pages.filter(p => p.seoReport);
+        const scores = pagesWithSeo.map(p => p.seoReport.score);
+        const avgScore = scores.length > 0
+            ? Math.round(scores.reduce((a, b) => a + b, 0) / scores.length)
+            : 0;
+        const pagesWithErrors = pagesWithSeo.filter(p => p.seoReport.checks.some(c => c.status === 'fail')).length;
+        const pagesWithWarnings = pagesWithSeo.filter(p => p.seoReport.checks.some(c => c.status === 'warn')).length;
+        const duplicateTitles = issues.filter(i => i.type === 'duplicate-title').length;
+        const duplicateDescriptions = issues.filter(i => i.type === 'duplicate-description').length;
+        const duplicateH1s = issues.filter(i => i.type === 'duplicate-h1').length;
+        const orphanPages = issues
+            .filter(i => i.type === 'orphan-page')
+            .reduce((sum, i) => sum + i.affectedUrls.length, 0);
+        return {
+            totalPages: pages.length,
+            pagesWithErrors,
+            pagesWithWarnings,
+            avgScore,
+            duplicateTitles,
+            duplicateDescriptions,
+            duplicateH1s,
+            orphanPages,
+        };
+    }
+    scoreToGrade(score) {
+        if (score >= 90)
+            return 'A';
+        if (score >= 80)
+            return 'B';
+        if (score >= 70)
+            return 'C';
+        if (score >= 60)
+            return 'D';
+        return 'F';
+    }
+    async saveReport(result) {
+        if (!this.options.output)
+            return;
+        const reportData = {
+            ...result,
+            visited: Array.from(result.visited),
+            generatedAt: new Date().toISOString(),
+        };
+        await fs.writeFile(this.options.output, JSON.stringify(reportData, null, 2), 'utf-8');
+    }
+    abort() {
+        this.spider.abort();
+    }
+    isRunning() {
+        return this.spider.isRunning();
+    }
+}
+export async function seoSpider(url, options) {
+    const spider = new SeoSpider(options);
+    return spider.crawl(url);
+}

package/dist/seo/types.d.ts

@@ -86,11 +86,20 @@ export interface TechnicalSeo {
     hasLang: boolean;
     langValue?: string;
 }
+export interface SeoTiming {
+    ttfb?: number;
+    total?: number;
+    dns?: number;
+    tcp?: number;
+    tls?: number;
+    download?: number;
+}
 export interface SeoReport {
     url: string;
     timestamp: Date;
     grade: string;
     score: number;
+    timing?: SeoTiming;
     checks: SeoCheckResult[];
     title?: {
         text: string;
@@ -100,6 +109,21 @@ export interface SeoReport {
         text: string;
         length: number;
     };
+    openGraph?: {
+        title?: string;
+        description?: string;
+        image?: string;
+        url?: string;
+        type?: string;
+        siteName?: string;
+    };
+    twitterCard?: {
+        card?: string;
+        title?: string;
+        description?: string;
+        image?: string;
+        site?: string;
+    };
     headings: HeadingAnalysis;
     content: ContentMetrics;
     links: LinkAnalysis;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "recker",
-  "version": "1.0.28",
+  "version": "1.0.29",
   "description": "AI & DevX focused HTTP client for Node.js 18+",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",