recker 1.0.32 → 1.0.33-next.bbc56eb

package/dist/seo/validators/sitemap.js

@@ -0,0 +1,424 @@
+import * as cheerio from 'cheerio';
+const VALID_CHANGEFREQ = ['always', 'hourly', 'daily', 'weekly', 'monthly', 'yearly', 'never'];
+const MAX_URLS_PER_SITEMAP = 50000;
+const MAX_SITEMAP_SIZE = 50 * 1024 * 1024;
+export function parseSitemap(content, compressed = false) {
+    const errors = [];
+    const warnings = [];
+    const urls = [];
+    const sitemaps = [];
+    let type = 'unknown';
+    try {
+        const $ = cheerio.load(content, { xmlMode: true });
+        if ($('urlset').length > 0) {
+            type = 'urlset';
+            $('url').each((_, elem) => {
+                const $url = $(elem);
+                const loc = $url.find('loc').first().text().trim();
+                if (!loc) {
+                    errors.push('URL entry missing <loc> element');
+                    return;
+                }
+                try {
+                    new URL(loc);
+                }
+                catch {
+                    errors.push(`Invalid URL: ${loc}`);
+                    return;
+                }
+                const url = { loc };
+                const lastmod = $url.find('lastmod').first().text().trim();
+                if (lastmod) {
+                    if (isValidDate(lastmod)) {
+                        url.lastmod = lastmod;
+                    }
+                    else {
+                        warnings.push(`Invalid lastmod date for ${loc}: ${lastmod}`);
+                    }
+                }
+                const changefreq = $url.find('changefreq').first().text().trim().toLowerCase();
+                if (changefreq) {
+                    if (VALID_CHANGEFREQ.includes(changefreq)) {
+                        url.changefreq = changefreq;
+                    }
+                    else {
+                        warnings.push(`Invalid changefreq for ${loc}: ${changefreq}`);
+                    }
+                }
+                const priority = $url.find('priority').first().text().trim();
+                if (priority) {
+                    const p = parseFloat(priority);
+                    if (isNaN(p) || p < 0 || p > 1) {
+                        warnings.push(`Invalid priority for ${loc}: ${priority} (must be 0.0-1.0)`);
+                    }
+                    else {
+                        url.priority = p;
+                    }
+                }
+                const images = [];
+                $url.find('image\\:image, image').each((_, imgElem) => {
+                    const $img = $(imgElem);
+                    const imgLoc = $img.find('image\\:loc, loc').first().text().trim();
+                    if (imgLoc) {
+                        images.push({
+                            loc: imgLoc,
+                            caption: $img.find('image\\:caption, caption').first().text().trim() || undefined,
+                            title: $img.find('image\\:title, title').first().text().trim() || undefined,
+                        });
+                    }
+                });
+                if (images.length > 0) {
+                    url.images = images;
+                }
+                const alternates = [];
+                $url.find('xhtml\\:link[rel="alternate"], link[rel="alternate"]').each((_, linkElem) => {
+                    const $link = $(linkElem);
+                    const hreflang = $link.attr('hreflang');
+                    const href = $link.attr('href');
+                    if (hreflang && href) {
+                        alternates.push({ hreflang, href });
+                    }
+                });
+                if (alternates.length > 0) {
+                    url.alternates = alternates;
+                }
+                urls.push(url);
+            });
+        }
+        else if ($('sitemapindex').length > 0) {
+            type = 'sitemapindex';
+            $('sitemap').each((_, elem) => {
+                const $sitemap = $(elem);
+                const loc = $sitemap.find('loc').first().text().trim();
+                if (!loc) {
+                    errors.push('Sitemap entry missing <loc> element');
+                    return;
+                }
+                const sitemap = { loc };
+                const lastmod = $sitemap.find('lastmod').first().text().trim();
+                if (lastmod) {
+                    if (isValidDate(lastmod)) {
+                        sitemap.lastmod = lastmod;
+                    }
+                    else {
+                        warnings.push(`Invalid lastmod date for sitemap ${loc}: ${lastmod}`);
+                    }
+                }
+                sitemaps.push(sitemap);
+            });
+        }
+        else {
+            errors.push('Invalid sitemap: must contain <urlset> or <sitemapindex>');
+        }
+    }
+    catch (e) {
+        errors.push(`XML parsing error: ${e instanceof Error ? e.message : 'Unknown error'}`);
+    }
+    return {
+        type,
+        valid: errors.length === 0,
+        errors,
+        warnings,
+        urls,
+        sitemaps,
+        urlCount: type === 'urlset' ? urls.length : sitemaps.reduce((sum, s) => sum + 1, 0),
+        size: content.length,
+        compressed,
+    };
+}
+export function validateSitemap(content, baseUrl) {
+    const parseResult = parseSitemap(content);
+    const issues = [];
+    for (const error of parseResult.errors) {
+        issues.push({
+            type: 'error',
+            code: 'PARSE_ERROR',
+            message: error,
+        });
+    }
+    for (const warning of parseResult.warnings) {
+        issues.push({
+            type: 'warning',
+            code: 'PARSE_WARNING',
+            message: warning,
+        });
+    }
+    if (parseResult.urls.length > MAX_URLS_PER_SITEMAP) {
+        issues.push({
+            type: 'error',
+            code: 'TOO_MANY_URLS',
+            message: `Sitemap has ${parseResult.urls.length} URLs (max: ${MAX_URLS_PER_SITEMAP})`,
+            recommendation: 'Split into multiple sitemaps and use a sitemap index',
+        });
+    }
+    if (parseResult.size > MAX_SITEMAP_SIZE) {
+        issues.push({
+            type: 'error',
+            code: 'FILE_TOO_LARGE',
+            message: `Sitemap is ${Math.round(parseResult.size / (1024 * 1024))}MB (max: 50MB)`,
+            recommendation: 'Split into multiple sitemaps or compress with gzip',
+        });
+    }
+    if (parseResult.type === 'urlset' && parseResult.urls.length === 0) {
+        issues.push({
+            type: 'warning',
+            code: 'EMPTY_SITEMAP',
+            message: 'Sitemap contains no URLs',
+            recommendation: 'Add URLs to your sitemap for search engines to discover',
+        });
+    }
+    if (parseResult.type === 'sitemapindex' && parseResult.sitemaps.length === 0) {
+        issues.push({
+            type: 'warning',
+            code: 'EMPTY_INDEX',
+            message: 'Sitemap index contains no sitemaps',
+            recommendation: 'Add sitemap references to your sitemap index',
+        });
+    }
+    const seenUrls = new Set();
+    const duplicates = [];
+    for (const url of parseResult.urls) {
+        const normalized = normalizeUrl(url.loc);
+        if (seenUrls.has(normalized)) {
+            duplicates.push(url.loc);
+        }
+        else {
+            seenUrls.add(normalized);
+        }
+    }
+    if (duplicates.length > 0) {
+        issues.push({
+            type: 'warning',
+            code: 'DUPLICATE_URLS',
+            message: `${duplicates.length} duplicate URL(s) found`,
+            recommendation: 'Remove duplicate URLs from sitemap',
+        });
+    }
+    if (baseUrl) {
+        const baseHost = new URL(baseUrl).hostname;
+        for (const url of parseResult.urls) {
+            try {
+                const urlHost = new URL(url.loc).hostname;
+                if (urlHost !== baseHost && !urlHost.endsWith(`.${baseHost}`)) {
+                    issues.push({
+                        type: 'warning',
+                        code: 'CROSS_DOMAIN_URL',
+                        message: `URL belongs to different domain: ${url.loc}`,
+                        url: url.loc,
+                        recommendation: 'Sitemaps should only contain URLs from the same domain',
+                    });
+                }
+            }
+            catch {
+            }
+        }
+    }
+    const oneYearAgo = new Date();
+    oneYearAgo.setFullYear(oneYearAgo.getFullYear() - 1);
+    let oldDateCount = 0;
+    for (const url of parseResult.urls) {
+        if (url.lastmod) {
+            const date = new Date(url.lastmod);
+            if (!isNaN(date.getTime()) && date < oneYearAgo) {
+                oldDateCount++;
+            }
+        }
+    }
+    if (oldDateCount > parseResult.urls.length * 0.5) {
+        issues.push({
+            type: 'info',
+            code: 'OLD_LASTMOD',
+            message: `${oldDateCount} URLs have lastmod dates over 1 year old`,
+            recommendation: 'Consider updating lastmod dates when content changes',
+        });
+    }
+    const missingLastmod = parseResult.urls.filter(u => !u.lastmod).length;
+    if (missingLastmod > 0 && missingLastmod === parseResult.urls.length) {
+        issues.push({
+            type: 'info',
+            code: 'NO_LASTMOD',
+            message: 'No URLs have lastmod dates',
+            recommendation: 'Add lastmod dates to help search engines prioritize crawling',
+        });
+    }
+    const priorities = parseResult.urls.filter(u => u.priority !== undefined).map(u => u.priority);
+    if (priorities.length > 0) {
+        const allSamePriority = priorities.every(p => p === priorities[0]);
+        if (allSamePriority && parseResult.urls.length > 10) {
+            issues.push({
+                type: 'info',
+                code: 'UNIFORM_PRIORITY',
+                message: `All ${priorities.length} URLs have the same priority (${priorities[0]})`,
+                recommendation: 'Vary priority values to indicate relative page importance',
+            });
+        }
+    }
+    return {
+        valid: issues.filter(i => i.type === 'error').length === 0,
+        issues,
+        parseResult,
+    };
+}
+export async function discoverSitemaps(baseUrl, robotsTxtContent, fetcher) {
+    const discovered = new Set();
+    const base = new URL(baseUrl);
+    const commonLocations = [
+        '/sitemap.xml',
+        '/sitemap_index.xml',
+        '/sitemap-index.xml',
+        '/sitemaps.xml',
+        '/sitemap1.xml',
+        '/post-sitemap.xml',
+        '/page-sitemap.xml',
+    ];
+    if (robotsTxtContent) {
+        const lines = robotsTxtContent.split(/\r?\n/);
+        for (const line of lines) {
+            const match = line.match(/^sitemap:\s*(.+)$/i);
+            if (match) {
+                try {
+                    const url = new URL(match[1].trim(), baseUrl).href;
+                    discovered.add(url);
+                }
+                catch {
+                }
+            }
+        }
+    }
+    for (const path of commonLocations) {
+        const url = new URL(path, base).href;
+        try {
+            let response;
+            if (fetcher) {
+                response = await fetcher(url);
+            }
+            else {
+                const fetchResponse = await fetch(url, { method: 'HEAD' });
+                response = { status: fetchResponse.status, text: '' };
+            }
+            if (response.status === 200) {
+                discovered.add(url);
+            }
+        }
+        catch {
+        }
+    }
+    return Array.from(discovered);
+}
+export async function fetchAndValidateSitemap(url, fetcher) {
+    try {
+        let response;
+        if (fetcher) {
+            response = await fetcher(url);
+        }
+        else {
+            const fetchResponse = await fetch(url);
+            response = {
+                status: fetchResponse.status,
+                text: await fetchResponse.text(),
+                headers: Object.fromEntries(fetchResponse.headers.entries()),
+            };
+        }
+        if (response.status === 404) {
+            return {
+                exists: false,
+                status: 404,
+                valid: false,
+                issues: [{
+                        type: 'warning',
+                        code: 'NOT_FOUND',
+                        message: 'Sitemap not found (404)',
+                        recommendation: 'Create a sitemap.xml to help search engines discover your content',
+                    }],
+                parseResult: {
+                    type: 'unknown',
+                    valid: false,
+                    errors: [],
+                    warnings: [],
+                    urls: [],
+                    sitemaps: [],
+                    urlCount: 0,
+                    size: 0,
+                    compressed: false,
+                },
+            };
+        }
+        if (response.status >= 400) {
+            return {
+                exists: false,
+                status: response.status,
+                valid: false,
+                issues: [{
+                        type: 'error',
+                        code: 'FETCH_ERROR',
+                        message: `Failed to fetch sitemap (HTTP ${response.status})`,
+                    }],
+                parseResult: {
+                    type: 'unknown',
+                    valid: false,
+                    errors: [],
+                    warnings: [],
+                    urls: [],
+                    sitemaps: [],
+                    urlCount: 0,
+                    size: 0,
+                    compressed: false,
+                },
+            };
+        }
+        const baseUrl = new URL(url).origin;
+        const validation = validateSitemap(response.text, baseUrl);
+        return {
+            ...validation,
+            exists: true,
+            status: response.status,
+        };
+    }
+    catch (error) {
+        return {
+            exists: false,
+            valid: false,
+            issues: [{
+                    type: 'error',
+                    code: 'FETCH_ERROR',
+                    message: `Failed to fetch sitemap: ${error instanceof Error ? error.message : 'Unknown error'}`,
+                }],
+            parseResult: {
+                type: 'unknown',
+                valid: false,
+                errors: [],
+                warnings: [],
+                urls: [],
+                sitemaps: [],
+                urlCount: 0,
+                size: 0,
+                compressed: false,
+            },
+        };
+    }
+}
+function isValidDate(dateString) {
+    const patterns = [
+        /^\d{4}-\d{2}-\d{2}$/,
+        /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}([+-]\d{2}:\d{2}|Z)$/,
+        /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}([+-]\d{2}:\d{2}|Z)$/,
+    ];
+    if (!patterns.some(p => p.test(dateString))) {
+        return false;
+    }
+    const date = new Date(dateString);
+    return !isNaN(date.getTime());
+}
+function normalizeUrl(url) {
+    try {
+        const parsed = new URL(url);
+        let path = parsed.pathname;
+        if (path !== '/' && path.endsWith('/')) {
+            path = path.slice(0, -1);
+        }
+        return `${parsed.protocol}//${parsed.host}${path}${parsed.search}`.toLowerCase();
+    }
+    catch {
+        return url.toLowerCase();
+    }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "recker",
-  "version": "1.0.32",
+  "version": "1.0.33-next.bbc56eb",
   "description": "AI & DevX focused HTTP client for Node.js 18+",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",