recker 1.0.31 → 1.0.32-next.e0741bf

package/dist/seo/validators/robots.d.ts

@@ -0,0 +1,54 @@
+export interface RobotsDirective {
+    type: 'user-agent' | 'allow' | 'disallow' | 'sitemap' | 'crawl-delay' | 'host' | 'clean-param';
+    value: string;
+    line: number;
+}
+export interface RobotsUserAgentBlock {
+    userAgents: string[];
+    rules: Array<{
+        type: 'allow' | 'disallow';
+        path: string;
+        line: number;
+    }>;
+    crawlDelay?: number;
+}
+export interface RobotsParseResult {
+    valid: boolean;
+    errors: Array<{
+        line: number;
+        message: string;
+    }>;
+    warnings: Array<{
+        line: number;
+        message: string;
+    }>;
+    directives: RobotsDirective[];
+    userAgentBlocks: RobotsUserAgentBlock[];
+    sitemaps: string[];
+    host?: string;
+    blocksAllRobots: boolean;
+    blocksImportantPaths: boolean;
+    size: number;
+}
+export interface RobotsValidationIssue {
+    type: 'error' | 'warning' | 'info';
+    code: string;
+    message: string;
+    line?: number;
+    recommendation?: string;
+}
+export interface RobotsValidationResult {
+    valid: boolean;
+    issues: RobotsValidationIssue[];
+    parseResult: RobotsParseResult;
+}
+export declare function parseRobotsTxt(content: string): RobotsParseResult;
+export declare function validateRobotsTxt(content: string, baseUrl?: string): RobotsValidationResult;
+export declare function isPathAllowed(parseResult: RobotsParseResult, path: string, userAgent?: string): boolean;
+export declare function fetchAndValidateRobotsTxt(url: string, fetcher?: (url: string) => Promise<{
+    status: number;
+    text: string;
+}>): Promise<RobotsValidationResult & {
+    exists: boolean;
+    status?: number;
+}>;

package/dist/seo/validators/robots.js

@@ -0,0 +1,382 @@
+const IMPORTANT_PATHS = [
+    '/',
+    '/sitemap.xml',
+    '/sitemap',
+    '/.well-known/',
+    '/robots.txt',
+];
+const SEO_CRITICAL_PATHS = [
+    '/css',
+    '/js',
+    '/images',
+    '/assets',
+    '/static',
+    '/fonts',
+];
+export function parseRobotsTxt(content) {
+    const lines = content.split(/\r?\n/);
+    const directives = [];
+    const errors = [];
+    const warnings = [];
+    const sitemaps = [];
+    const userAgentBlocks = [];
+    let currentBlock = null;
+    let host;
+    let blocksAllRobots = false;
+    let blocksImportantPaths = false;
+    for (let i = 0; i < lines.length; i++) {
+        const lineNum = i + 1;
+        let line = lines[i].trim();
+        const commentIndex = line.indexOf('#');
+        if (commentIndex !== -1) {
+            line = line.substring(0, commentIndex).trim();
+        }
+        if (!line)
+            continue;
+        const colonIndex = line.indexOf(':');
+        if (colonIndex === -1) {
+            errors.push({ line: lineNum, message: `Invalid syntax: missing colon` });
+            continue;
+        }
+        const directiveType = line.substring(0, colonIndex).trim().toLowerCase();
+        const value = line.substring(colonIndex + 1).trim();
+        switch (directiveType) {
+            case 'user-agent':
+                if (!value) {
+                    errors.push({ line: lineNum, message: 'Empty user-agent value' });
+                    continue;
+                }
+                if (!currentBlock || currentBlock.rules.length > 0) {
+                    currentBlock = { userAgents: [value], rules: [] };
+                    userAgentBlocks.push(currentBlock);
+                }
+                else {
+                    currentBlock.userAgents.push(value);
+                }
+                directives.push({ type: 'user-agent', value, line: lineNum });
+                break;
+            case 'disallow':
+                if (!currentBlock) {
+                    warnings.push({ line: lineNum, message: 'Disallow without preceding User-agent' });
+                    currentBlock = { userAgents: ['*'], rules: [] };
+                    userAgentBlocks.push(currentBlock);
+                }
+                currentBlock.rules.push({ type: 'disallow', path: value, line: lineNum });
+                directives.push({ type: 'disallow', value, line: lineNum });
+                if (value === '/' && currentBlock.userAgents.includes('*')) {
+                    blocksAllRobots = true;
+                }
+                for (const importantPath of IMPORTANT_PATHS) {
+                    if (value === importantPath || (value.endsWith('/') && importantPath.startsWith(value))) {
+                        blocksImportantPaths = true;
+                    }
+                }
+                break;
+            case 'allow':
+                if (!currentBlock) {
+                    warnings.push({ line: lineNum, message: 'Allow without preceding User-agent' });
+                    currentBlock = { userAgents: ['*'], rules: [] };
+                    userAgentBlocks.push(currentBlock);
+                }
+                currentBlock.rules.push({ type: 'allow', path: value, line: lineNum });
+                directives.push({ type: 'allow', value, line: lineNum });
+                break;
+            case 'sitemap':
+                if (!value) {
+                    errors.push({ line: lineNum, message: 'Empty sitemap URL' });
+                    continue;
+                }
+                try {
+                    new URL(value);
+                    sitemaps.push(value);
+                    directives.push({ type: 'sitemap', value, line: lineNum });
+                }
+                catch {
+                    errors.push({ line: lineNum, message: `Invalid sitemap URL: ${value}` });
+                }
+                break;
+            case 'crawl-delay':
+                if (!currentBlock) {
+                    warnings.push({ line: lineNum, message: 'Crawl-delay without preceding User-agent' });
+                }
+                else {
+                    const delay = parseFloat(value);
+                    if (isNaN(delay) || delay < 0) {
+                        errors.push({ line: lineNum, message: `Invalid crawl-delay value: ${value}` });
+                    }
+                    else {
+                        currentBlock.crawlDelay = delay;
+                        if (delay > 10) {
+                            warnings.push({ line: lineNum, message: `High crawl-delay (${delay}s) may slow indexing` });
+                        }
+                    }
+                }
+                directives.push({ type: 'crawl-delay', value, line: lineNum });
+                break;
+            case 'host':
+                host = value;
+                directives.push({ type: 'host', value, line: lineNum });
+                break;
+            case 'clean-param':
+                directives.push({ type: 'clean-param', value, line: lineNum });
+                break;
+            default:
+                warnings.push({ line: lineNum, message: `Unknown directive: ${directiveType}` });
+        }
+    }
+    return {
+        valid: errors.length === 0,
+        errors,
+        warnings,
+        directives,
+        userAgentBlocks,
+        sitemaps,
+        host,
+        blocksAllRobots,
+        blocksImportantPaths,
+        size: content.length,
+    };
+}
+export function validateRobotsTxt(content, baseUrl) {
+    const parseResult = parseRobotsTxt(content);
+    const issues = [];
+    for (const error of parseResult.errors) {
+        issues.push({
+            type: 'error',
+            code: 'PARSE_ERROR',
+            message: error.message,
+            line: error.line,
+        });
+    }
+    for (const warning of parseResult.warnings) {
+        issues.push({
+            type: 'warning',
+            code: 'PARSE_WARNING',
+            message: warning.message,
+            line: warning.line,
+        });
+    }
+    if (content.trim().length === 0) {
+        issues.push({
+            type: 'warning',
+            code: 'EMPTY_FILE',
+            message: 'robots.txt is empty',
+            recommendation: 'Add at least User-agent: * and basic Allow/Disallow rules',
+        });
+    }
+    if (parseResult.blocksAllRobots) {
+        issues.push({
+            type: 'error',
+            code: 'BLOCKS_ALL_ROBOTS',
+            message: 'robots.txt blocks all search engines (Disallow: /)',
+            recommendation: 'Remove "Disallow: /" or add specific Allow rules for indexable content',
+        });
+    }
+    if (parseResult.sitemaps.length === 0) {
+        issues.push({
+            type: 'warning',
+            code: 'NO_SITEMAP',
+            message: 'No Sitemap directive found',
+            recommendation: 'Add Sitemap: https://example.com/sitemap.xml to help search engines discover your content',
+        });
+    }
+    if (baseUrl) {
+        const baseHost = new URL(baseUrl).hostname;
+        for (const sitemap of parseResult.sitemaps) {
+            try {
+                const sitemapHost = new URL(sitemap).hostname;
+                if (sitemapHost !== baseHost) {
+                    issues.push({
+                        type: 'warning',
+                        code: 'SITEMAP_CROSS_DOMAIN',
+                        message: `Sitemap points to different domain: ${sitemap}`,
+                        recommendation: 'Ensure sitemap URLs are on the same domain',
+                    });
+                }
+            }
+            catch {
+            }
+        }
+    }
+    for (const block of parseResult.userAgentBlocks) {
+        if (block.userAgents.includes('*') || block.userAgents.includes('Googlebot')) {
+            for (const rule of block.rules) {
+                if (rule.type === 'disallow') {
+                    for (const criticalPath of SEO_CRITICAL_PATHS) {
+                        if (rule.path.includes(criticalPath)) {
+                            issues.push({
+                                type: 'warning',
+                                code: 'BLOCKS_RESOURCES',
+                                message: `Blocking ${rule.path} may prevent proper rendering`,
+                                line: rule.line,
+                                recommendation: 'Allow CSS, JS, and image files for proper page rendering',
+                            });
+                        }
+                    }
+                }
+            }
+        }
+    }
+    if (parseResult.size > 500 * 1024) {
+        issues.push({
+            type: 'warning',
+            code: 'FILE_TOO_LARGE',
+            message: `robots.txt is ${Math.round(parseResult.size / 1024)}KB (recommended max: 500KB)`,
+            recommendation: 'Simplify robots.txt to reduce file size',
+        });
+    }
+    if (parseResult.userAgentBlocks.length === 0 && content.trim().length > 0) {
+        issues.push({
+            type: 'warning',
+            code: 'NO_USER_AGENT',
+            message: 'No User-agent directive found',
+            recommendation: 'Add at least User-agent: * for universal rules',
+        });
+    }
+    for (const block of parseResult.userAgentBlocks) {
+        if (block.crawlDelay && block.crawlDelay > 30) {
+            issues.push({
+                type: 'error',
+                code: 'EXCESSIVE_CRAWL_DELAY',
+                message: `Crawl-delay of ${block.crawlDelay}s is excessive for ${block.userAgents.join(', ')}`,
+                recommendation: 'Crawl-delay over 30 seconds may significantly slow indexing',
+            });
+        }
+    }
+    return {
+        valid: issues.filter(i => i.type === 'error').length === 0,
+        issues,
+        parseResult,
+    };
+}
+export function isPathAllowed(parseResult, path, userAgent = '*') {
+    let matchingBlock;
+    for (const block of parseResult.userAgentBlocks) {
+        if (block.userAgents.some(ua => ua.toLowerCase() === userAgent.toLowerCase() ||
+            userAgent.toLowerCase().includes(ua.toLowerCase()))) {
+            matchingBlock = block;
+            break;
+        }
+    }
+    if (!matchingBlock) {
+        matchingBlock = parseResult.userAgentBlocks.find(b => b.userAgents.includes('*'));
+    }
+    if (!matchingBlock) {
+        return true;
+    }
+    let isAllowed = true;
+    let longestMatch = -1;
+    for (const rule of matchingBlock.rules) {
+        const pattern = rule.path;
+        let regex;
+        try {
+            const escapedPattern = pattern
+                .replace(/[.+?^${}()|[\]\\]/g, '\\$&')
+                .replace(/\*/g, '.*')
+                .replace(/\\\$/g, '$');
+            regex = new RegExp(`^${escapedPattern}`);
+        }
+        catch {
+            continue;
+        }
+        if (regex.test(path)) {
+            const matchLength = pattern.replace(/\*/g, '').length;
+            if (matchLength > longestMatch) {
+                longestMatch = matchLength;
+                isAllowed = rule.type === 'allow';
+            }
+        }
+    }
+    return isAllowed;
+}
+export async function fetchAndValidateRobotsTxt(url, fetcher) {
+    const robotsUrl = new URL('/robots.txt', url).href;
+    try {
+        let response;
+        if (fetcher) {
+            response = await fetcher(robotsUrl);
+        }
+        else {
+            const fetchResponse = await fetch(robotsUrl);
+            response = {
+                status: fetchResponse.status,
+                text: await fetchResponse.text(),
+            };
+        }
+        if (response.status === 404) {
+            return {
+                exists: false,
+                status: 404,
+                valid: false,
+                issues: [{
+                        type: 'warning',
+                        code: 'NOT_FOUND',
+                        message: 'robots.txt not found (404)',
+                        recommendation: 'Create a robots.txt file to control search engine crawling',
+                    }],
+                parseResult: {
+                    valid: false,
+                    errors: [],
+                    warnings: [],
+                    directives: [],
+                    userAgentBlocks: [],
+                    sitemaps: [],
+                    blocksAllRobots: false,
+                    blocksImportantPaths: false,
+                    size: 0,
+                },
+            };
+        }
+        if (response.status >= 400) {
+            return {
+                exists: false,
+                status: response.status,
+                valid: false,
+                issues: [{
+                        type: 'error',
+                        code: 'FETCH_ERROR',
+                        message: `Failed to fetch robots.txt (HTTP ${response.status})`,
+                    }],
+                parseResult: {
+                    valid: false,
+                    errors: [],
+                    warnings: [],
+                    directives: [],
+                    userAgentBlocks: [],
+                    sitemaps: [],
+                    blocksAllRobots: false,
+                    blocksImportantPaths: false,
+                    size: 0,
+                },
+            };
+        }
+        const validation = validateRobotsTxt(response.text, url);
+        return {
+            ...validation,
+            exists: true,
+            status: response.status,
+        };
+    }
+    catch (error) {
+        return {
+            exists: false,
+            valid: false,
+            issues: [{
+                    type: 'error',
+                    code: 'FETCH_ERROR',
+                    message: `Failed to fetch robots.txt: ${error instanceof Error ? error.message : 'Unknown error'}`,
+                }],
+            parseResult: {
+                valid: false,
+                errors: [],
+                warnings: [],
+                directives: [],
+                userAgentBlocks: [],
+                sitemaps: [],
+                blocksAllRobots: false,
+                blocksImportantPaths: false,
+                size: 0,
+            },
+        };
+    }
+}

package/dist/seo/validators/sitemap.d.ts

@@ -0,0 +1,69 @@
+export interface SitemapUrl {
+    loc: string;
+    lastmod?: string;
+    changefreq?: 'always' | 'hourly' | 'daily' | 'weekly' | 'monthly' | 'yearly' | 'never';
+    priority?: number;
+    images?: Array<{
+        loc: string;
+        caption?: string;
+        title?: string;
+    }>;
+    videos?: Array<{
+        thumbnailLoc: string;
+        title: string;
+        description: string;
+        contentLoc?: string;
+        playerLoc?: string;
+    }>;
+    news?: {
+        publicationName: string;
+        publicationLanguage: string;
+        publicationDate: string;
+        title: string;
+    };
+    alternates?: Array<{
+        hreflang: string;
+        href: string;
+    }>;
+}
+export interface SitemapIndex {
+    loc: string;
+    lastmod?: string;
+}
+export interface SitemapParseResult {
+    type: 'urlset' | 'sitemapindex' | 'unknown';
+    valid: boolean;
+    errors: string[];
+    warnings: string[];
+    urls: SitemapUrl[];
+    sitemaps: SitemapIndex[];
+    urlCount: number;
+    size: number;
+    compressed: boolean;
+}
+export interface SitemapValidationIssue {
+    type: 'error' | 'warning' | 'info';
+    code: string;
+    message: string;
+    url?: string;
+    recommendation?: string;
+}
+export interface SitemapValidationResult {
+    valid: boolean;
+    issues: SitemapValidationIssue[];
+    parseResult: SitemapParseResult;
+}
+export declare function parseSitemap(content: string, compressed?: boolean): SitemapParseResult;
+export declare function validateSitemap(content: string, baseUrl?: string): SitemapValidationResult;
+export declare function discoverSitemaps(baseUrl: string, robotsTxtContent?: string, fetcher?: (url: string) => Promise<{
+    status: number;
+    text: string;
+}>): Promise<string[]>;
+export declare function fetchAndValidateSitemap(url: string, fetcher?: (url: string) => Promise<{
+    status: number;
+    text: string;
+    headers?: Record<string, string>;
+}>): Promise<SitemapValidationResult & {
+    exists: boolean;
+    status?: number;
+}>;