s3db.js 18.0.11-next.1534f717 → 18.0.11-next.e8e71b5b

package/dist/plugins/spider/recker-llms-validator.js

@@ -0,0 +1,334 @@
+import { createHttpClient } from '#src/concerns/http-client.js';
+export class ReckerLlmsTxtValidator {
+    config;
+    _context;
+    cache;
+    _httpClient;
+    reckerAvailable = null;
+    parseLlmsTxt = null;
+    validateLlmsTxt = null;
+    fetchAndValidateLlmsTxt = null;
+    generateLlmsTxtTemplate = null;
+    constructor(config = {}) {
+        this.config = {
+            userAgent: config.userAgent || 's3db-spider',
+            fetchTimeout: config.fetchTimeout || 10000,
+            cacheTimeout: config.cacheTimeout || 3600000,
+            context: config.context || null
+        };
+        this._context = this.config.context;
+        this.cache = new Map();
+        this._httpClient = null;
+    }
+    async _checkReckerAvailability() {
+        if (this.reckerAvailable !== null) {
+            return this.reckerAvailable;
+        }
+        try {
+            const llmsModule = await import('recker/seo/validators/llms-txt');
+            this.parseLlmsTxt = llmsModule.parseLlmsTxt;
+            this.validateLlmsTxt = llmsModule.validateLlmsTxt;
+            this.fetchAndValidateLlmsTxt = llmsModule.fetchAndValidateLlmsTxt;
+            this.generateLlmsTxtTemplate = llmsModule.generateLlmsTxtTemplate;
+            this.reckerAvailable = true;
+            return true;
+        }
+        catch {
+            this.reckerAvailable = false;
+            return false;
+        }
+    }
+    async _getHttpClient() {
+        if (!this._httpClient) {
+            const baseConfig = this._context
+                ? this._context.getHttpClientConfig('https://example.com')
+                : {
+                    headers: {
+                        'User-Agent': this.config.userAgent
+                    }
+                };
+            this._httpClient = await createHttpClient({
+                ...baseConfig,
+                timeout: this.config.fetchTimeout,
+                retry: {
+                    maxAttempts: 2,
+                    delay: 500,
+                    backoff: 'exponential',
+                    retryAfter: true,
+                    retryOn: [429, 500, 502, 503, 504]
+                }
+            });
+        }
+        return this._httpClient;
+    }
+    async check(domain) {
+        const isReckerAvailable = await this._checkReckerAvailability();
+        if (!isReckerAvailable) {
+            return this._fallbackCheck(domain);
+        }
+        const normalizedDomain = domain.replace(/\/$/, '');
+        const llmsUrl = `${normalizedDomain}/llms.txt`;
+        const cached = this.cache.get(normalizedDomain);
+        if (cached && Date.now() - cached.timestamp < this.config.cacheTimeout) {
+            return cached.result;
+        }
+        try {
+            const fetchResult = await this.fetchAndValidateLlmsTxt(llmsUrl, async (url) => {
+                const client = await this._getHttpClient();
+                const response = await client.get(url);
+                if (this._context) {
+                    this._context.processResponse(response, url);
+                }
+                return {
+                    status: response.ok ? 200 : response.status,
+                    text: response.ok ? await response.text() : ''
+                };
+            });
+            const result = {
+                exists: fetchResult.exists,
+                valid: fetchResult.valid,
+                status: fetchResult.status,
+                fullVersionExists: fetchResult.fullVersionExists,
+                siteName: fetchResult.parseResult.siteName,
+                siteDescription: fetchResult.parseResult.siteDescription,
+                sections: fetchResult.parseResult.sections,
+                links: fetchResult.parseResult.links,
+                issues: fetchResult.issues,
+                errors: fetchResult.parseResult.errors,
+                warnings: fetchResult.parseResult.warnings,
+                size: fetchResult.parseResult.size
+            };
+            this.cache.set(normalizedDomain, { result, timestamp: Date.now() });
+            return result;
+        }
+        catch (error) {
+            return {
+                exists: false,
+                valid: false,
+                sections: [],
+                links: [],
+                issues: [{
+                        type: 'error',
+                        code: 'FETCH_ERROR',
+                        message: `Failed to fetch llms.txt: ${error.message}`
+                    }],
+                errors: [error.message],
+                warnings: []
+            };
+        }
+    }
+    async _fallbackCheck(domain) {
+        const normalizedDomain = domain.replace(/\/$/, '');
+        const llmsUrl = `${normalizedDomain}/llms.txt`;
+        try {
+            const client = await this._getHttpClient();
+            const response = await client.get(llmsUrl);
+            if (!response.ok) {
+                return {
+                    exists: false,
+                    valid: false,
+                    status: response.status,
+                    sections: [],
+                    links: [],
+                    issues: [{
+                            type: 'info',
+                            code: 'NOT_FOUND',
+                            message: 'llms.txt file not found',
+                            recommendation: 'Consider adding an llms.txt file for AI SEO'
+                        }],
+                    errors: [],
+                    warnings: []
+                };
+            }
+            const content = await response.text();
+            const parsed = this._simpleParse(content);
+            return {
+                exists: true,
+                valid: parsed.sections.length > 0 || !!parsed.siteName,
+                status: 200,
+                siteName: parsed.siteName,
+                siteDescription: parsed.siteDescription,
+                sections: parsed.sections,
+                links: parsed.links,
+                issues: [],
+                errors: [],
+                warnings: ['Recker not available - using basic parsing'],
+                size: content.length
+            };
+        }
+        catch (error) {
+            return {
+                exists: false,
+                valid: false,
+                sections: [],
+                links: [],
+                issues: [],
+                errors: [error.message],
+                warnings: []
+            };
+        }
+    }
+    _simpleParse(content) {
+        const lines = content.split(/\r?\n/);
+        const sections = [];
+        const links = [];
+        let siteName;
+        let siteDescription;
+        let currentSection = null;
+        for (const line of lines) {
+            const trimmed = line.trim();
+            if (!trimmed || trimmed.startsWith('#'))
+                continue;
+            if (trimmed.startsWith('# ') && !siteName) {
+                siteName = trimmed.slice(2).trim();
+                continue;
+            }
+            if (trimmed.startsWith('>') && !siteDescription) {
+                siteDescription = trimmed.slice(1).trim();
+                continue;
+            }
+            if (trimmed.startsWith('## ')) {
+                if (currentSection) {
+                    sections.push(currentSection);
+                }
+                currentSection = {
+                    title: trimmed.slice(3).trim(),
+                    content: '',
+                    links: []
+                };
+                continue;
+            }
+            const linkMatch = trimmed.match(/^\[([^\]]+)\]\(([^)]+)\)(?:\s*-\s*(.+))?$/);
+            if (linkMatch) {
+                const link = {
+                    text: linkMatch[1],
+                    url: linkMatch[2],
+                    description: linkMatch[3]?.trim(),
+                    section: currentSection?.title
+                };
+                links.push(link);
+                if (currentSection) {
+                    currentSection.links.push(link);
+                }
+                continue;
+            }
+            if (currentSection && trimmed) {
+                currentSection.content += (currentSection.content ? '\n' : '') + trimmed;
+            }
+        }
+        if (currentSection) {
+            sections.push(currentSection);
+        }
+        return { siteName, siteDescription, sections, links };
+    }
+    async validate(domain) {
+        const isReckerAvailable = await this._checkReckerAvailability();
+        if (!isReckerAvailable) {
+            return null;
+        }
+        const checkResult = await this.check(domain);
+        if (!checkResult.exists) {
+            return null;
+        }
+        const normalizedDomain = domain.replace(/\/$/, '');
+        const llmsUrl = `${normalizedDomain}/llms.txt`;
+        try {
+            const client = await this._getHttpClient();
+            const response = await client.get(llmsUrl);
+            const content = await response.text();
+            return this.validateLlmsTxt(content, normalizedDomain);
+        }
+        catch {
+            return null;
+        }
+    }
+    validateContent(content, baseUrl) {
+        if (!this.reckerAvailable || !this.validateLlmsTxt) {
+            return null;
+        }
+        return this.validateLlmsTxt(content, baseUrl);
+    }
+    parseContent(content) {
+        if (!this.reckerAvailable || !this.parseLlmsTxt) {
+            return null;
+        }
+        return this.parseLlmsTxt(content);
+    }
+    generateTemplate(options) {
+        if (!this.reckerAvailable || !this.generateLlmsTxtTemplate) {
+            return this._fallbackGenerateTemplate(options);
+        }
+        return this.generateLlmsTxtTemplate(options);
+    }
+    _fallbackGenerateTemplate(options) {
+        const lines = [];
+        lines.push(`# ${options.siteName}`);
+        lines.push('');
+        lines.push(`> ${options.siteDescription}`);
+        lines.push('');
+        if (options.sections) {
+            for (const section of options.sections) {
+                lines.push(`## ${section.title}`);
+                lines.push('');
+                for (const link of section.links) {
+                    if (link.description) {
+                        lines.push(`[${link.text}](${link.url}) - ${link.description}`);
+                    }
+                    else {
+                        lines.push(`[${link.text}](${link.url})`);
+                    }
+                }
+                lines.push('');
+            }
+        }
+        return lines.join('\n');
+    }
+    async checkFullVersion(domain) {
+        const normalizedDomain = domain.replace(/\/$/, '');
+        const llmsFullUrl = `${normalizedDomain}/llms-full.txt`;
+        try {
+            const client = await this._getHttpClient();
+            const response = await client.get(llmsFullUrl);
+            if (!response.ok) {
+                return { exists: false, status: response.status };
+            }
+            const content = await response.text();
+            return {
+                exists: true,
+                status: 200,
+                size: content.length
+            };
+        }
+        catch {
+            return { exists: false };
+        }
+    }
+    async getLinks(domain) {
+        const result = await this.check(domain);
+        return result.links;
+    }
+    async getSections(domain) {
+        const result = await this.check(domain);
+        return result.sections;
+    }
+    clearCache(domain) {
+        if (domain) {
+            const normalizedDomain = domain.replace(/\/$/, '');
+            this.cache.delete(normalizedDomain);
+        }
+        else {
+            this.cache.clear();
+        }
+    }
+    getCacheStats() {
+        return {
+            size: this.cache.size,
+            domains: [...this.cache.keys()]
+        };
+    }
+    isReckerEnabled() {
+        return this.reckerAvailable === true;
+    }
+}
+export default ReckerLlmsTxtValidator;
+//# sourceMappingURL=recker-llms-validator.js.map

package/dist/plugins/spider/recker-llms-validator.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"recker-llms-validator.js","sourceRoot":"","sources":["../../../src/plugins/spider/recker-llms-validator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,MAAM,8BAA8B,CAAC;AAqGhE,MAAM,OAAO,sBAAsB;IACzB,MAAM,CAEZ;IACM,QAAQ,CAAsB;IAC9B,KAAK,CAA0B;IAC/B,WAAW,CAAoB;IAE/B,eAAe,GAAmB,IAAI,CAAC;IACvC,YAAY,GAA8B,IAAI,CAAC;IAC/C,eAAe,GAAiC,IAAI,CAAC;IACrD,uBAAuB,GAAyC,IAAI,CAAC;IACrE,uBAAuB,GAAyC,IAAI,CAAC;IAE7E,YAAY,SAAiC,EAAE;QAC7C,IAAI,CAAC,MAAM,GAAG;YACZ,SAAS,EAAE,MAAM,CAAC,SAAS,IAAI,aAAa;YAC5C,YAAY,EAAE,MAAM,CAAC,YAAY,IAAI,KAAK;YAC1C,YAAY,EAAE,MAAM,CAAC,YAAY,IAAI,OAAO;YAC5C,OAAO,EAAE,MAAM,CAAC,OAAO,IAAI,IAAI;SAChC,CAAC;QAEF,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC;QACpC,IAAI,CAAC,KAAK,GAAG,IAAI,GAAG,EAAE,CAAC;QACvB,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;IAC1B,CAAC;IAEO,KAAK,CAAC,wBAAwB;QACpC,IAAI,IAAI,CAAC,eAAe,KAAK,IAAI,EAAE,CAAC;YAClC,OAAO,IAAI,CAAC,eAAe,CAAC;QAC9B,CAAC;QAED,IAAI,CAAC;YACH,MAAM,UAAU,GAAG,MAAM,MAAM,CAAC,gCAAgC,CAAC,CAAC;YAClE,IAAI,CAAC,YAAY,GAAG,UAAU,CAAC,YAAY,CAAC;YAC5C,IAAI,CAAC,eAAe,GAAG,UAAU,CAAC,eAAe,CAAC;YAClD,IAAI,CAAC,uBAAuB,GAAG,UAAU,CAAC,uBAAuB,CAAC;YAClE,IAAI,CAAC,uBAAuB,GAAG,UAAU,CAAC,uBAAuB,CAAC;YAClE,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC;YAC5B,OAAO,IAAI,CAAC;QACd,CAAC;QAAC,MAAM,CAAC;YACP,IAAI,CAAC,eAAe,GAAG,KAAK,CAAC;YAC7B,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,cAAc;QAC1B,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;YACtB,MAAM,UAAU,GAAG,IAAI,CAAC,QAAQ;gBAC9B,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,mBAAmB,CAAC,qBAAqB,CAAC;gBAC1D,CAAC,CAAC;oBACE,OAAO,EAAE;wBACP,YAAY,EAAE,IAAI,CAAC,MAAM,CAAC,SAAS;qBACpC;iBACF,CAAC;YAEN,IAAI,CAAC,WAAW,GAAG,MAAM,gBAAgB,CAAC;gBACxC,GAAG,UAAU;gBACb,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,YAAY;gBACjC,KAAK,EAAE;oBACL,WAAW,EAAE,CAAC;oBACd,KAAK,EAAE,GAAG;oBACV,OAAO,EAAE,aAAa;oBACtB,UAAU,EAAE,IAAI;oBAChB,OAAO,EAAE,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC;iBACnC;aACF,CAA0B,CAAC;QAC9B,CAAC;QACD,OAAO,IAAI,CAAC,WAAW,CAAC;IAC1B,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,MAAc;QACxB,MAAM,iBAAiB,GAAG,MAAM,IAAI,CAAC,wBAAwB,EAAE,CAAC;QAEhE,IAAI,CAAC,iBAAiB,EAAE,CAAC;YACvB,OAAO,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC;QACrC,CAAC;QAED,MAAM,gBAAgB,GAAG,MAAM,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QACnD,MAAM,OAAO,GAAG,GAAG,gBAAgB,WAAW,CAAC;QAE/C,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC;QAChD,IAAI,MAAM,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,MAAM,CAAC,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,YAAY,EAAE,CAAC;YACvE,OAAO,MAAM,CAAC,MAAM,CAAC;QACvB,CAAC;QAED,IAAI,CAAC;YACH,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,uBAAwB,CAAC,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE;gBAC7E,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;gBAC3C,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;gBAEvC,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;oBAClB,IAAI,CAAC,QAAQ,CAAC,eAAe,CAC3B,QAA0E,EAC1E,GAAG,CACJ,CAAC;gBACJ,CAAC;gBAED,OAAO;oBACL,MAAM,EAAE,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM;oBAC3C,IAAI,EAAE,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE;iBAC/C,CAAC;YACJ,CAAC,CAAC,CAAC;YAEH,MAAM,MAAM,GAAuB;gBACjC,MAAM,EAAE,WAAW,CAAC,MAAM;gBAC1B,KAAK,EAAE,WAAW,CAAC,KAAK;gBACxB,MAAM,EAAE,WAAW,CAAC,MAAM;gBAC1B,iBAAiB,EAAE,WAAW,CAAC,iBAAiB;gBAChD,QAAQ,EAAE,WAAW,CAAC,WAAW,CAAC,QAAQ;gBAC1C,eAAe,EAAE,WAAW,CAAC,WAAW,CAAC,eAAe;gBACxD,QAAQ,EAAE,WAAW,CAAC,WAAW,CAAC,QAAQ;gBAC1C,KAAK,EAAE,WAAW,CAAC,WAAW,CAAC,KAAK;gBACpC,MAAM,EAAE,WAAW,CAAC,MAAM;gBAC1B,MAAM,EAAE,WAAW,CAAC,WAAW,CAAC,MAAM;gBACtC,QAAQ,EAAE,WAAW,CAAC,WAAW,CAAC,QAAQ;gBAC1C,IAAI,EAAE,WAAW,CAAC,WAAW,CAAC,IAAI;aACnC,CAAC;YAEF,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,gBAAgB,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;YAEpE,OAAO,MAAM,CAAC;QAEhB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO;gBACL,MAAM,EAAE,KAAK;gBACb,KAAK,EAAE,KAAK;gBACZ,QAAQ,EAAE,EAAE;gBACZ,KAAK,EAAE,EAAE;gBACT,MAAM,EAAE,CAAC;wBACP,IAAI,EAAE,OAAO;wBACb,IAAI,EAAE,aAAa;wBACnB,OAAO,EAAE,6BAA8B,KAAe,CAAC,OAAO,EAAE;qBACjE,CAAC;gBACF,MAAM,EAAE,CAAE,KAAe,CAAC,OAAO,CAAC;gBAClC,QAAQ,EAAE,EAAE;aACb,CAAC;QACJ,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,cAAc,CAAC,MAAc;QACzC,MAAM,gBAAgB,GAAG,MAAM,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QACnD,MAAM,OAAO,GAAG,GAAG,gBAAgB,WAAW,CAAC;QAE/C,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;YAC3C,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;YAE3C,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;gBACjB,OAAO;oBACL,MAAM,EAAE,KAAK;oBACb,KAAK,EAAE,KAAK;oBACZ,MAAM,EAAE,QAAQ,CAAC,MAAM;oBACvB,QAAQ,EAAE,EAAE;oBACZ,KAAK,EAAE,EAAE;oBACT,MAAM,EAAE,CAAC;4BACP,IAAI,EAAE,MAAM;4BACZ,IAAI,EAAE,WAAW;4BACjB,OAAO,EAAE,yBAAyB;4BAClC,cAAc,EAAE,6CAA6C;yBAC9D,CAAC;oBACF,MAAM,EAAE,EAAE;oBACV,QAAQ,EAAE,EAAE;iBACb,CAAC;YACJ,CAAC;YAED,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;YACtC,MAAM,MAAM,GAAG,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC;YAE1C,OAAO;gBACL,MAAM,EAAE,IAAI;gBACZ,KAAK,EAAE,MAAM,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,QAAQ;gBACtD,MAAM,EAAE,GAAG;gBACX,QAAQ,EAAE,MAAM,CAAC,QAAQ;gBACzB,eAAe,EAAE,MAAM,CAAC,eAAe;gBACvC,QAAQ,EAAE,MAAM,CAAC,QAAQ;gBACzB,KAAK,EAAE,MAAM,CAAC,KAAK;gBACnB,MAAM,EAAE,EAAE;gBACV,MAAM,EAAE,EAAE;gBACV,QAAQ,EAAE,CAAC,4CAA4C,CAAC;gBACxD,IAAI,EAAE,OAAO,CAAC,MAAM;aACrB,CAAC;QAEJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO;gBACL,MAAM,EAAE,KAAK;gBACb,KAAK,EAAE,KAAK;gBACZ,QAAQ,EAAE,EAAE;gBACZ,KAAK,EAAE,EAAE;gBACT,MAAM,EAAE,EAAE;gBACV,MAAM,EAAE,CAAE,KAAe,CAAC,OAAO,CAAC;gBAClC,QAAQ,EAAE,EAAE;aACb,CAAC;QACJ,CAAC;IACH,CAAC;IAEO,YAAY,CAAC,OAAe;QAMlC,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QACrC,MAAM,QAAQ,GAAqB,EAAE,CAAC;QACtC,MAAM,KAAK,GAAkB,EAAE,CAAC;QAChC,IAAI,QAA4B,CAAC;QACjC,IAAI,eAAmC,CAAC;QACxC,IAAI,cAAc,GAA0B,IAAI,CAAC;QAEjD,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;YAE5B,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC;gBAAE,SAAS;YAElD,IAAI,OAAO,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;gBAC1C,QAAQ,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;gBACnC,SAAS;YACX,CAAC;YAED,IAAI,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,eAAe,EAAE,CAAC;gBAChD,eAAe,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;gBAC1C,SAAS;YACX,CAAC;YAED,IAAI,OAAO,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;gBAC9B,IAAI,cAAc,EAAE,CAAC;oBACnB,QAAQ,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;gBAChC,CAAC;gBACD,cAAc,GAAG;oBACf,KAAK,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE;oBAC9B,OAAO,EAAE,EAAE;oBACX,KAAK,EAAE,EAAE;iBACV,CAAC;gBACF,SAAS;YACX,CAAC;YAED,MAAM,SAAS,GAAG,OAAO,CAAC,KAAK,CAAC,2CAA2C,CAAC,CAAC;YAC7E,IAAI,SAAS,EAAE,CAAC;gBACd,MAAM,IAAI,GAAgB;oBACxB,IAAI,EAAE,SAAS,CAAC,CAAC,CAAE;oBACnB,GAAG,EAAE,SAAS,CAAC,CAAC,CAAE;oBAClB,WAAW,EAAE,SAAS,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE;oBACjC,OAAO,EAAE,cAAc,EAAE,KAAK;iBAC/B,CAAC;gBACF,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBACjB,IAAI,cAAc,EAAE,CAAC;oBACnB,cAAc,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBAClC,CAAC;gBACD,SAAS;YACX,CAAC;YAED,IAAI,cAAc,IAAI,OAAO,EAAE,CAAC;gBAC9B,cAAc,CAAC,OAAO,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC;YAC3E,CAAC;QACH,CAAC;QAED,IAAI,cAAc,EAAE,CAAC;YACnB,QAAQ,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;QAChC,CAAC;QAED,OAAO,EAAE,QAAQ,EAAE,eAAe,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC;IACxD,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,MAAc;QAC3B,MAAM,iBAAiB,GAAG,MAAM,IAAI,CAAC,wBAAwB,EAAE,CAAC;QAEhE,IAAI,CAAC,iBAAiB,EAAE,CAAC;YACvB,OAAO,IAAI,CAAC;QACd,CAAC;QAED,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QAE7C,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,CAAC;YACxB,OAAO,IAAI,CAAC;QACd,CAAC;QAED,MAAM,gBAAgB,GAAG,MAAM,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QACnD,MAAM,OAAO,GAAG,GAAG,gBAAgB,WAAW,CAAC;QAE/C,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;YAC3C,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;YAC3C,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;YAEtC,OAAO,IAAI,CAAC,eAAgB,CAAC,OAAO,EAAE,gBAAgB,CAAC,CAAC;QAC1D,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAED,eAAe,CAAC,OAAe,EAAE,OAAgB;QAC/C,IAAI,CAAC,IAAI,CAAC,eAAe,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE,CAAC;YACnD,OAAO,IAAI,CAAC;QACd,CAAC;QAED,OAAO,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;IAChD,CAAC;IAED,YAAY,CAAC,OAAe;QAC1B,IAAI,CAAC,IAAI,CAAC,eAAe,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC;YAChD,OAAO,IAAI,CAAC;QACd,CAAC;QAED,OAAO,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC;IACpC,CAAC;IAED,gBAAgB,CAAC,OAA+B;QAC9C,IAAI,CAAC,IAAI,CAAC,eAAe,IAAI,CAAC,IAAI,CAAC,uBAAuB,EAAE,CAAC;YAC3D,OAAO,IAAI,CAAC,yBAAyB,CAAC,OAAO,CAAC,CAAC;QACjD,CAAC;QAED,OAAO,IAAI,CAAC,uBAAuB,CAAC,OAAO,CAAC,CAAC;IAC/C,CAAC;IAEO,yBAAyB,CAAC,OAA+B;QAC/D,MAAM,KAAK,GAAa,EAAE,CAAC;QAE3B,KAAK,CAAC,IAAI,CAAC,KAAK,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC;QACpC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,CAAC,IAAI,CAAC,KAAK,OAAO,CAAC,eAAe,EAAE,CAAC,CAAC;QAC3C,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAEf,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;YACrB,KAAK,MAAM,OAAO,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;gBACvC,KAAK,CAAC,IAAI,CAAC,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC;gBAClC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;gBACf,KAAK,MAAM,IAAI,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;oBACjC,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;wBACrB,KAAK,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,KAAK,IAAI,CAAC,GAAG,OAAO,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC;oBAClE,CAAC;yBAAM,CAAC;wBACN,KAAK,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,KAAK,IAAI,CAAC,GAAG,GAAG,CAAC,CAAC;oBAC5C,CAAC;gBACH,CAAC;gBACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACjB,CAAC;QACH,CAAC;QAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC1B,CAAC;IAED,KAAK,CAAC,gBAAgB,CAAC,MAAc;QAKnC,MAAM,gBAAgB,GAAG,MAAM,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QACnD,MAAM,WAAW,GAAG,GAAG,gBAAgB,gBAAgB,CAAC;QAExD,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;YAC3C,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;YAE/C,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;gBACjB,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,CAAC;YACpD,CAAC;YAED,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;YACtC,OAAO;gBACL,MAAM,EAAE,IAAI;gBACZ,MAAM,EAAE,GAAG;gBACX,IAAI,EAAE,OAAO,CAAC,MAAM;aACrB,CAAC;QACJ,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;QAC3B,CAAC;IACH,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,MAAc;QAC3B,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QACxC,OAAO,MAAM,CAAC,KAAK,CAAC;IACtB,CAAC;IAED,KAAK,CAAC,WAAW,CAAC,MAAc;QAC9B,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QACxC,OAAO,MAAM,CAAC,QAAQ,CAAC;IACzB,CAAC;IAED,UAAU,CAAC,MAAe;QACxB,IAAI,MAAM,EAAE,CAAC;YACX,MAAM,gBAAgB,GAAG,MAAM,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;YACnD,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,gBAAgB,CAAC,CAAC;QACtC,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;QACrB,CAAC;IACH,CAAC;IAED,aAAa;QACX,OAAO;YACL,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI;YACrB,OAAO,EAAE,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;SAChC,CAAC;IACJ,CAAC;IAED,eAAe;QACb,OAAO,IAAI,CAAC,eAAe,KAAK,IAAI,CAAC;IACvC,CAAC;CACF;AAED,eAAe,sBAAsB,CAAC"}

package/dist/plugins/spider/recker-robots-validator.js

@@ -0,0 +1,336 @@
+import { createHttpClient } from '#src/concerns/http-client.js';
+export class ReckerRobotsValidator {
+    config;
+    _context;
+    cache;
+    fetcher;
+    _httpClient;
+    reckerAvailable = null;
+    parseRobotsTxt = null;
+    validateRobotsTxt = null;
+    isPathAllowed = null;
+    fetchAndValidateRobotsTxt = null;
+    fallbackParser = null;
+    constructor(config = {}) {
+        this.config = {
+            userAgent: config.userAgent || 's3db-spider',
+            defaultAllow: config.defaultAllow !== false,
+            cacheTimeout: config.cacheTimeout || 3600000,
+            fetchTimeout: config.fetchTimeout || 10000,
+            context: config.context || null,
+            fetcher: config.fetcher || null
+        };
+        this._context = this.config.context;
+        this.cache = new Map();
+        this.fetcher = this.config.fetcher;
+        this._httpClient = null;
+    }
+    async _checkReckerAvailability() {
+        if (this.reckerAvailable !== null) {
+            return this.reckerAvailable;
+        }
+        try {
+            const robotsModule = await import('recker/seo/validators/robots');
+            this.parseRobotsTxt = robotsModule.parseRobotsTxt;
+            this.validateRobotsTxt = robotsModule.validateRobotsTxt;
+            this.isPathAllowed = robotsModule.isPathAllowed;
+            this.fetchAndValidateRobotsTxt = robotsModule.fetchAndValidateRobotsTxt;
+            this.reckerAvailable = true;
+            return true;
+        }
+        catch {
+            this.reckerAvailable = false;
+            return false;
+        }
+    }
+    async _getFallbackParser() {
+        if (!this.fallbackParser) {
+            const { RobotsParser } = await import('./robots-parser.js');
+            this.fallbackParser = new RobotsParser(this.config);
+        }
+        return this.fallbackParser;
+    }
+    setFetcher(fetcher) {
+        this.fetcher = fetcher;
+        if (this.fallbackParser) {
+            this.fallbackParser.setFetcher(fetcher);
+        }
+    }
+    async isAllowed(url) {
+        const isReckerAvailable = await this._checkReckerAvailability();
+        if (!isReckerAvailable) {
+            const fallback = await this._getFallbackParser();
+            return fallback.isAllowed(url);
+        }
+        try {
+            const urlObj = new URL(url);
+            const domain = `${urlObj.protocol}//${urlObj.host}`;
+            const path = urlObj.pathname + urlObj.search;
+            const cached = await this._getCachedOrFetch(domain);
+            if (!cached.parseResult) {
+                return { allowed: this.config.defaultAllow, source: 'no-robots-txt' };
+            }
+            const allowed = this.isPathAllowed(cached.parseResult, path, this.config.userAgent);
+            const crawlDelay = this._getCrawlDelayFromParseResult(cached.parseResult);
+            const matchedRule = this._findMatchedRule(cached.parseResult, path);
+            return {
+                allowed,
+                crawlDelay,
+                source: 'robots-txt',
+                matchedRule
+            };
+        }
+        catch (error) {
+            return {
+                allowed: this.config.defaultAllow,
+                source: 'error',
+                error: error.message
+            };
+        }
+    }
+    async _getCachedOrFetch(domain) {
+        const cached = this.cache.get(domain);
+        if (cached && Date.now() - cached.timestamp < this.config.cacheTimeout) {
+            return cached;
+        }
+        const robotsUrl = `${domain}/robots.txt`;
+        let content = null;
+        try {
+            if (this.fetcher) {
+                content = await this.fetcher(robotsUrl);
+            }
+            else {
+                content = await this._fetchRobotsTxt(robotsUrl);
+            }
+        }
+        catch {
+            const entry = {
+                parseResult: null,
+                validationResult: null,
+                timestamp: Date.now()
+            };
+            this.cache.set(domain, entry);
+            return entry;
+        }
+        const parseResult = this.parseRobotsTxt(content);
+        const validationResult = this.validateRobotsTxt(content, domain);
+        const entry = {
+            parseResult,
+            validationResult,
+            timestamp: Date.now()
+        };
+        this.cache.set(domain, entry);
+        return entry;
+    }
+    async _getHttpClient() {
+        if (!this._httpClient) {
+            const baseConfig = this._context
+                ? this._context.getHttpClientConfig('https://example.com')
+                : {
+                    headers: {
+                        'User-Agent': this.config.userAgent
+                    }
+                };
+            this._httpClient = await createHttpClient({
+                ...baseConfig,
+                timeout: this.config.fetchTimeout,
+                retry: {
+                    maxAttempts: 2,
+                    delay: 500,
+                    backoff: 'exponential',
+                    retryAfter: true,
+                    retryOn: [429, 500, 502, 503, 504]
+                }
+            });
+        }
+        return this._httpClient;
+    }
+    async _fetchRobotsTxt(url) {
+        const client = await this._getHttpClient();
+        const response = await client.get(url);
+        if (this._context) {
+            this._context.processResponse(response, url);
+        }
+        if (!response.ok) {
+            throw new Error(`HTTP ${response.status}`);
+        }
+        return await response.text();
+    }
+    _getCrawlDelayFromParseResult(parseResult) {
+        const userAgentLower = this.config.userAgent.toLowerCase();
+        for (const block of parseResult.userAgentBlocks) {
+            const agents = block.userAgents.map(a => a.toLowerCase());
+            if (agents.includes(userAgentLower)) {
+                return block.crawlDelay ? block.crawlDelay * 1000 : null;
+            }
+        }
+        for (const block of parseResult.userAgentBlocks) {
+            const agents = block.userAgents.map(a => a.toLowerCase());
+            for (const agent of agents) {
+                if (agent !== '*' && (agent.includes(userAgentLower) || userAgentLower.includes(agent))) {
+                    return block.crawlDelay ? block.crawlDelay * 1000 : null;
+                }
+            }
+        }
+        for (const block of parseResult.userAgentBlocks) {
+            if (block.userAgents.map(a => a.toLowerCase()).includes('*')) {
+                return block.crawlDelay ? block.crawlDelay * 1000 : null;
+            }
+        }
+        return null;
+    }
+    _findMatchedRule(parseResult, path) {
+        const userAgentLower = this.config.userAgent.toLowerCase();
+        let targetBlock = null;
+        for (const block of parseResult.userAgentBlocks) {
+            const agents = block.userAgents.map(a => a.toLowerCase());
+            if (agents.includes(userAgentLower)) {
+                targetBlock = block;
+                break;
+            }
+        }
+        if (!targetBlock) {
+            for (const block of parseResult.userAgentBlocks) {
+                const agents = block.userAgents.map(a => a.toLowerCase());
+                for (const agent of agents) {
+                    if (agent !== '*' && (agent.includes(userAgentLower) || userAgentLower.includes(agent))) {
+                        targetBlock = block;
+                        break;
+                    }
+                }
+                if (targetBlock)
+                    break;
+            }
+        }
+        if (!targetBlock) {
+            for (const block of parseResult.userAgentBlocks) {
+                if (block.userAgents.map(a => a.toLowerCase()).includes('*')) {
+                    targetBlock = block;
+                    break;
+                }
+            }
+        }
+        if (!targetBlock)
+            return undefined;
+        const sortedRules = [...targetBlock.rules].sort((a, b) => {
+            const lenA = a.path.replace(/\*/g, '').length;
+            const lenB = b.path.replace(/\*/g, '').length;
+            return lenB - lenA;
+        });
+        for (const rule of sortedRules) {
+            if (this._pathMatches(path, rule.path)) {
+                return rule.path;
+            }
+        }
+        return undefined;
+    }
+    _pathMatches(path, pattern) {
+        let escaped = pattern.replace(/[.+?^{}()|[\]\\]/g, '\\$&');
+        escaped = escaped.replace(/\*/g, '.*');
+        if (escaped.endsWith('$')) {
+            escaped = escaped.slice(0, -1) + '$';
+        }
+        else {
+            escaped = escaped + '.*';
+        }
+        const regex = new RegExp(`^${escaped}$`, 'i');
+        return regex.test(path);
+    }
+    async getSitemaps(domain) {
+        const isReckerAvailable = await this._checkReckerAvailability();
+        if (!isReckerAvailable) {
+            const fallback = await this._getFallbackParser();
+            return fallback.getSitemaps(domain);
+        }
+        const cached = await this._getCachedOrFetch(domain);
+        return cached.parseResult?.sitemaps || [];
+    }
+    async getCrawlDelay(domain) {
+        const isReckerAvailable = await this._checkReckerAvailability();
+        if (!isReckerAvailable) {
+            const fallback = await this._getFallbackParser();
+            return fallback.getCrawlDelay(domain);
+        }
+        const cached = await this._getCachedOrFetch(domain);
+        if (!cached.parseResult)
+            return null;
+        return this._getCrawlDelayFromParseResult(cached.parseResult);
+    }
+    async preload(domain) {
+        await this._getCachedOrFetch(domain);
+    }
+    clearCache(domain) {
+        if (domain) {
+            this.cache.delete(domain);
+        }
+        else {
+            this.cache.clear();
+        }
+    }
+    getCacheStats() {
+        return {
+            size: this.cache.size,
+            domains: [...this.cache.keys()]
+        };
+    }
+    async validate(url) {
+        const isReckerAvailable = await this._checkReckerAvailability();
+        if (!isReckerAvailable) {
+            return null;
+        }
+        try {
+            const urlObj = new URL(url);
+            const domain = `${urlObj.protocol}//${urlObj.host}`;
+            const cached = await this._getCachedOrFetch(domain);
+            if (!cached.validationResult || !cached.parseResult) {
+                return null;
+            }
+            return {
+                valid: cached.validationResult.valid,
+                issues: cached.validationResult.issues,
+                blocksAllRobots: cached.parseResult.blocksAllRobots,
+                blocksImportantPaths: cached.parseResult.blocksImportantPaths,
+                host: cached.parseResult.host,
+                size: cached.parseResult.size
+            };
+        }
+        catch {
+            return null;
+        }
+    }
+    async validateContent(content, baseUrl) {
+        const isReckerAvailable = await this._checkReckerAvailability();
+        if (!isReckerAvailable || !this.validateRobotsTxt) {
+            return null;
+        }
+        return this.validateRobotsTxt(content, baseUrl);
+    }
+    parseContent(content) {
+        if (!this.reckerAvailable || !this.parseRobotsTxt) {
+            return null;
+        }
+        return this.parseRobotsTxt(content);
+    }
+    getBlockingStatus(domain) {
+        const cached = this.cache.get(domain);
+        if (!cached?.parseResult)
+            return null;
+        return {
+            blocksAllRobots: cached.parseResult.blocksAllRobots,
+            blocksImportantPaths: cached.parseResult.blocksImportantPaths
+        };
+    }
+    getHost(domain) {
+        const cached = this.cache.get(domain);
+        return cached?.parseResult?.host || null;
+    }
+    async getValidationIssues(domain) {
+        const cached = await this._getCachedOrFetch(domain);
+        return cached.validationResult?.issues || [];
+    }
+    isReckerEnabled() {
+        return this.reckerAvailable === true;
+    }
+}
+export default ReckerRobotsValidator;
+//# sourceMappingURL=recker-robots-validator.js.map