recker 1.0.31 → 1.0.32-next.e0741bf

package/dist/seo/utils/similarity.js

@@ -0,0 +1,273 @@
+export const DEFAULT_SIMILARITY_THRESHOLD = 85;
+const SHINGLE_SIZE = 3;
+const SIMHASH_BITS = 64;
+export function normalizeText(text, removeStopWords = false) {
+    let normalized = text
+        .toLowerCase()
+        .replace(/[\r\n\t]+/g, ' ')
+        .replace(/[^\w\s]/g, ' ')
+        .replace(/\s+/g, ' ')
+        .trim();
+    if (removeStopWords) {
+        const stopWords = new Set([
+            'a', 'an', 'the', 'and', 'or', 'but', 'in', 'on', 'at', 'to', 'for',
+            'of', 'with', 'by', 'from', 'as', 'is', 'was', 'are', 'were', 'been',
+            'be', 'have', 'has', 'had', 'do', 'does', 'did', 'will', 'would',
+            'could', 'should', 'may', 'might', 'must', 'shall', 'can', 'need',
+            'it', 'its', 'this', 'that', 'these', 'those', 'i', 'you', 'he',
+            'she', 'we', 'they', 'what', 'which', 'who', 'whom', 'when', 'where',
+            'why', 'how', 'all', 'each', 'every', 'both', 'few', 'more', 'most',
+            'other', 'some', 'such', 'no', 'nor', 'not', 'only', 'own', 'same',
+            'so', 'than', 'too', 'very', 'just',
+        ]);
+        normalized = normalized
+            .split(' ')
+            .filter(word => !stopWords.has(word) && word.length > 1)
+            .join(' ');
+    }
+    return normalized;
+}
+export function tokenize(text) {
+    return text.split(/\s+/).filter(word => word.length > 0);
+}
+export function createShingles(tokens, size = SHINGLE_SIZE) {
+    const shingles = new Set();
+    if (tokens.length < size) {
+        for (const token of tokens) {
+            shingles.add(hashString(token));
+        }
+        return shingles;
+    }
+    for (let i = 0; i <= tokens.length - size; i++) {
+        const shingle = tokens.slice(i, i + size).join(' ');
+        shingles.add(hashString(shingle));
+    }
+    return shingles;
+}
+function hashString(str) {
+    let hash = 2166136261;
+    for (let i = 0; i < str.length; i++) {
+        hash ^= str.charCodeAt(i);
+        hash = Math.imul(hash, 16777619);
+    }
+    return hash >>> 0;
+}
+function hash64(str) {
+    let h1 = 2166136261n;
+    let h2 = 2166136261n;
+    for (let i = 0; i < str.length; i++) {
+        const c = BigInt(str.charCodeAt(i));
+        h1 ^= c;
+        h1 = (h1 * 16777619n) & 0xffffffffn;
+        h2 ^= c * 31n;
+        h2 = (h2 * 16777619n) & 0xffffffffn;
+    }
+    return (h1 << 32n) | h2;
+}
+export function calculateSimHash(text) {
+    const normalized = normalizeText(text, true);
+    const tokens = tokenize(normalized);
+    if (tokens.length === 0) {
+        return 0n;
+    }
+    const bitCounts = new Array(SIMHASH_BITS).fill(0);
+    for (const token of tokens) {
+        const hash = hash64(token);
+        for (let i = 0; i < SIMHASH_BITS; i++) {
+            const bit = (hash >> BigInt(i)) & 1n;
+            bitCounts[i] += bit === 1n ? 1 : -1;
+        }
+    }
+    let simhash = 0n;
+    for (let i = 0; i < SIMHASH_BITS; i++) {
+        if (bitCounts[i] > 0) {
+            simhash |= 1n << BigInt(i);
+        }
+    }
+    return simhash;
+}
+export function hammingDistance(hash1, hash2) {
+    let xor = hash1 ^ hash2;
+    let distance = 0;
+    while (xor > 0n) {
+        distance += Number(xor & 1n);
+        xor >>= 1n;
+    }
+    return distance;
+}
+export function simhashSimilarity(hash1, hash2) {
+    const distance = hammingDistance(hash1, hash2);
+    return ((SIMHASH_BITS - distance) / SIMHASH_BITS) * 100;
+}
+export function jaccardSimilarity(setA, setB) {
+    if (setA.size === 0 && setB.size === 0) {
+        return 100;
+    }
+    if (setA.size === 0 || setB.size === 0) {
+        return 0;
+    }
+    let intersection = 0;
+    for (const item of setA) {
+        if (setB.has(item)) {
+            intersection++;
+        }
+    }
+    const union = setA.size + setB.size - intersection;
+    return (intersection / union) * 100;
+}
+export function createFingerprint(url, text) {
+    const normalizedText = normalizeText(text, false);
+    const tokens = tokenize(normalizeText(text, true));
+    return {
+        url,
+        simhash: calculateSimHash(text),
+        shingles: createShingles(tokens),
+        wordCount: tokens.length,
+        normalizedText,
+    };
+}
+export function compareFingerprints(fpA, fpB) {
+    if (fpA.normalizedText === fpB.normalizedText) {
+        return {
+            urlA: fpA.url,
+            urlB: fpB.url,
+            similarity: 100,
+            type: 'exact',
+            simhashDistance: 0,
+            jaccardIndex: 100,
+        };
+    }
+    const simhashSim = simhashSimilarity(fpA.simhash, fpB.simhash);
+    const jaccardSim = jaccardSimilarity(fpA.shingles, fpB.shingles);
+    const similarity = (simhashSim * 0.3 + jaccardSim * 0.7);
+    let type;
+    if (similarity >= 95) {
+        type = 'near-duplicate';
+    }
+    else if (similarity >= 70) {
+        type = 'similar';
+    }
+    else {
+        type = 'different';
+    }
+    return {
+        urlA: fpA.url,
+        urlB: fpB.url,
+        similarity: Math.round(similarity * 10) / 10,
+        type,
+        simhashDistance: hammingDistance(fpA.simhash, fpB.simhash),
+        jaccardIndex: Math.round(jaccardSim * 10) / 10,
+    };
+}
+export function findDuplicateContent(pages, threshold = DEFAULT_SIMILARITY_THRESHOLD) {
+    const duplicates = [];
+    const fingerprints = pages.map(p => createFingerprint(p.url, p.content));
+    for (let i = 0; i < fingerprints.length; i++) {
+        for (let j = i + 1; j < fingerprints.length; j++) {
+            const result = compareFingerprints(fingerprints[i], fingerprints[j]);
+            if (result.similarity >= threshold) {
+                duplicates.push(result);
+            }
+        }
+    }
+    return duplicates.sort((a, b) => b.similarity - a.similarity);
+}
+export function findDuplicateMetadata(pages, threshold = DEFAULT_SIMILARITY_THRESHOLD) {
+    const groups = [];
+    const findFieldDuplicates = (field, getValue) => {
+        const exactGroups = new Map();
+        const values = [];
+        for (const page of pages) {
+            const value = getValue(page);
+            if (!value || value.trim().length === 0)
+                continue;
+            const normalized = normalizeText(value);
+            values.push({ url: page.url, value, normalized });
+            if (!exactGroups.has(normalized)) {
+                exactGroups.set(normalized, []);
+            }
+            exactGroups.get(normalized).push(page.url);
+        }
+        for (const [normalized, urls] of exactGroups) {
+            if (urls.length > 1) {
+                const originalValue = values.find(v => v.normalized === normalized)?.value || normalized;
+                groups.push({
+                    type: field,
+                    value: originalValue,
+                    urls,
+                    similarity: 100,
+                });
+            }
+        }
+        if (threshold < 100) {
+            const uniqueNormalized = [...new Set(values.map(v => v.normalized))];
+            for (let i = 0; i < uniqueNormalized.length; i++) {
+                for (let j = i + 1; j < uniqueNormalized.length; j++) {
+                    const a = uniqueNormalized[i];
+                    const b = uniqueNormalized[j];
+                    const similarity = calculateStringSimilarity(a, b);
+                    if (similarity >= threshold && similarity < 100) {
+                        const urlsA = values.filter(v => v.normalized === a).map(v => v.url);
+                        const urlsB = values.filter(v => v.normalized === b).map(v => v.url);
+                        const allUrls = [...new Set([...urlsA, ...urlsB])];
+                        if (allUrls.length > 1) {
+                            groups.push({
+                                type: field,
+                                value: values.find(v => v.normalized === a)?.value || a,
+                                urls: allUrls,
+                                similarity: Math.round(similarity),
+                            });
+                        }
+                    }
+                }
+            }
+        }
+    };
+    findFieldDuplicates('title', p => p.title);
+    findFieldDuplicates('description', p => p.description);
+    findFieldDuplicates('h1', p => p.h1);
+    return groups;
+}
+export function calculateStringSimilarity(a, b) {
+    if (a === b)
+        return 100;
+    if (!a || !b)
+        return 0;
+    const normA = normalizeText(a);
+    const normB = normalizeText(b);
+    if (normA === normB)
+        return 100;
+    const tokensA = new Set(tokenize(normA));
+    const tokensB = new Set(tokenize(normB));
+    let intersection = 0;
+    for (const token of tokensA) {
+        if (tokensB.has(token))
+            intersection++;
+    }
+    const tokenSimilarity = (2 * intersection) / (tokensA.size + tokensB.size) * 100;
+    const lengthRatio = Math.min(normA.length, normB.length) / Math.max(normA.length, normB.length);
+    const lengthSimilarity = lengthRatio * 100;
+    return (tokenSimilarity * 0.7 + lengthSimilarity * 0.3);
+}
+export function calculateTextToHtmlRatio(html, text) {
+    const htmlLength = html.length;
+    const textLength = text.length;
+    if (htmlLength === 0)
+        return 0;
+    return (textLength / htmlLength) * 100;
+}
+export function isThinContent(wordCount, textToHtmlRatio, minWords = 300, minRatio = 10) {
+    if (wordCount < minWords) {
+        return {
+            isThin: true,
+            reason: `Low word count: ${wordCount} words (min: ${minWords})`,
+        };
+    }
+    if (textToHtmlRatio !== undefined && textToHtmlRatio < minRatio) {
+        return {
+            isThin: true,
+            reason: `Low text-to-HTML ratio: ${textToHtmlRatio.toFixed(1)}% (min: ${minRatio}%)`,
+        };
+    }
+    return { isThin: false };
+}

package/dist/seo/validators/index.d.ts

@@ -0,0 +1,3 @@
+export * from './robots.js';
+export * from './sitemap.js';
+export * from './llms-txt.js';

package/dist/seo/validators/index.js

@@ -0,0 +1,3 @@
+export * from './robots.js';
+export * from './sitemap.js';
+export * from './llms-txt.js';

package/dist/seo/validators/llms-txt.d.ts

@@ -0,0 +1,57 @@
+export interface LlmsTxtLink {
+    text: string;
+    url: string;
+    description?: string;
+    section?: string;
+}
+export interface LlmsTxtSection {
+    title: string;
+    content: string;
+    links: LlmsTxtLink[];
+}
+export interface LlmsTxtParseResult {
+    valid: boolean;
+    errors: string[];
+    warnings: string[];
+    siteName?: string;
+    siteDescription?: string;
+    sections: LlmsTxtSection[];
+    links: LlmsTxtLink[];
+    hasFullVersion: boolean;
+    rawContent: string;
+    size: number;
+}
+export interface LlmsTxtValidationIssue {
+    type: 'error' | 'warning' | 'info';
+    code: string;
+    message: string;
+    line?: number;
+    recommendation?: string;
+}
+export interface LlmsTxtValidationResult {
+    valid: boolean;
+    issues: LlmsTxtValidationIssue[];
+    parseResult: LlmsTxtParseResult;
+}
+export declare function parseLlmsTxt(content: string): LlmsTxtParseResult;
+export declare function validateLlmsTxt(content: string, baseUrl?: string): LlmsTxtValidationResult;
+export declare function fetchAndValidateLlmsTxt(url: string, fetcher?: (url: string) => Promise<{
+    status: number;
+    text: string;
+}>): Promise<LlmsTxtValidationResult & {
+    exists: boolean;
+    status?: number;
+    fullVersionExists?: boolean;
+}>;
+export declare function generateLlmsTxtTemplate(options: {
+    siteName: string;
+    siteDescription: string;
+    sections?: Array<{
+        title: string;
+        links: Array<{
+            text: string;
+            url: string;
+            description?: string;
+        }>;
+    }>;
+}): string;

package/dist/seo/validators/llms-txt.js

@@ -0,0 +1,317 @@
+const MAX_FILE_SIZE = 100 * 1024;
+const MIN_DESCRIPTION_LENGTH = 50;
+const MAX_DESCRIPTION_LENGTH = 500;
+export function parseLlmsTxt(content) {
+    const errors = [];
+    const warnings = [];
+    const sections = [];
+    const links = [];
+    let siteName;
+    let siteDescription;
+    let hasFullVersion = false;
+    let currentSection = null;
+    const lines = content.split(/\r?\n/);
+    for (let i = 0; i < lines.length; i++) {
+        const line = lines[i];
+        const trimmedLine = line.trim();
+        if (!trimmedLine)
+            continue;
+        if (trimmedLine.startsWith('# ') && !siteName) {
+            siteName = trimmedLine.substring(2).trim();
+            continue;
+        }
+        if (trimmedLine.startsWith('> ') && !siteDescription) {
+            siteDescription = trimmedLine.substring(2).trim();
+            continue;
+        }
+        if (trimmedLine.startsWith('## ')) {
+            const title = trimmedLine.substring(3).trim();
+            currentSection = { title, content: '', links: [] };
+            sections.push(currentSection);
+            continue;
+        }
+        const linkMatches = trimmedLine.matchAll(/\[([^\]]+)\]\(([^)]+)\)/g);
+        for (const match of linkMatches) {
+            const linkText = match[1];
+            const linkUrl = match[2];
+            if (linkUrl.includes('llms-full.txt') || linkUrl.includes('llms_full.txt')) {
+                hasFullVersion = true;
+            }
+            const link = {
+                text: linkText,
+                url: linkUrl,
+                section: currentSection?.title,
+            };
+            const afterLink = trimmedLine.substring(trimmedLine.indexOf(match[0]) + match[0].length).trim();
+            if (afterLink.startsWith(':') || afterLink.startsWith('-')) {
+                link.description = afterLink.substring(1).trim();
+            }
+            links.push(link);
+            if (currentSection) {
+                currentSection.links.push(link);
+            }
+        }
+        if (currentSection && !trimmedLine.startsWith('##')) {
+            if (currentSection.content) {
+                currentSection.content += '\n' + trimmedLine;
+            }
+            else {
+                currentSection.content = trimmedLine;
+            }
+        }
+    }
+    if (!siteName) {
+        errors.push('Missing site name (should be first line starting with #)');
+    }
+    if (!siteDescription) {
+        warnings.push('Missing site description (should be a > blockquote after site name)');
+    }
+    if (links.length === 0) {
+        warnings.push('No links found in llms.txt');
+    }
+    return {
+        valid: errors.length === 0,
+        errors,
+        warnings,
+        siteName,
+        siteDescription,
+        sections,
+        links,
+        hasFullVersion,
+        rawContent: content,
+        size: content.length,
+    };
+}
+export function validateLlmsTxt(content, baseUrl) {
+    const parseResult = parseLlmsTxt(content);
+    const issues = [];
+    for (const error of parseResult.errors) {
+        issues.push({
+            type: 'error',
+            code: 'PARSE_ERROR',
+            message: error,
+        });
+    }
+    for (const warning of parseResult.warnings) {
+        issues.push({
+            type: 'warning',
+            code: 'PARSE_WARNING',
+            message: warning,
+        });
+    }
+    if (parseResult.size > MAX_FILE_SIZE) {
+        issues.push({
+            type: 'warning',
+            code: 'FILE_TOO_LARGE',
+            message: `llms.txt is ${Math.round(parseResult.size / 1024)}KB (recommended max: 100KB)`,
+            recommendation: 'Keep llms.txt concise; use llms-full.txt for detailed content',
+        });
+    }
+    if (parseResult.siteDescription) {
+        if (parseResult.siteDescription.length < MIN_DESCRIPTION_LENGTH) {
+            issues.push({
+                type: 'info',
+                code: 'SHORT_DESCRIPTION',
+                message: `Site description is short (${parseResult.siteDescription.length} chars)`,
+                recommendation: `Provide at least ${MIN_DESCRIPTION_LENGTH} characters of description`,
+            });
+        }
+        else if (parseResult.siteDescription.length > MAX_DESCRIPTION_LENGTH) {
+            issues.push({
+                type: 'info',
+                code: 'LONG_DESCRIPTION',
+                message: `Site description is long (${parseResult.siteDescription.length} chars)`,
+                recommendation: `Keep description under ${MAX_DESCRIPTION_LENGTH} characters`,
+            });
+        }
+    }
+    const sectionTitles = parseResult.sections.map(s => s.title.toLowerCase());
+    const recommendedSections = ['docs', 'documentation', 'api', 'getting started', 'about'];
+    const hasSomeRecommended = recommendedSections.some(s => sectionTitles.some(t => t.includes(s)));
+    if (!hasSomeRecommended && parseResult.sections.length === 0) {
+        issues.push({
+            type: 'info',
+            code: 'NO_SECTIONS',
+            message: 'No content sections found',
+            recommendation: 'Add sections like ## Docs, ## API, ## About for better organization',
+        });
+    }
+    if (baseUrl) {
+        const baseHost = new URL(baseUrl).hostname;
+        for (const link of parseResult.links) {
+            try {
+                const linkUrl = new URL(link.url, baseUrl);
+                if (!link.url.startsWith('http') && !link.url.startsWith('/')) {
+                    issues.push({
+                        type: 'warning',
+                        code: 'RELATIVE_URL',
+                        message: `Relative URL may not resolve correctly: ${link.url}`,
+                        recommendation: 'Use absolute URLs or URLs starting with /',
+                    });
+                }
+            }
+            catch {
+                issues.push({
+                    type: 'error',
+                    code: 'INVALID_URL',
+                    message: `Invalid URL: ${link.url}`,
+                });
+            }
+        }
+    }
+    if (!parseResult.hasFullVersion && parseResult.size > 10000) {
+        issues.push({
+            type: 'info',
+            code: 'CONSIDER_FULL_VERSION',
+            message: 'Large llms.txt without llms-full.txt reference',
+            recommendation: 'Consider creating llms-full.txt for detailed content',
+        });
+    }
+    const seenUrls = new Set();
+    for (const link of parseResult.links) {
+        const normalized = link.url.toLowerCase();
+        if (seenUrls.has(normalized)) {
+            issues.push({
+                type: 'info',
+                code: 'DUPLICATE_LINK',
+                message: `Duplicate link: ${link.url}`,
+            });
+        }
+        else {
+            seenUrls.add(normalized);
+        }
+    }
+    return {
+        valid: issues.filter(i => i.type === 'error').length === 0,
+        issues,
+        parseResult,
+    };
+}
+export async function fetchAndValidateLlmsTxt(url, fetcher) {
+    const llmsTxtUrl = new URL('/llms.txt', url).href;
+    try {
+        let response;
+        if (fetcher) {
+            response = await fetcher(llmsTxtUrl);
+        }
+        else {
+            const fetchResponse = await fetch(llmsTxtUrl);
+            response = {
+                status: fetchResponse.status,
+                text: await fetchResponse.text(),
+            };
+        }
+        if (response.status === 404) {
+            return {
+                exists: false,
+                status: 404,
+                valid: false,
+                issues: [{
+                        type: 'info',
+                        code: 'NOT_FOUND',
+                        message: 'llms.txt not found',
+                        recommendation: 'Create llms.txt to optimize for AI/LLM discovery (see llmstxt.org)',
+                    }],
+                parseResult: {
+                    valid: false,
+                    errors: [],
+                    warnings: [],
+                    sections: [],
+                    links: [],
+                    hasFullVersion: false,
+                    rawContent: '',
+                    size: 0,
+                },
+            };
+        }
+        if (response.status >= 400) {
+            return {
+                exists: false,
+                status: response.status,
+                valid: false,
+                issues: [{
+                        type: 'error',
+                        code: 'FETCH_ERROR',
+                        message: `Failed to fetch llms.txt (HTTP ${response.status})`,
+                    }],
+                parseResult: {
+                    valid: false,
+                    errors: [],
+                    warnings: [],
+                    sections: [],
+                    links: [],
+                    hasFullVersion: false,
+                    rawContent: '',
+                    size: 0,
+                },
+            };
+        }
+        const validation = validateLlmsTxt(response.text, url);
+        let fullVersionExists = false;
+        if (validation.parseResult.hasFullVersion) {
+            try {
+                const fullUrl = new URL('/llms-full.txt', url).href;
+                let fullResponse;
+                if (fetcher) {
+                    fullResponse = await fetcher(fullUrl);
+                }
+                else {
+                    fullResponse = await fetch(fullUrl, { method: 'HEAD' });
+                }
+                fullVersionExists = fullResponse.status === 200;
+            }
+            catch {
+            }
+        }
+        return {
+            ...validation,
+            exists: true,
+            status: response.status,
+            fullVersionExists,
+        };
+    }
+    catch (error) {
+        return {
+            exists: false,
+            valid: false,
+            issues: [{
+                    type: 'error',
+                    code: 'FETCH_ERROR',
+                    message: `Failed to fetch llms.txt: ${error instanceof Error ? error.message : 'Unknown error'}`,
+                }],
+            parseResult: {
+                valid: false,
+                errors: [],
+                warnings: [],
+                sections: [],
+                links: [],
+                hasFullVersion: false,
+                rawContent: '',
+                size: 0,
+            },
+        };
+    }
+}
+export function generateLlmsTxtTemplate(options) {
+    const lines = [];
+    lines.push(`# ${options.siteName}`);
+    lines.push('');
+    lines.push(`> ${options.siteDescription}`);
+    lines.push('');
+    if (options.sections) {
+        for (const section of options.sections) {
+            lines.push(`## ${section.title}`);
+            lines.push('');
+            for (const link of section.links) {
+                if (link.description) {
+                    lines.push(`- [${link.text}](${link.url}): ${link.description}`);
+                }
+                else {
+                    lines.push(`- [${link.text}](${link.url})`);
+                }
+            }
+            lines.push('');
+        }
+    }
+    return lines.join('\n');
+}