recker 1.0.26 → 1.0.27

package/dist/browser/utils/header-parser.js

@@ -0,0 +1,617 @@
+export function parseCacheInfo(headers) {
+    const info = { hit: false };
+    const cfCacheStatus = headers.get('cf-cache-status');
+    if (cfCacheStatus) {
+        info.provider = 'cloudflare';
+        info.status = cfCacheStatus.toUpperCase();
+        info.hit = cfCacheStatus.toUpperCase() === 'HIT';
+    }
+    const xCache = headers.get('x-cache');
+    if (xCache) {
+        const upper = xCache.toUpperCase();
+        info.hit = upper.includes('HIT');
+        if (upper.includes('FASTLY'))
+            info.provider = 'fastly';
+        else if (upper.includes('VARNISH'))
+            info.provider = 'varnish';
+        if (upper.includes('HIT'))
+            info.status = 'HIT';
+        else if (upper.includes('MISS'))
+            info.status = 'MISS';
+        else if (upper.includes('EXPIRED'))
+            info.status = 'EXPIRED';
+        else if (upper.includes('STALE'))
+            info.status = 'STALE';
+    }
+    const akamaiCache = headers.get('x-akamai-cache-status');
+    if (akamaiCache) {
+        info.provider = 'akamai';
+        info.hit = akamaiCache.toUpperCase().includes('HIT');
+    }
+    const cloudFrontCache = headers.get('x-cache');
+    const cloudFrontId = headers.get('x-amz-cf-id');
+    if (cloudFrontId) {
+        info.provider = 'cloudfront';
+    }
+    const cacheControl = headers.get('cache-control');
+    if (cacheControl) {
+        const maxAgeMatch = cacheControl.match(/max-age=(\d+)/);
+        if (maxAgeMatch) {
+            info.maxAge = parseInt(maxAgeMatch[1], 10);
+        }
+    }
+    const age = headers.get('age');
+    if (age) {
+        info.age = parseInt(age, 10);
+    }
+    return info;
+}
+const PLATFORM_DETECTORS = [
+    {
+        name: 'cloudflare',
+        category: 'cdn',
+        headers: ['cf-ray', 'cf-cache-status'],
+        detect: (headers) => headers.get('cf-ray') !== null,
+        extract: (headers) => {
+            const ray = headers.get('cf-ray');
+            const metadata = {};
+            if (ray) {
+                metadata.ray = ray;
+                const parts = ray.split('-');
+                if (parts.length > 1) {
+                    return { region: parts[1], metadata };
+                }
+            }
+            return { metadata };
+        }
+    },
+    {
+        name: 'fastly',
+        category: 'cdn',
+        headers: ['fastly-debug-digest', 'x-served-by'],
+        detect: (headers) => {
+            const servedBy = headers.get('x-served-by');
+            return servedBy?.includes('cache-') || headers.get('fastly-debug-digest') !== null;
+        },
+        extract: (headers) => ({
+            server: headers.get('x-served-by') || undefined,
+            metadata: {
+                digest: headers.get('fastly-debug-digest')
+            }
+        })
+    },
+    {
+        name: 'akamai',
+        category: 'cdn',
+        headers: ['x-akamai-request-id', 'x-akamai-cache-status'],
+        detect: (headers) => headers.get('x-akamai-request-id') !== null,
+        extract: (headers) => ({
+            metadata: {
+                requestId: headers.get('x-akamai-request-id'),
+                cacheId: headers.get('x-akamai-cache-status')
+            }
+        })
+    },
+    {
+        name: 'aws-cloudfront',
+        category: 'cloud',
+        headers: ['x-amz-cf-id', 'x-amz-cf-pop'],
+        detect: (headers) => headers.get('x-amz-cf-id') !== null,
+        extract: (headers) => ({
+            region: headers.get('x-amz-cf-pop') || undefined,
+            metadata: {
+                requestId: headers.get('x-amz-cf-id'),
+                pop: headers.get('x-amz-cf-pop')
+            }
+        })
+    },
+    {
+        name: 'gcp',
+        category: 'cloud',
+        headers: ['x-cloud-trace-context', 'x-goog-request-id'],
+        detect: (headers) => headers.get('x-cloud-trace-context') !== null || headers.get('x-goog-request-id') !== null,
+        extract: (headers) => {
+            const trace = headers.get('x-cloud-trace-context');
+            return {
+                metadata: {
+                    traceId: trace ? trace.split('/')[0] : undefined,
+                    requestId: headers.get('x-goog-request-id')
+                }
+            };
+        }
+    },
+    {
+        name: 'azure',
+        category: 'cloud',
+        headers: ['x-ms-request-id', 'x-azure-ref'],
+        detect: (headers) => headers.get('x-ms-request-id') !== null,
+        extract: (headers) => ({
+            metadata: {
+                requestId: headers.get('x-ms-request-id'),
+                ref: headers.get('x-azure-ref')
+            }
+        })
+    },
+    {
+        name: 'oracle-cloud',
+        category: 'cloud',
+        headers: ['x-oracle-dms-rid', 'x-oracle-dms-ecid'],
+        detect: (headers) => headers.get('x-oracle-dms-rid') !== null,
+        extract: (headers) => ({
+            metadata: {
+                requestId: headers.get('x-oracle-dms-rid'),
+                ecid: headers.get('x-oracle-dms-ecid')
+            }
+        })
+    },
+    {
+        name: 'alibaba-cloud',
+        category: 'cloud',
+        headers: ['ali-swift-global-savetime', 'eagleeye-traceid'],
+        detect: (headers) => headers.get('eagleeye-traceid') !== null,
+        extract: (headers) => ({
+            metadata: {
+                traceId: headers.get('eagleeye-traceid')
+            }
+        })
+    },
+    {
+        name: 'vercel',
+        category: 'hosting',
+        headers: ['x-vercel-id', 'x-vercel-cache'],
+        detect: (headers) => headers.get('x-vercel-id') !== null,
+        extract: (headers) => ({
+            metadata: {
+                requestId: headers.get('x-vercel-id'),
+                cache: headers.get('x-vercel-cache')
+            }
+        })
+    },
+    {
+        name: 'netlify',
+        category: 'hosting',
+        headers: ['x-nf-request-id', 'x-nf-trace-id'],
+        detect: (headers) => headers.get('x-nf-request-id') !== null,
+        extract: (headers) => ({
+            metadata: {
+                requestId: headers.get('x-nf-request-id'),
+                traceId: headers.get('x-nf-trace-id')
+            }
+        })
+    },
+    {
+        name: 'render',
+        category: 'hosting',
+        headers: ['x-render-origin-server'],
+        detect: (headers) => headers.get('x-render-origin-server') !== null,
+        extract: (headers) => ({
+            metadata: {
+                origin: headers.get('x-render-origin-server')
+            }
+        })
+    },
+    {
+        name: 'datadome',
+        category: 'security',
+        headers: ['x-datadome', 'x-datadome-cid'],
+        detect: (headers) => headers.get('x-datadome') !== null || headers.get('x-datadome-cid') !== null,
+        extract: (headers) => ({
+            metadata: { cid: headers.get('x-datadome-cid') }
+        })
+    },
+    {
+        name: 'incapsula',
+        category: 'security',
+        headers: ['x-iinfo', 'x-cdn'],
+        detect: (headers) => headers.get('x-iinfo') !== null || (headers.get('x-cdn') || '').includes('Incapsula'),
+        extract: (headers) => ({
+            metadata: { iinfo: headers.get('x-iinfo') }
+        })
+    },
+    {
+        name: 'imperva',
+        category: 'security',
+        headers: ['x-imperva-uuid'],
+        detect: (headers) => headers.get('x-imperva-uuid') !== null,
+        extract: (headers) => ({
+            metadata: { uuid: headers.get('x-imperva-uuid') }
+        })
+    },
+    {
+        name: 'aws-waf',
+        category: 'security',
+        headers: ['x-amzn-waf-action'],
+        detect: (headers) => headers.get('server') === 'awselb/2.0',
+        extract: (headers) => ({})
+    },
+    {
+        name: 'envoy',
+        category: 'proxy',
+        headers: ['x-envoy-upstream-service-time'],
+        detect: (headers) => headers.get('x-envoy-upstream-service-time') !== null || headers.get('server') === 'envoy',
+        extract: (headers) => ({})
+    },
+    {
+        name: 'traefik',
+        category: 'proxy',
+        headers: [],
+        detect: (headers) => (headers.get('server') || '').toLowerCase().includes('traefik'),
+        extract: (headers) => ({})
+    },
+    {
+        name: 'caddy',
+        category: 'server',
+        headers: [],
+        detect: (headers) => (headers.get('server') || '').toLowerCase().includes('caddy'),
+        extract: (headers) => ({})
+    },
+    {
+        name: 'haproxy',
+        category: 'proxy',
+        headers: [],
+        detect: (headers) => (headers.get('server') || '').toLowerCase().includes('haproxy'),
+        extract: (headers) => ({})
+    },
+    {
+        name: 'iis',
+        category: 'server',
+        headers: ['x-powered-by'],
+        detect: (headers) => (headers.get('server') || '').toLowerCase().includes('iis'),
+        extract: (headers) => ({})
+    },
+    {
+        name: 'kestrel',
+        category: 'server',
+        headers: [],
+        detect: (headers) => (headers.get('server') || '').toLowerCase().includes('kestrel'),
+        extract: (headers) => ({})
+    },
+    {
+        name: 'express',
+        category: 'framework',
+        headers: ['x-powered-by'],
+        detect: (headers) => (headers.get('x-powered-by') || '').toLowerCase().includes('express'),
+        extract: (headers) => ({})
+    },
+    {
+        name: 'rails',
+        category: 'framework',
+        headers: ['x-runtime', 'x-request-id'],
+        detect: (headers) => headers.get('x-runtime') !== null && !(headers.get('x-powered-by') || '').includes('Express'),
+        extract: (headers) => ({
+            metadata: { runtime: headers.get('x-runtime') }
+        })
+    },
+    {
+        name: 'django',
+        category: 'framework',
+        headers: [],
+        detect: (headers) => (headers.get('server') || '').toLowerCase().includes('wsgi'),
+        extract: (headers) => ({})
+    },
+    {
+        name: 'aspnet',
+        category: 'framework',
+        headers: ['x-aspnet-version', 'x-powered-by'],
+        detect: (headers) => headers.get('x-aspnet-version') !== null || (headers.get('x-powered-by') || '').includes('ASP.NET'),
+        extract: (headers) => ({
+            metadata: { version: headers.get('x-aspnet-version') }
+        })
+    },
+    {
+        name: 'php',
+        category: 'framework',
+        headers: ['x-powered-by'],
+        detect: (headers) => (headers.get('x-powered-by') || '').toLowerCase().includes('php'),
+        extract: (headers) => ({
+            metadata: { version: headers.get('x-powered-by') }
+        })
+    }
+];
+export function parsePlatformInfo(headers) {
+    const info = {
+        metadata: {}
+    };
+    for (const detector of PLATFORM_DETECTORS) {
+        if (detector.detect(headers)) {
+            info.provider = detector.name;
+            info.category = detector.category;
+            const extracted = detector.extract(headers);
+            if (extracted.region)
+                info.region = extracted.region;
+            if (extracted.server)
+                info.server = extracted.server;
+            if (extracted.metadata) {
+                Object.assign(info.metadata, extracted.metadata);
+            }
+            break;
+        }
+    }
+    if (!info.provider) {
+        const server = headers.get('server');
+        if (server) {
+            info.server = server;
+            const serverLower = server.toLowerCase();
+            if (serverLower.includes('cloudflare')) {
+                info.provider = 'cloudflare';
+                info.category = 'cdn';
+            }
+            else if (serverLower.includes('cloudfront')) {
+                info.provider = 'aws-cloudfront';
+                info.category = 'cloud';
+            }
+            else if (serverLower.includes('fastly')) {
+                info.provider = 'fastly';
+                info.category = 'cdn';
+            }
+            else if (serverLower.includes('akamai')) {
+                info.provider = 'akamai';
+                info.category = 'cdn';
+            }
+            else if (serverLower.includes('nginx')) {
+                info.provider = 'nginx';
+                info.category = 'server';
+            }
+            else if (serverLower.includes('apache')) {
+                info.provider = 'apache';
+                info.category = 'server';
+            }
+            else if (serverLower.includes('varnish')) {
+                info.provider = 'varnish';
+                info.category = 'proxy';
+            }
+            else {
+                info.provider = 'unknown';
+                info.category = 'unknown';
+            }
+        }
+    }
+    if (!info.provider) {
+        const via = headers.get('via');
+        if (via) {
+            const viaLower = via.toLowerCase();
+            if (viaLower.includes('cloudflare')) {
+                info.provider = 'cloudflare';
+                info.category = 'cdn';
+            }
+            else if (viaLower.includes('akamai')) {
+                info.provider = 'akamai';
+                info.category = 'cdn';
+            }
+            else if (viaLower.includes('varnish')) {
+                info.provider = 'varnish';
+                info.category = 'proxy';
+            }
+            else if (viaLower.includes('squid')) {
+                info.provider = 'squid';
+                info.category = 'proxy';
+            }
+        }
+    }
+    if (Object.keys(info.metadata || {}).length === 0) {
+        delete info.metadata;
+    }
+    return info;
+}
+export function parseRateLimitInfo(headers, status) {
+    const info = { limited: status === 429 };
+    const limit = headers.get('x-ratelimit-limit') || headers.get('ratelimit-limit');
+    const remaining = headers.get('x-ratelimit-remaining') || headers.get('ratelimit-remaining');
+    const reset = headers.get('x-ratelimit-reset') || headers.get('ratelimit-reset');
+    if (limit)
+        info.limit = parseInt(limit, 10);
+    if (remaining)
+        info.remaining = parseInt(remaining, 10);
+    if (reset) {
+        const resetValue = parseInt(reset, 10);
+        if (resetValue > 1000000000) {
+            info.reset = new Date(resetValue * 1000);
+        }
+        else {
+            info.reset = new Date(Date.now() + resetValue * 1000);
+        }
+    }
+    const retryAfter = headers.get('retry-after');
+    if (retryAfter) {
+        const seconds = parseInt(retryAfter, 10);
+        if (!isNaN(seconds)) {
+            info.retryAfter = seconds;
+        }
+        else {
+            const retryDate = new Date(retryAfter);
+            if (!isNaN(retryDate.getTime())) {
+                info.retryAfter = Math.ceil((retryDate.getTime() - Date.now()) / 1000);
+            }
+        }
+    }
+    const policy = headers.get('ratelimit-policy');
+    if (policy) {
+        info.policy = policy;
+    }
+    if (info.remaining !== undefined && info.remaining === 0) {
+        info.limited = true;
+    }
+    return info;
+}
+export function parseCompressionInfo(headers) {
+    const info = {};
+    const encoding = headers.get('content-encoding');
+    if (encoding) {
+        info.encoding = encoding.toLowerCase();
+    }
+    const xOriginalSize = headers.get('x-original-size') || headers.get('x-uncompressed-size');
+    if (xOriginalSize) {
+        info.originalSize = parseInt(xOriginalSize, 10);
+    }
+    const contentLength = headers.get('content-length');
+    if (contentLength) {
+        info.compressedSize = parseInt(contentLength, 10);
+    }
+    if (info.originalSize && info.compressedSize) {
+        info.ratio = info.originalSize / info.compressedSize;
+    }
+    return info;
+}
+export function parseCSPInfo(headers) {
+    const info = {
+        directives: {},
+        reportOnly: false
+    };
+    let cspHeader = headers.get('content-security-policy');
+    if (!cspHeader) {
+        cspHeader = headers.get('content-security-policy-report-only');
+        if (cspHeader) {
+            info.reportOnly = true;
+        }
+    }
+    if (cspHeader) {
+        info.policy = cspHeader;
+        const directives = cspHeader.split(';').map(d => d.trim()).filter(Boolean);
+        for (const directive of directives) {
+            const [name, ...values] = directive.split(/\s+/);
+            if (name) {
+                info.directives[name] = values;
+            }
+        }
+    }
+    return info;
+}
+export function parseContentType(headers) {
+    const info = {};
+    const contentType = headers.get('content-type');
+    if (!contentType)
+        return info;
+    const parts = contentType.split(';').map(p => p.trim());
+    if (parts[0]) {
+        info.mediaType = parts[0];
+        const [type, subtype] = parts[0].split('/');
+        if (type)
+            info.type = type;
+        if (subtype)
+            info.subtype = subtype;
+    }
+    for (let i = 1; i < parts.length; i++) {
+        const param = parts[i];
+        const [key, value] = param.split('=').map(s => s.trim());
+        if (key === 'charset' && value) {
+            info.charset = value.replace(/['"]/g, '');
+        }
+        else if (key === 'boundary' && value) {
+            info.boundary = value.replace(/['"]/g, '');
+        }
+    }
+    return info;
+}
+export function parseAcceptInfo(headers) {
+    const info = {
+        types: [],
+        encodings: [],
+        languages: []
+    };
+    const accept = headers.get('accept');
+    if (accept) {
+        const types = accept.split(',').map(t => t.trim());
+        for (const typeStr of types) {
+            const [mediaType, ...params] = typeStr.split(';').map(s => s.trim());
+            let q = 1.0;
+            for (const param of params) {
+                if (param.startsWith('q=')) {
+                    q = parseFloat(param.substring(2)) || 1.0;
+                }
+            }
+            const [type, subtype] = mediaType.split('/');
+            info.types.push({
+                mediaType,
+                q,
+                type: type || undefined,
+                subtype: subtype || undefined
+            });
+        }
+        info.types.sort((a, b) => b.q - a.q);
+    }
+    const acceptEncoding = headers.get('accept-encoding');
+    if (acceptEncoding) {
+        const encodings = acceptEncoding.split(',').map(e => e.trim());
+        for (const encodingStr of encodings) {
+            const [encoding, ...params] = encodingStr.split(';').map(s => s.trim());
+            let q = 1.0;
+            for (const param of params) {
+                if (param.startsWith('q=')) {
+                    q = parseFloat(param.substring(2)) || 1.0;
+                }
+            }
+            info.encodings.push({ encoding, q });
+        }
+        info.encodings.sort((a, b) => b.q - a.q);
+    }
+    const acceptLanguage = headers.get('accept-language');
+    if (acceptLanguage) {
+        const languages = acceptLanguage.split(',').map(l => l.trim());
+        for (const languageStr of languages) {
+            const [language, ...params] = languageStr.split(';').map(s => s.trim());
+            let q = 1.0;
+            for (const param of params) {
+                if (param.startsWith('q=')) {
+                    q = parseFloat(param.substring(2)) || 1.0;
+                }
+            }
+            info.languages.push({ language, q });
+        }
+        info.languages.sort((a, b) => b.q - a.q);
+    }
+    return info;
+}
+export function parseAuthInfo(headers) {
+    const info = { methods: [] };
+    const wwwAuth = headers.get('www-authenticate');
+    if (wwwAuth) {
+        if (wwwAuth.toLowerCase().includes('basic'))
+            info.methods.push('Basic');
+        if (wwwAuth.toLowerCase().includes('bearer'))
+            info.methods.push('Bearer');
+        if (wwwAuth.toLowerCase().includes('digest'))
+            info.methods.push('Digest');
+        if (wwwAuth.toLowerCase().includes('negotiate'))
+            info.methods.push('Negotiate');
+        if (wwwAuth.toLowerCase().includes('aws4-hmac-sha256'))
+            info.methods.push('AWS4');
+        const realmMatch = wwwAuth.match(/realm="([^"]+)"/);
+        if (realmMatch)
+            info.realm = realmMatch[1];
+        const errorMatch = wwwAuth.match(/error="([^"]+)"/);
+        if (errorMatch)
+            info.error = errorMatch[1];
+        const descMatch = wwwAuth.match(/error_description="([^"]+)"/);
+        if (descMatch)
+            info.errorDescription = descMatch[1];
+    }
+    const xAuthError = headers.get('x-auth-error') || headers.get('x-authentication-error');
+    if (xAuthError) {
+        if (!info.error)
+            info.error = xAuthError;
+    }
+    return info;
+}
+export function parseClockSkew(headers) {
+    const dateHeader = headers.get('date');
+    if (!dateHeader)
+        return {};
+    const serverTime = new Date(dateHeader);
+    if (isNaN(serverTime.getTime()))
+        return {};
+    const skewMs = serverTime.getTime() - Date.now();
+    return { serverTime, skewMs };
+}
+export function parseHeaders(headers, status) {
+    return {
+        cache: parseCacheInfo(headers),
+        platform: parsePlatformInfo(headers),
+        rateLimit: parseRateLimitInfo(headers, status),
+        compression: parseCompressionInfo(headers),
+        csp: parseCSPInfo(headers),
+        contentType: parseContentType(headers),
+        accept: parseAcceptInfo(headers),
+        auth: parseAuthInfo(headers),
+        clockSkew: parseClockSkew(headers)
+    };
+}

package/dist/browser/utils/html-cleaner.d.ts

@@ -0,0 +1 @@
+export declare function cleanHtml(html: string): string;

package/dist/browser/utils/html-cleaner.js

@@ -0,0 +1,21 @@
+export function cleanHtml(html) {
+    if (!html)
+        return '';
+    let text = html;
+    text = text.replace(/<script\b[^>]*>[\s\S]*?<\/script>/gi, '');
+    text = text.replace(/<style\b[^>]*>[\s\S]*?<\/style>/gi, '');
+    text = text.replace(/<!--[\s\S]*?-->/g, '');
+    text = text.replace(/<\/(div|p|h[1-6]|li|ul|ol|tr|table|section|article|main|header|footer|nav)>/gi, '\n');
+    text = text.replace(/<(br|hr)\s*\/?>/gi, '\n');
+    text = text.replace(/<[^>]+>/g, ' ');
+    text = text
+        .replace(/&nbsp;/g, ' ')
+        .replace(/&amp;/g, '&')
+        .replace(/&lt;/g, '<')
+        .replace(/&gt;/g, '>')
+        .replace(/&quot;/g, '"')
+        .replace(/&#39;/g, "'");
+    text = text.replace(/[ \t]+/g, ' ');
+    text = text.replace(/\n\s*\n\s*\n+/g, '\n\n');
+    return text.trim();
+}