recker 1.0.27 → 1.0.28-next.3bf98c7

package/dist/scrape/spider.js

@@ -0,0 +1,250 @@
+import { createClient } from '../core/client.js';
+import { ScrapeDocument } from './document.js';
+import { RequestPool } from '../utils/request-pool.js';
+const TRACKING_PARAMS = new Set([
+    'utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content',
+    'gclid', 'gclsrc', 'dclid',
+    'fbclid', 'fb_action_ids', 'fb_action_types', 'fb_source', 'fb_ref',
+    'msclkid',
+    'twclid',
+    'ref', 'referer', 'referrer', 'source',
+    '_ga', '_gl', '_hsenc', '_hsmi',
+    'mc_cid', 'mc_eid',
+    'yclid', 'ymclid',
+    'igshid',
+    '_t', 't', 'timestamp', 'ts', 'nocache', 'cache',
+]);
+function normalizeUrl(urlStr) {
+    try {
+        const url = new URL(urlStr);
+        url.hash = '';
+        const paramsToDelete = [];
+        url.searchParams.forEach((_, key) => {
+            if (TRACKING_PARAMS.has(key.toLowerCase())) {
+                paramsToDelete.push(key);
+            }
+        });
+        paramsToDelete.forEach(key => url.searchParams.delete(key));
+        url.searchParams.sort();
+        if (url.pathname !== '/' && url.pathname.endsWith('/')) {
+            url.pathname = url.pathname.slice(0, -1);
+        }
+        return url.toString();
+    }
+    catch {
+        return urlStr;
+    }
+}
+function shouldCrawl(url, baseHost, options) {
+    try {
+        const parsed = new URL(url);
+        if (!['http:', 'https:'].includes(parsed.protocol)) {
+            return false;
+        }
+        if (options.sameDomain !== false && parsed.hostname !== baseHost) {
+            return false;
+        }
+        const skipExtensions = [
+            '.jpg', '.jpeg', '.png', '.gif', '.webp', '.svg', '.ico',
+            '.pdf', '.zip', '.tar', '.gz', '.rar',
+            '.mp3', '.mp4', '.avi', '.mov', '.webm',
+            '.css', '.js', '.json', '.xml', '.rss',
+            '.woff', '.woff2', '.ttf', '.eot',
+        ];
+        const pathname = parsed.pathname.toLowerCase();
+        if (skipExtensions.some(ext => pathname.endsWith(ext))) {
+            return false;
+        }
+        if (options.exclude?.some(pattern => pattern.test(url))) {
+            return false;
+        }
+        if (options.include?.length) {
+            if (!options.include.some(pattern => pattern.test(url))) {
+                return false;
+            }
+        }
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+function sleep(ms) {
+    return new Promise(resolve => setTimeout(resolve, ms));
+}
+export class Spider {
+    options;
+    client;
+    pool;
+    visited = new Set();
+    queue = [];
+    results = [];
+    errors = [];
+    baseHost = '';
+    running = false;
+    aborted = false;
+    pendingCount = 0;
+    constructor(options = {}) {
+        this.options = {
+            maxDepth: options.maxDepth ?? 4,
+            maxPages: options.maxPages ?? 100,
+            sameDomain: options.sameDomain ?? true,
+            concurrency: options.concurrency ?? 5,
+            timeout: options.timeout ?? 10000,
+            delay: options.delay ?? 100,
+            userAgent: options.userAgent ?? 'Recker Spider/1.0',
+            respectRobotsTxt: options.respectRobotsTxt ?? true,
+            exclude: options.exclude,
+            include: options.include,
+            onPage: options.onPage,
+            onProgress: options.onProgress,
+        };
+        this.client = createClient({
+            baseUrl: 'http://localhost',
+            timeout: this.options.timeout,
+            headers: {
+                'User-Agent': this.options.userAgent,
+            },
+        });
+        this.pool = new RequestPool({
+            concurrency: this.options.concurrency,
+            ...(this.options.delay > 0 ? {
+                requestsPerInterval: 1,
+                interval: this.options.delay,
+            } : {}),
+        });
+    }
+    async crawl(startUrl) {
+        const startTime = performance.now();
+        const normalizedStart = normalizeUrl(startUrl);
+        this.baseHost = new URL(normalizedStart).hostname;
+        this.visited.clear();
+        this.queue = [];
+        this.results = [];
+        this.errors = [];
+        this.running = true;
+        this.aborted = false;
+        this.pendingCount = 0;
+        const pending = new Map();
+        const scheduleUrl = (item) => {
+            const normalized = normalizeUrl(item.url);
+            if (this.visited.has(normalized))
+                return;
+            if (pending.has(normalized))
+                return;
+            if (item.depth > this.options.maxDepth)
+                return;
+            if (this.results.length + pending.size >= this.options.maxPages)
+                return;
+            this.visited.add(normalized);
+            this.pendingCount++;
+            const promise = this.pool.run(() => this.crawlPage({ ...item, url: normalized }))
+                .finally(() => {
+                pending.delete(normalized);
+                this.pendingCount--;
+            });
+            pending.set(normalized, promise);
+        };
+        scheduleUrl({ url: normalizedStart, depth: 0 });
+        while ((pending.size > 0 || this.queue.length > 0) && !this.aborted) {
+            while (this.queue.length > 0 && !this.aborted) {
+                const item = this.queue.shift();
+                if (this.results.length + pending.size >= this.options.maxPages)
+                    break;
+                scheduleUrl(item);
+            }
+            if (pending.size > 0) {
+                await Promise.race(pending.values());
+            }
+        }
+        if (pending.size > 0) {
+            await Promise.all(pending.values());
+        }
+        this.running = false;
+        return {
+            startUrl: normalizedStart,
+            pages: this.results,
+            visited: this.visited,
+            duration: Math.round(performance.now() - startTime),
+            errors: this.errors,
+        };
+    }
+    async crawlPage(item) {
+        const startTime = performance.now();
+        this.options.onProgress?.({
+            crawled: this.results.length,
+            queued: this.queue.length,
+            total: this.visited.size,
+            currentUrl: item.url,
+            depth: item.depth,
+        });
+        try {
+            const response = await this.client.get(item.url);
+            const status = response.status;
+            const contentType = response.headers.get('content-type') || '';
+            if (!contentType.includes('text/html')) {
+                return;
+            }
+            const html = await response.text();
+            const doc = await ScrapeDocument.create(html, { baseUrl: item.url });
+            const title = doc.selectFirst('title').text() || '';
+            const links = doc.links({ absolute: true });
+            const result = {
+                url: item.url,
+                status,
+                title,
+                depth: item.depth,
+                links,
+                duration: Math.round(performance.now() - startTime),
+            };
+            this.results.push(result);
+            this.options.onPage?.(result);
+            for (const link of links) {
+                if (!link.href)
+                    continue;
+                const normalized = normalizeUrl(link.href);
+                if (this.visited.has(normalized))
+                    continue;
+                if (!shouldCrawl(normalized, this.baseHost, this.options))
+                    continue;
+                this.queue.push({
+                    url: normalized,
+                    depth: item.depth + 1,
+                });
+            }
+        }
+        catch (error) {
+            const errorResult = {
+                url: item.url,
+                status: 0,
+                title: '',
+                depth: item.depth,
+                links: [],
+                duration: Math.round(performance.now() - startTime),
+                error: error.message,
+            };
+            this.results.push(errorResult);
+            this.errors.push({ url: item.url, error: error.message });
+            this.options.onPage?.(errorResult);
+        }
+    }
+    abort() {
+        this.aborted = true;
+    }
+    isRunning() {
+        return this.running;
+    }
+    getProgress() {
+        return {
+            crawled: this.results.length,
+            queued: this.queue.length,
+            total: this.visited.size,
+            currentUrl: '',
+            depth: 0,
+        };
+    }
+}
+export async function spider(url, options) {
+    const s = new Spider(options);
+    return s.crawl(url);
+}

package/dist/scrape/types.d.ts

@@ -14,13 +14,14 @@ export interface ExtractedImage {
     height?: number;
     srcset?: string;
     loading?: 'lazy' | 'eager';
+    decoding?: 'async' | 'auto' | 'sync';
 }
 export interface ExtractedMeta {
     title?: string;
     description?: string;
     keywords?: string[];
     author?: string;
-    robots?: string;
+    robots?: string[];
     canonical?: string;
     viewport?: string;
     charset?: string;

package/dist/seo/analyzer.d.ts

@@ -0,0 +1,42 @@
+import type { CheerioAPI } from 'cheerio';
+import type { SeoReport, SeoAnalyzerOptions } from './types.js';
+import { type RulesEngineOptions } from './rules/index.js';
+export interface SeoAnalyzerFullOptions extends SeoAnalyzerOptions {
+    rules?: RulesEngineOptions;
+}
+export declare class SeoAnalyzer {
+    private $;
+    private options;
+    private rulesEngine;
+    constructor($: CheerioAPI, options?: SeoAnalyzerFullOptions);
+    static fromHtml(html: string, options?: SeoAnalyzerFullOptions): Promise<SeoAnalyzer>;
+    analyze(): SeoReport;
+    private buildRuleContext;
+    private analyzeUrlQuality;
+    private analyzeJsRendering;
+    private checkMixedContent;
+    private analyzeLinkSecurity;
+    private detectFavicon;
+    private analyzePerformanceHints;
+    private analyzeCWVHints;
+    private analyzeAccessibility;
+    private analyzeStructuralHtml;
+    private analyzeBreadcrumbs;
+    private analyzeMultimedia;
+    private analyzeTrustSignals;
+    private calculateTextHtmlRatio;
+    private convertToCheckResults;
+    private analyzeHeadings;
+    private analyzeContent;
+    private buildLinkAnalysis;
+    private buildImageAnalysis;
+    private buildSocialAnalysis;
+    private buildTechnicalAnalysis;
+    private calculateScore;
+    getRules(): import("./rules/types.js").SeoRule[];
+    getRulesByCategory(category: string): import("./rules/types.js").SeoRule[];
+    getCategories(): import("./rules/types.js").RuleCategory[];
+}
+export declare function analyzeSeo(html: string, options?: SeoAnalyzerFullOptions): Promise<SeoReport>;
+export { SEO_THRESHOLDS, createRulesEngine, SeoRulesEngine } from './rules/index.js';
+export type { RuleContext, RuleResult, RulesEngineOptions, RuleCategory, RuleSeverity, SeoRule } from './rules/index.js';