recker 1.0.28 → 1.0.29

package/dist/scrape/spider.js

@@ -0,0 +1,250 @@
+import { createClient } from '../core/client.js';
+import { ScrapeDocument } from './document.js';
+import { RequestPool } from '../utils/request-pool.js';
+const TRACKING_PARAMS = new Set([
+    'utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content',
+    'gclid', 'gclsrc', 'dclid',
+    'fbclid', 'fb_action_ids', 'fb_action_types', 'fb_source', 'fb_ref',
+    'msclkid',
+    'twclid',
+    'ref', 'referer', 'referrer', 'source',
+    '_ga', '_gl', '_hsenc', '_hsmi',
+    'mc_cid', 'mc_eid',
+    'yclid', 'ymclid',
+    'igshid',
+    '_t', 't', 'timestamp', 'ts', 'nocache', 'cache',
+]);
+function normalizeUrl(urlStr) {
+    try {
+        const url = new URL(urlStr);
+        url.hash = '';
+        const paramsToDelete = [];
+        url.searchParams.forEach((_, key) => {
+            if (TRACKING_PARAMS.has(key.toLowerCase())) {
+                paramsToDelete.push(key);
+            }
+        });
+        paramsToDelete.forEach(key => url.searchParams.delete(key));
+        url.searchParams.sort();
+        if (url.pathname !== '/' && url.pathname.endsWith('/')) {
+            url.pathname = url.pathname.slice(0, -1);
+        }
+        return url.toString();
+    }
+    catch {
+        return urlStr;
+    }
+}
+function shouldCrawl(url, baseHost, options) {
+    try {
+        const parsed = new URL(url);
+        if (!['http:', 'https:'].includes(parsed.protocol)) {
+            return false;
+        }
+        if (options.sameDomain !== false && parsed.hostname !== baseHost) {
+            return false;
+        }
+        const skipExtensions = [
+            '.jpg', '.jpeg', '.png', '.gif', '.webp', '.svg', '.ico',
+            '.pdf', '.zip', '.tar', '.gz', '.rar',
+            '.mp3', '.mp4', '.avi', '.mov', '.webm',
+            '.css', '.js', '.json', '.xml', '.rss',
+            '.woff', '.woff2', '.ttf', '.eot',
+        ];
+        const pathname = parsed.pathname.toLowerCase();
+        if (skipExtensions.some(ext => pathname.endsWith(ext))) {
+            return false;
+        }
+        if (options.exclude?.some(pattern => pattern.test(url))) {
+            return false;
+        }
+        if (options.include?.length) {
+            if (!options.include.some(pattern => pattern.test(url))) {
+                return false;
+            }
+        }
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+function sleep(ms) {
+    return new Promise(resolve => setTimeout(resolve, ms));
+}
+export class Spider {
+    options;
+    client;
+    pool;
+    visited = new Set();
+    queue = [];
+    results = [];
+    errors = [];
+    baseHost = '';
+    running = false;
+    aborted = false;
+    pendingCount = 0;
+    constructor(options = {}) {
+        this.options = {
+            maxDepth: options.maxDepth ?? 4,
+            maxPages: options.maxPages ?? 100,
+            sameDomain: options.sameDomain ?? true,
+            concurrency: options.concurrency ?? 5,
+            timeout: options.timeout ?? 10000,
+            delay: options.delay ?? 100,
+            userAgent: options.userAgent ?? 'Recker Spider/1.0',
+            respectRobotsTxt: options.respectRobotsTxt ?? true,
+            exclude: options.exclude,
+            include: options.include,
+            onPage: options.onPage,
+            onProgress: options.onProgress,
+        };
+        this.client = createClient({
+            baseUrl: 'http://localhost',
+            timeout: this.options.timeout,
+            headers: {
+                'User-Agent': this.options.userAgent,
+            },
+        });
+        this.pool = new RequestPool({
+            concurrency: this.options.concurrency,
+            ...(this.options.delay > 0 ? {
+                requestsPerInterval: 1,
+                interval: this.options.delay,
+            } : {}),
+        });
+    }
+    async crawl(startUrl) {
+        const startTime = performance.now();
+        const normalizedStart = normalizeUrl(startUrl);
+        this.baseHost = new URL(normalizedStart).hostname;
+        this.visited.clear();
+        this.queue = [];
+        this.results = [];
+        this.errors = [];
+        this.running = true;
+        this.aborted = false;
+        this.pendingCount = 0;
+        const pending = new Map();
+        const scheduleUrl = (item) => {
+            const normalized = normalizeUrl(item.url);
+            if (this.visited.has(normalized))
+                return;
+            if (pending.has(normalized))
+                return;
+            if (item.depth > this.options.maxDepth)
+                return;
+            if (this.results.length + pending.size >= this.options.maxPages)
+                return;
+            this.visited.add(normalized);
+            this.pendingCount++;
+            const promise = this.pool.run(() => this.crawlPage({ ...item, url: normalized }))
+                .finally(() => {
+                pending.delete(normalized);
+                this.pendingCount--;
+            });
+            pending.set(normalized, promise);
+        };
+        scheduleUrl({ url: normalizedStart, depth: 0 });
+        while ((pending.size > 0 || this.queue.length > 0) && !this.aborted) {
+            while (this.queue.length > 0 && !this.aborted) {
+                const item = this.queue.shift();
+                if (this.results.length + pending.size >= this.options.maxPages)
+                    break;
+                scheduleUrl(item);
+            }
+            if (pending.size > 0) {
+                await Promise.race(pending.values());
+            }
+        }
+        if (pending.size > 0) {
+            await Promise.all(pending.values());
+        }
+        this.running = false;
+        return {
+            startUrl: normalizedStart,
+            pages: this.results,
+            visited: this.visited,
+            duration: Math.round(performance.now() - startTime),
+            errors: this.errors,
+        };
+    }
+    async crawlPage(item) {
+        const startTime = performance.now();
+        this.options.onProgress?.({
+            crawled: this.results.length,
+            queued: this.queue.length,
+            total: this.visited.size,
+            currentUrl: item.url,
+            depth: item.depth,
+        });
+        try {
+            const response = await this.client.get(item.url);
+            const status = response.status;
+            const contentType = response.headers.get('content-type') || '';
+            if (!contentType.includes('text/html')) {
+                return;
+            }
+            const html = await response.text();
+            const doc = await ScrapeDocument.create(html, { baseUrl: item.url });
+            const title = doc.selectFirst('title').text() || '';
+            const links = doc.links({ absolute: true });
+            const result = {
+                url: item.url,
+                status,
+                title,
+                depth: item.depth,
+                links,
+                duration: Math.round(performance.now() - startTime),
+            };
+            this.results.push(result);
+            this.options.onPage?.(result);
+            for (const link of links) {
+                if (!link.href)
+                    continue;
+                const normalized = normalizeUrl(link.href);
+                if (this.visited.has(normalized))
+                    continue;
+                if (!shouldCrawl(normalized, this.baseHost, this.options))
+                    continue;
+                this.queue.push({
+                    url: normalized,
+                    depth: item.depth + 1,
+                });
+            }
+        }
+        catch (error) {
+            const errorResult = {
+                url: item.url,
+                status: 0,
+                title: '',
+                depth: item.depth,
+                links: [],
+                duration: Math.round(performance.now() - startTime),
+                error: error.message,
+            };
+            this.results.push(errorResult);
+            this.errors.push({ url: item.url, error: error.message });
+            this.options.onPage?.(errorResult);
+        }
+    }
+    abort() {
+        this.aborted = true;
+    }
+    isRunning() {
+        return this.running;
+    }
+    getProgress() {
+        return {
+            crawled: this.results.length,
+            queued: this.queue.length,
+            total: this.visited.size,
+            currentUrl: '',
+            depth: 0,
+        };
+    }
+}
+export async function spider(url, options) {
+    const s = new Spider(options);
+    return s.crawl(url);
+}

package/dist/seo/analyzer.js

@@ -57,6 +57,21 @@ export class SeoAnalyzer {
             checks,
             title: meta.title ? { text: meta.title, length: meta.title.length } : undefined,
             metaDescription: meta.description ? { text: meta.description, length: meta.description.length } : undefined,
+            openGraph: Object.keys(og).length > 0 ? {
+                title: og.title,
+                description: og.description,
+                image: Array.isArray(og.image) ? og.image[0] : og.image,
+                url: og.url,
+                type: og.type,
+                siteName: og.siteName,
+            } : undefined,
+            twitterCard: Object.keys(twitter).length > 0 ? {
+                card: twitter.card,
+                title: twitter.title,
+                description: twitter.description,
+                image: Array.isArray(twitter.image) ? twitter.image[0] : twitter.image,
+                site: twitter.site,
+            } : undefined,
             headings: headings,
             content,
             links: linkAnalysis,
@@ -72,6 +87,16 @@ export class SeoAnalyzer {
     buildRuleContext(data) {
         const { meta, og, twitter, jsonLd, headings, content, linkAnalysis, imageAnalysis, links } = data;
         const htmlLang = this.$('html').attr('lang');
+        const hreflangTags = [];
+        this.$('link[rel="alternate"][hreflang]').each((_, el) => {
+            const $el = this.$(el);
+            const lang = $el.attr('hreflang');
+            const href = $el.attr('href');
+            if (lang && href) {
+                hreflangTags.push({ lang, href });
+            }
+        });
+        const ogLocale = this.$('meta[property="og:locale"]').attr('content');
         const genericTexts = SEO_THRESHOLDS.links.genericTexts;
         const genericTextLinks = links.filter((l) => {
             const text = l.text?.toLowerCase().trim();
@@ -196,6 +221,8 @@ export class SeoAnalyzer {
             titleMatchesH1: meta.title && h1Text ? meta.title.toLowerCase().trim() === h1Text.toLowerCase().trim() : undefined,
             ...this.analyzeUrlQuality(),
             ...this.analyzeJsRendering(content),
+            hreflangTags: hreflangTags.length > 0 ? hreflangTags : undefined,
+            ogLocale,
         };
     }
     analyzeUrlQuality() {

package/dist/seo/index.d.ts

@@ -1,5 +1,7 @@
 export { SeoAnalyzer, analyzeSeo } from './analyzer.js';
+export { SeoSpider, seoSpider } from './seo-spider.js';
+export type { SeoSpiderOptions, SeoPageResult, SiteWideIssue, SeoSpiderResult, } from './seo-spider.js';
 export { SeoRulesEngine, createRulesEngine, SEO_THRESHOLDS, ALL_SEO_RULES, } from './rules/index.js';
-export type { SeoReport, SeoCheckResult, SeoStatus, HeadingAnalysis, HeadingInfo, ContentMetrics, LinkAnalysis, ImageAnalysis, SocialMetaAnalysis, TechnicalSeo, SeoAnalyzerOptions, } from './types.js';
+export type { SeoReport, SeoCheckResult, SeoStatus, SeoTiming, HeadingAnalysis, HeadingInfo, ContentMetrics, LinkAnalysis, ImageAnalysis, SocialMetaAnalysis, TechnicalSeo, SeoAnalyzerOptions, } from './types.js';
 export type { SeoRule, RuleContext, RuleResult, RuleEvidence, RuleCategory, RuleSeverity, RulesEngineOptions, } from './rules/index.js';
 export type { SeoAnalyzerFullOptions } from './analyzer.js';

package/dist/seo/index.js

@@ -1,2 +1,3 @@
 export { SeoAnalyzer, analyzeSeo } from './analyzer.js';
+export { SeoSpider, seoSpider } from './seo-spider.js';
 export { SeoRulesEngine, createRulesEngine, SEO_THRESHOLDS, ALL_SEO_RULES, } from './rules/index.js';