recker 1.0.33-next.fbc1682 → 1.0.34

package/dist/scrape/spider.d.ts

@@ -1,4 +1,5 @@
 import type { ExtractedLink } from './types.js';
+import { type SitemapUrl } from '../seo/validators/sitemap.js';
 export interface SpiderOptions {
     maxDepth?: number;
     maxPages?: number;
@@ -10,6 +11,8 @@ export interface SpiderOptions {
     include?: RegExp[];
     userAgent?: string;
     respectRobotsTxt?: boolean;
+    useSitemap?: boolean;
+    sitemapUrl?: string;
     onPage?: (result: SpiderPageResult) => void;
     onProgress?: (progress: SpiderProgress) => void;
 }
@@ -29,6 +32,29 @@ export interface SpiderProgress {
     currentUrl: string;
     depth: number;
 }
+export interface SitemapAnalysis {
+    found: boolean;
+    url?: string;
+    totalUrls: number;
+    crawledFromSitemap: number;
+    orphanUrls: string[];
+    missingFromSitemap: string[];
+    blockedBySitemapRobots: string[];
+    validationIssues: Array<{
+        type: string;
+        message: string;
+    }>;
+    sitemapUrls: SitemapUrl[];
+}
+export interface RobotsAnalysis {
+    found: boolean;
+    sitemaps: string[];
+    blocksAll: boolean;
+    issues: Array<{
+        type: string;
+        message: string;
+    }>;
+}
 export interface SpiderResult {
     startUrl: string;
     pages: SpiderPageResult[];
@@ -38,6 +64,8 @@ export interface SpiderResult {
         url: string;
         error: string;
     }>;
+    sitemap?: SitemapAnalysis;
+    robots?: RobotsAnalysis;
 }
 export declare class Spider {
     private options;
@@ -51,8 +79,17 @@ export declare class Spider {
     private running;
     private aborted;
     private pendingCount;
+    private sitemapUrls;
+    private sitemapUrlSet;
+    private robotsData;
+    private sitemapValidation;
+    private robotsValidation;
     constructor(options?: SpiderOptions);
     crawl(startUrl: string): Promise<SpiderResult>;
+    private fetchRobotsTxt;
+    private fetchSitemaps;
+    private buildSitemapAnalysis;
+    private buildRobotsAnalysis;
     private crawlPage;
     abort(): void;
     isRunning(): boolean;

package/dist/scrape/spider.js

@@ -1,6 +1,8 @@
 import { createClient } from '../core/client.js';
 import { ScrapeDocument } from './document.js';
 import { RequestPool } from '../utils/request-pool.js';
+import { discoverSitemaps, fetchAndValidateSitemap, } from '../seo/validators/sitemap.js';
+import { fetchAndValidateRobotsTxt, isPathAllowed, } from '../seo/validators/robots.js';
 const TRACKING_PARAMS = new Set([
     'utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content',
     'gclid', 'gclsrc', 'dclid',
@@ -84,6 +86,11 @@ export class Spider {
     running = false;
     aborted = false;
     pendingCount = 0;
+    sitemapUrls = [];
+    sitemapUrlSet = new Set();
+    robotsData = null;
+    sitemapValidation = null;
+    robotsValidation = null;
     constructor(options = {}) {
         this.options = {
             maxDepth: options.maxDepth ?? 5,
@@ -94,6 +101,8 @@ export class Spider {
             delay: options.delay ?? 100,
             userAgent: options.userAgent ?? 'Recker Spider/1.0',
             respectRobotsTxt: options.respectRobotsTxt ?? true,
+            useSitemap: options.useSitemap ?? false,
+            sitemapUrl: options.sitemapUrl,
             exclude: options.exclude,
             include: options.include,
             onPage: options.onPage,
@@ -117,6 +126,7 @@ export class Spider {
     async crawl(startUrl) {
         const startTime = performance.now();
         const normalizedStart = normalizeUrl(startUrl);
+        const baseUrl = new URL(normalizedStart).origin;
         this.baseHost = new URL(normalizedStart).hostname;
         this.visited.clear();
         this.queue = [];
@@ -125,8 +135,19 @@ export class Spider {
         this.running = true;
         this.aborted = false;
         this.pendingCount = 0;
+        this.sitemapUrls = [];
+        this.sitemapUrlSet.clear();
+        this.robotsData = null;
+        this.sitemapValidation = null;
+        this.robotsValidation = null;
+        if (this.options.respectRobotsTxt || this.options.useSitemap) {
+            await this.fetchRobotsTxt(baseUrl);
+        }
+        if (this.options.useSitemap) {
+            await this.fetchSitemaps(baseUrl);
+        }
         const pending = new Map();
-        const scheduleUrl = (item) => {
+        const scheduleUrl = (item, fromSitemap = false) => {
             const normalized = normalizeUrl(item.url);
             if (this.visited.has(normalized))
                 return;
@@ -136,6 +157,17 @@ export class Spider {
                 return;
             if (this.results.length + pending.size >= this.options.maxPages)
                 return;
+            if (this.options.respectRobotsTxt && this.robotsData) {
+                try {
+                    const urlPath = new URL(normalized).pathname;
+                    if (!isPathAllowed(this.robotsData, urlPath, this.options.userAgent)) {
+                        return;
+                    }
+                }
+                catch {
+                    return;
+                }
+            }
             this.visited.add(normalized);
             this.pendingCount++;
             const promise = this.pool.run(() => this.crawlPage({ ...item, url: normalized }))
@@ -146,6 +178,18 @@ export class Spider {
             pending.set(normalized, promise);
         };
         scheduleUrl({ url: normalizedStart, depth: 0 });
+        if (this.options.useSitemap && this.sitemapUrls.length > 0) {
+            for (const sitemapUrl of this.sitemapUrls) {
+                try {
+                    const urlHost = new URL(sitemapUrl.loc).hostname;
+                    if (urlHost === this.baseHost) {
+                        scheduleUrl({ url: sitemapUrl.loc, depth: 1 }, true);
+                    }
+                }
+                catch {
+                }
+            }
+        }
         while ((pending.size > 0 || this.queue.length > 0) && !this.aborted) {
             while (this.queue.length > 0 && !this.aborted) {
                 const item = this.queue.shift();
@@ -161,12 +205,154 @@ export class Spider {
             await Promise.all(pending.values());
         }
         this.running = false;
+        const sitemapAnalysis = this.buildSitemapAnalysis();
+        const robotsAnalysis = this.buildRobotsAnalysis();
         return {
             startUrl: normalizedStart,
             pages: this.results,
             visited: this.visited,
             duration: Math.round(performance.now() - startTime),
             errors: this.errors,
+            sitemap: this.options.useSitemap ? sitemapAnalysis : undefined,
+            robots: robotsAnalysis,
+        };
+    }
+    async fetchRobotsTxt(baseUrl) {
+        try {
+            const fetcher = async (url) => {
+                const response = await this.client.get(url);
+                return {
+                    status: response.status,
+                    text: await response.text(),
+                };
+            };
+            const result = await fetchAndValidateRobotsTxt(baseUrl, fetcher);
+            if (result.exists) {
+                this.robotsData = result.parseResult;
+                this.robotsValidation = {
+                    found: true,
+                    issues: result.issues.map(i => ({ type: i.type, message: i.message })),
+                };
+            }
+            else {
+                this.robotsValidation = {
+                    found: false,
+                    issues: [{ type: 'info', message: 'robots.txt not found' }],
+                };
+            }
+        }
+        catch (error) {
+            this.robotsValidation = {
+                found: false,
+                issues: [{ type: 'error', message: `Failed to fetch robots.txt: ${error instanceof Error ? error.message : 'Unknown error'}` }],
+            };
+        }
+    }
+    async fetchSitemaps(baseUrl) {
+        const fetcher = async (url) => {
+            const response = await this.client.get(url);
+            return {
+                status: response.status,
+                text: await response.text(),
+                headers: Object.fromEntries([...response.headers.entries()]),
+            };
+        };
+        try {
+            let sitemapUrls = [];
+            if (this.options.sitemapUrl) {
+                sitemapUrls = [this.options.sitemapUrl];
+            }
+            else if (this.robotsData?.sitemaps.length) {
+                sitemapUrls = this.robotsData.sitemaps;
+            }
+            else {
+                sitemapUrls = await discoverSitemaps(baseUrl, undefined, fetcher);
+            }
+            for (const sitemapUrl of sitemapUrls) {
+                try {
+                    const result = await fetchAndValidateSitemap(sitemapUrl, fetcher);
+                    if (result.exists && result.parseResult.valid) {
+                        this.sitemapValidation = result;
+                        if (result.parseResult.type === 'sitemapindex') {
+                            for (const childSitemap of result.parseResult.sitemaps) {
+                                try {
+                                    const childResult = await fetchAndValidateSitemap(childSitemap.loc, fetcher);
+                                    if (childResult.exists && childResult.parseResult.urls.length > 0) {
+                                        this.sitemapUrls.push(...childResult.parseResult.urls);
+                                    }
+                                }
+                                catch {
+                                }
+                            }
+                        }
+                        else {
+                            this.sitemapUrls.push(...result.parseResult.urls);
+                        }
+                    }
+                }
+                catch {
+                }
+            }
+            for (const url of this.sitemapUrls) {
+                this.sitemapUrlSet.add(normalizeUrl(url.loc));
+            }
+        }
+        catch (error) {
+        }
+    }
+    buildSitemapAnalysis() {
+        const crawledUrls = new Set(this.results.map(r => normalizeUrl(r.url)));
+        const crawledFromSitemap = this.sitemapUrls.filter(u => crawledUrls.has(normalizeUrl(u.loc))).length;
+        const linkedUrls = new Set();
+        for (const page of this.results) {
+            for (const link of page.links) {
+                if (link.href) {
+                    linkedUrls.add(normalizeUrl(link.href));
+                }
+            }
+        }
+        const orphanUrls = this.sitemapUrls
+            .filter(u => {
+            const normalized = normalizeUrl(u.loc);
+            return !linkedUrls.has(normalized) && crawledUrls.has(normalized);
+        })
+            .map(u => u.loc);
+        const missingFromSitemap = Array.from(crawledUrls)
+            .filter(url => !this.sitemapUrlSet.has(url));
+        const blockedBySitemapRobots = [];
+        if (this.robotsData) {
+            for (const sitemapUrl of this.sitemapUrls) {
+                try {
+                    const urlPath = new URL(sitemapUrl.loc).pathname;
+                    if (!isPathAllowed(this.robotsData, urlPath, this.options.userAgent)) {
+                        blockedBySitemapRobots.push(sitemapUrl.loc);
+                    }
+                }
+                catch {
+                }
+            }
+        }
+        return {
+            found: this.sitemapUrls.length > 0,
+            url: this.sitemapValidation?.parseResult ? undefined : undefined,
+            totalUrls: this.sitemapUrls.length,
+            crawledFromSitemap,
+            orphanUrls,
+            missingFromSitemap,
+            blockedBySitemapRobots,
+            validationIssues: this.sitemapValidation?.issues.map(i => ({
+                type: i.type,
+                message: i.message,
+            })) || [],
+            sitemapUrls: this.sitemapUrls,
+        };
+    }
+    buildRobotsAnalysis() {
+        return {
+            found: this.robotsValidation?.found ?? false,
+            sitemaps: this.robotsData?.sitemaps ?? [],
+            blocksAll: this.robotsData?.blocksAllRobots ?? false,
+            issues: this.robotsValidation?.issues ?? [],
         };
     }
     async crawlPage(item) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "recker",
-  "version": "1.0.33-next.fbc1682",
+  "version": "1.0.34",
   "description": "AI & DevX focused HTTP client for Node.js 18+",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",