webmcp-cli 1.2.2 → 1.2.3

package/dist/config/schema.js

@@ -0,0 +1,42 @@
+/**
+ * Configuration Schema
+ *
+ * Zod schema for validating configuration files.
+ */
+import { z } from 'zod';
+export const crawlSchema = z.object({
+    maxPages: z.number().int().positive().default(20),
+    maxDepth: z.number().int().positive().default(4),
+    timeout: z.number().int().positive().default(30000),
+    respectRobotsTxt: z.boolean().default(true),
+}).strict();
+export const ruleOverrideSchema = z.object({
+    severity: z.enum(['critical', 'warning', 'info']).optional(),
+    maxScore: z.number().int().nonnegative().optional(),
+});
+export const rulesSchema = z.object({
+    disabled: z.array(z.string()).default([]),
+    overrides: z.record(z.string(), ruleOverrideSchema).default({}),
+}).strict();
+export const scoringWeightsSchema = z.object({
+    implementation: z.number().min(0).max(1).default(0.25),
+    coverage: z.number().min(0).max(1).default(0.20),
+    security: z.number().min(0).max(1).default(0.20),
+    bestPractices: z.number().min(0).max(1).default(0.15),
+    promptHandling: z.number().min(0).max(1).default(0.20),
+}).strict();
+export const scoringSchema = z.object({
+    weights: scoringWeightsSchema.default({}),
+}).strict();
+export const outputSchema = z.object({
+    directory: z.string().default('./agentready-report'),
+    formats: z.array(z.enum(['html', 'json', 'markdown'])).default(['html', 'json']),
+    includeScreenshots: z.boolean().default(true),
+}).strict();
+export const configSchema = z.object({
+    crawl: crawlSchema.default({}),
+    rules: rulesSchema.default({}),
+    scoring: scoringSchema.default({}),
+    output: outputSchema.default({}),
+}).strict();
+//# sourceMappingURL=schema.js.map

package/dist/config/schema.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"schema.js","sourceRoot":"","sources":["../../src/config/schema.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,CAAC,MAAM,CAAC;IAClC,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC;IACjD,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;IAChD,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC;IACnD,gBAAgB,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC;CAC5C,CAAC,CAAC,MAAM,EAAE,CAAC;AAEZ,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAAC,CAAC,MAAM,CAAC;IACzC,QAAQ,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,UAAU,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC,CAAC,QAAQ,EAAE;IAC5D,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,WAAW,EAAE,CAAC,QAAQ,EAAE;CACpD,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,CAAC,MAAM,CAAC;IAClC,QAAQ,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC;IACzC,SAAS,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,kBAAkB,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC;CAChE,CAAC,CAAC,MAAM,EAAE,CAAC;AAEZ,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC3C,cAAc,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC;IACtD,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC;IAChD,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC;IAChD,aAAa,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC;IACrD,cAAc,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC;CACvD,CAAC,CAAC,MAAM,EAAE,CAAC;AAEZ,MAAM,CAAC,MAAM,aAAa,GAAG,CAAC,CAAC,MAAM,CAAC;IACpC,OAAO,EAAE,oBAAoB,CAAC,OAAO,CAAC,EAAE,CAAC;CAC1C,CAAC,CAAC,MAAM,EAAE,CAAC;AAEZ,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC,CAAC,MAAM,CAAC;IACnC,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,qBAAqB,CAAC;IACpD,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,MAAM,EAAE,UAAU,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAChF,kBAAkB,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC;CAC9C,CAAC,CAAC,MAAM,EAAE,CAAC;AAEZ,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC,CAAC,MAAM,CAAC;IACnC,KAAK,EAAE,WAAW,CAAC,OAAO,CAAC,EAAE,CAAC;IAC9B,KAAK,EAAE,WAAW,CAAC,OAAO,CAAC,EAAE,CAAC;IAC9B,OAAO,EAAE,aAAa,CAAC,OAAO,CAAC,EAAE,CAAC;IAClC,MAAM,EAAE,YAAY,CAAC,OAAO,CAAC,EAAE,CAAC;CACjC,CAAC,CAAC,MAAM,EAAE,CAAC"}

package/dist/core/types/audit.d.ts

@@ -63,7 +63,7 @@ export interface CategoryScore {
     /**
      * Category ID
      */
-    id: 'implementation' | 'description' | 'schema' | 'security' | 'best-practices' | 'coverage';
+    id: 'implementation' | 'description' | 'schema' | 'security' | 'best-practices' | 'coverage' | 'accessibility';
     /**
      * Score achieved in this category
      */

package/dist/core/types/index.d.ts

@@ -4,3 +4,4 @@
 export * from './tool.js';
 export * from './audit.js';
 export * from './rule.js';
+export * from './recon.js';

package/dist/core/types/index.js

@@ -4,4 +4,5 @@
 export * from './tool.js';
 export * from './audit.js';
 export * from './rule.js';
+export * from './recon.js';
 //# sourceMappingURL=index.js.map

package/dist/core/types/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/core/types/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,cAAc,WAAW,CAAC;AAC1B,cAAc,YAAY,CAAC;AAC3B,cAAc,WAAW,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/core/types/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,cAAc,WAAW,CAAC;AAC1B,cAAc,YAAY,CAAC;AAC3B,cAAc,WAAW,CAAC;AAC1B,cAAc,YAAY,CAAC"}

package/dist/core/types/recon.d.ts

@@ -0,0 +1,265 @@
+/**
+ * Reconnaissance & Multi-Page Crawl Types
+ */
+/**
+ * Page type classification based on URL/content heuristics
+ */
+export type PageType = 'homepage' | 'search' | 'listing-results' | 'detail-page' | 'checkout-cart' | 'checkout-payment' | 'form-submission' | 'account-management' | 'settings' | 'contact' | 'content-article' | 'help-support' | 'authentication' | 'dashboard' | 'unknown';
+/**
+ * Route priority for crawl ordering
+ */
+export type RoutePriority = 'critical' | 'high' | 'medium' | 'low' | 'skip';
+/**
+ * Source that discovered a route
+ */
+export type RouteSource = 'sitemap' | 'navigation' | 'link' | 'form-action' | 'script' | 'inferred';
+/**
+ * A discovered route within the target site
+ */
+export interface Route {
+    /** Full URL of the route */
+    url: string;
+    /** How the route was discovered */
+    source: RouteSource;
+    /** Clicks from homepage (0 = root) */
+    depth: number;
+    /** Whether a <form> was detected in the source HTML (if known) */
+    hasForm: boolean;
+    /** Whether interactive elements (buttons, tabs, etc.) were detected */
+    hasInteractiveElements: boolean;
+    /** Heuristic page type classification */
+    estimatedPageType: PageType;
+    /** Priority for crawl ordering */
+    priority: RoutePriority;
+    /** Numeric priority score (higher = more important) */
+    priorityScore: number;
+    /** Sitemap lastmod date if available */
+    lastmod?: string;
+    /** Sitemap changefreq if available */
+    changefreq?: string;
+    /** Sitemap priority if available */
+    sitemapPriority?: number;
+}
+/**
+ * Parsed sitemap entry
+ */
+export interface SitemapEntry {
+    loc: string;
+    lastmod?: string;
+    changefreq?: string;
+    priority?: number;
+}
+/**
+ * Result of sitemap parsing
+ */
+export interface SitemapResult {
+    /** Whether a sitemap was found */
+    found: boolean;
+    /** URLs from the sitemap */
+    entries: SitemapEntry[];
+    /** Sitemap source URL */
+    source: string;
+    /** Whether this was a sitemap index */
+    isIndex: boolean;
+    /** Error message if parsing failed */
+    error?: string;
+}
+/**
+ * Parsed robots.txt directive
+ */
+export interface RobotsDirective {
+    userAgent: string;
+    allow: string[];
+    disallow: string[];
+    crawlDelay?: number;
+}
+/**
+ * Result of robots.txt parsing
+ */
+export interface RobotsResult {
+    /** Whether robots.txt was found */
+    found: boolean;
+    /** Parsed directives grouped by user-agent */
+    directives: RobotsDirective[];
+    /** Sitemap URLs referenced in robots.txt */
+    sitemapUrls: string[];
+    /** Raw robots.txt content */
+    raw: string;
+    /** Error message if parsing failed */
+    error?: string;
+}
+/**
+ * Result of web app manifest parsing
+ */
+export interface ManifestResult {
+    /** Whether a manifest was found */
+    found: boolean;
+    /** App name */
+    name?: string;
+    /** Short name */
+    shortName?: string;
+    /** App description */
+    description?: string;
+    /** Start URL */
+    startUrl?: string;
+    /** Display mode */
+    display?: string;
+    /** Theme color */
+    themeColor?: string;
+    /** Source URL of the manifest */
+    source?: string;
+    /** Raw manifest data */
+    raw?: Record<string, unknown>;
+    /** Error message if parsing failed */
+    error?: string;
+}
+/**
+ * Extracted meta tag information
+ */
+export interface MetaTagResult {
+    /** Meta tag name or property */
+    name: string;
+    /** Meta tag content */
+    content: string;
+    /** Category: og, twitter, schema, generic */
+    category: 'og' | 'twitter' | 'schema' | 'generic';
+}
+/**
+ * Schema.org JSON-LD data
+ */
+export interface SchemaOrgData {
+    '@type': string;
+    [key: string]: unknown;
+}
+/**
+ * Site type classification from meta analysis
+ */
+export type SiteClassification = 'e-commerce' | 'travel' | 'healthcare' | 'finance' | 'education' | 'news-media' | 'social' | 'saas' | 'government' | 'entertainment' | 'food-restaurant' | 'real-estate' | 'general';
+/**
+ * Result of meta tag extraction
+ */
+export interface MetaExtractResult {
+    /** All extracted meta tags */
+    metaTags: MetaTagResult[];
+    /** OpenGraph data */
+    openGraph: Record<string, string>;
+    /** Twitter card data */
+    twitterCard: Record<string, string>;
+    /** Schema.org JSON-LD entries */
+    schemaOrg: SchemaOrgData[];
+    /** Classified site type */
+    siteClassification: SiteClassification;
+    /** Page title from <title> tag */
+    pageTitle: string;
+    /** Page description from meta description */
+    pageDescription: string;
+    /** Canonical URL */
+    canonicalUrl?: string;
+}
+/**
+ * Full reconnaissance result
+ */
+export interface ReconResult {
+    /** Original target URL */
+    url: string;
+    /** Resolved URL after redirects */
+    resolvedUrl: string;
+    /** HTTP status code of root page */
+    statusCode: number;
+    /** Response time in ms for root page fetch */
+    responseTime: number;
+    /** Sitemap analysis */
+    sitemap: SitemapResult | null;
+    /** Robots.txt analysis */
+    robots: RobotsResult | null;
+    /** Web app manifest */
+    manifest: ManifestResult | null;
+    /** Meta tag extraction from root page */
+    meta: MetaExtractResult | null;
+    /** All discovered routes, sorted by priority */
+    discoveredRoutes: Route[];
+    /** Root page HTML (for downstream use) */
+    rootHtml: string;
+    /** Errors encountered during recon (non-fatal) */
+    errors: string[];
+}
+/**
+ * Configuration for multi-page crawling
+ */
+export interface CrawlConfig {
+    /** Maximum number of pages to crawl (default: 20) */
+    maxPages: number;
+    /** Maximum crawl depth from root (default: 4) */
+    maxDepth: number;
+    /** Per-page timeout in ms (default: 30000) */
+    timeout: number;
+    /** Extra wait time for SPA hydration in ms (default: 2000) */
+    extraWaitMs: number;
+    /** Whether to respect robots.txt disallow rules (default: true) */
+    respectRobotsTxt: boolean;
+}
+/**
+ * Progress callback for multi-page crawl
+ */
+export interface MultiPageCrawlProgress {
+    /** Current phase: 'recon' | 'crawling' | 'complete' */
+    phase: 'recon' | 'crawling' | 'complete';
+    /** Overall progress percentage (0-100) */
+    progress: number;
+    /** Current page being audited */
+    currentUrl: string;
+    /** Number of pages completed */
+    pagesCompleted: number;
+    /** Total pages planned */
+    pagesTotal: number;
+    /** Tools found so far across all pages */
+    toolsFound: number;
+    /** Human-readable status message */
+    message: string;
+}
+/**
+ * Per-page result within a multi-page audit
+ */
+export interface PageAuditSummary {
+    /** URL of the page */
+    url: string;
+    /** Page title */
+    title: string;
+    /** Whether page loaded successfully */
+    success: boolean;
+    /** Whether WebMCP was found on this page */
+    hasWebMCP: boolean;
+    /** Number of tools found on this page */
+    toolCount: number;
+    /** Number of opportunities found */
+    opportunityCount: number;
+    /** Page-level score (if available) */
+    score?: number;
+    /** Error message if page failed */
+    error?: string;
+    /** Route that led to this page */
+    route: Route;
+}
+/**
+ * Complete multi-page audit result
+ */
+export interface MultiPageAuditResult {
+    /** Reconnaissance data */
+    recon: ReconResult;
+    /** Per-page audit summaries */
+    pageSummaries: PageAuditSummary[];
+    /** Aggregate: all tools found across all pages */
+    allTools: import('./tool.js').DetectedTool[];
+    /** Aggregate: all opportunities across all pages */
+    allOpportunities: import('./tool.js').UnregisteredForm[];
+    /** Aggregate: all findings across all pages */
+    allFindings: import('./rule.js').Finding[];
+    /** Aggregate score */
+    aggregateScore: import('./audit.js').ScoreSummary;
+    /** Pages that had WebMCP */
+    webmcpPages: string[];
+    /** Total pages attempted */
+    pagesAttempted: number;
+    /** Total pages successfully audited */
+    pagesSucceeded: number;
+}

package/dist/core/types/recon.js

@@ -0,0 +1,5 @@
+/**
+ * Reconnaissance & Multi-Page Crawl Types
+ */
+export {};
+//# sourceMappingURL=recon.js.map

package/dist/core/types/recon.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"recon.js","sourceRoot":"","sources":["../../../src/core/types/recon.ts"],"names":[],"mappings":"AAAA;;GAEG"}

package/dist/core/types/rule.d.ts

@@ -7,7 +7,7 @@ import type { PageInfo } from './audit.js';
 /**
  * Rule categories
  */
-export type RuleCategory = 'implementation' | 'description' | 'schema' | 'security' | 'best-practices' | 'coverage';
+export type RuleCategory = 'implementation' | 'description' | 'schema' | 'security' | 'best-practices' | 'coverage' | 'accessibility';
 /**
  * Rule severity levels
  */

package/dist/core/types/rule.js

@@ -5,12 +5,13 @@
  * Score weights by category
  */
 export const CATEGORY_WEIGHTS = {
-    implementation: 25,
-    description: 15,
-    schema: 20,
-    security: 25,
-    'best-practices': 10,
+    implementation: 23,
+    description: 14,
+    schema: 18,
+    security: 23,
+    'best-practices': 9,
     coverage: 5,
+    accessibility: 8,
 };
 /**
  * Category display names
@@ -22,5 +23,6 @@ export const CATEGORY_NAMES = {
     security: 'Security',
     'best-practices': 'Best Practices',
     coverage: 'Coverage',
+    accessibility: 'Accessibility',
 };
 //# sourceMappingURL=rule.js.map

package/dist/core/types/rule.js.map

@@ -1 +1 @@
-{"version":3,"file":"rule.js","sourceRoot":"","sources":["../../../src/core/types/rule.ts"],"names":[],"mappings":"AAAA;;GAEG;AAiOH;;GAEG;AACH,MAAM,CAAC,MAAM,gBAAgB,GAAiC;IAC5D,cAAc,EAAE,EAAE;IAClB,WAAW,EAAE,EAAE;IACf,MAAM,EAAE,EAAE;IACV,QAAQ,EAAE,EAAE;IACZ,gBAAgB,EAAE,EAAE;IACpB,QAAQ,EAAE,CAAC;CACZ,CAAC;AAEF;;GAEG;AACH,MAAM,CAAC,MAAM,cAAc,GAAiC;IAC1D,cAAc,EAAE,gBAAgB;IAChC,WAAW,EAAE,qBAAqB;IAClC,MAAM,EAAE,mBAAmB;IAC3B,QAAQ,EAAE,UAAU;IACpB,gBAAgB,EAAE,gBAAgB;IAClC,QAAQ,EAAE,UAAU;CACrB,CAAC"}
+{"version":3,"file":"rule.js","sourceRoot":"","sources":["../../../src/core/types/rule.ts"],"names":[],"mappings":"AAAA;;GAEG;AAkOH;;GAEG;AACH,MAAM,CAAC,MAAM,gBAAgB,GAAiC;IAC5D,cAAc,EAAE,EAAE;IAClB,WAAW,EAAE,EAAE;IACf,MAAM,EAAE,EAAE;IACV,QAAQ,EAAE,EAAE;IACZ,gBAAgB,EAAE,CAAC;IACnB,QAAQ,EAAE,CAAC;IACX,aAAa,EAAE,CAAC;CACjB,CAAC;AAEF;;GAEG;AACH,MAAM,CAAC,MAAM,cAAc,GAAiC;IAC1D,cAAc,EAAE,gBAAgB;IAChC,WAAW,EAAE,qBAAqB;IAClC,MAAM,EAAE,mBAAmB;IAC3B,QAAQ,EAAE,UAAU;IACpB,gBAAgB,EAAE,gBAAgB;IAClC,QAAQ,EAAE,UAAU;IACpB,aAAa,EAAE,eAAe;CAC/B,CAAC"}

package/dist/crawler/depth-crawler.d.ts

@@ -0,0 +1,29 @@
+export interface CrawlPage {
+    url: string;
+    depth: number;
+    html: string;
+    title: string;
+    internalLinks: string[];
+    statusCode: number;
+    loadTime: number;
+}
+export interface CrawlResult {
+    pages: CrawlPage[];
+    totalPagesFound: number;
+    totalPagesCrawled: number;
+    crawlDuration: number;
+    errors: Array<{
+        url: string;
+        error: string;
+    }>;
+}
+export interface CrawlOptions {
+    maxPages?: number;
+    timeout?: number;
+    concurrency?: number;
+    onPageCrawled?: (page: CrawlPage, progress: {
+        done: number;
+        total: number;
+    }) => void;
+}
+export declare function crawlDepth2(startUrl: string, options?: CrawlOptions): Promise<CrawlResult>;

package/dist/crawler/depth-crawler.js

@@ -0,0 +1,212 @@
+import * as cheerio from 'cheerio';
+import { extractInternalLinks } from './link-extractor.js';
+const DEFAULT_MAX_PAGES = 30;
+const DEFAULT_TIMEOUT = 15_000;
+const DEFAULT_CONCURRENCY = 3;
+const MAX_DEPTH = 2;
+const MIN_TIMEOUT = 1_000;
+const USER_AGENT = 'WebMCP-CLI DepthCrawler/1.0 (+https://webmcp.org)';
+function normalizeCrawlUrl(url) {
+    try {
+        const parsed = new URL(url);
+        parsed.hash = '';
+        parsed.searchParams.sort();
+        if (parsed.pathname.length > 1 && parsed.pathname.endsWith('/')) {
+            parsed.pathname = parsed.pathname.slice(0, -1);
+        }
+        return parsed.href;
+    }
+    catch {
+        return null;
+    }
+}
+function extractTitle(html) {
+    const $ = cheerio.load(html);
+    return $('title').first().text().trim();
+}
+function isSameHostname(url, hostname) {
+    try {
+        return new URL(url).hostname === hostname;
+    }
+    catch {
+        return false;
+    }
+}
+async function fetchPage(url, timeout) {
+    const controller = new AbortController();
+    const timer = setTimeout(() => controller.abort(), timeout);
+    const started = Date.now();
+    try {
+        const response = await fetch(url, {
+            signal: controller.signal,
+            redirect: 'follow',
+            headers: {
+                'User-Agent': USER_AGENT,
+                Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+            },
+        });
+        const html = await response.text();
+        return {
+            html,
+            statusCode: response.status,
+            finalUrl: response.url,
+            loadTime: Date.now() - started,
+        };
+    }
+    finally {
+        clearTimeout(timer);
+    }
+}
+export async function crawlDepth2(startUrl, options) {
+    const crawlStart = Date.now();
+    const pages = [];
+    const errors = [];
+    const discovered = new Set();
+    const enqueued = new Set();
+    const visited = new Set();
+    const maxPages = Math.max(1, Math.floor(options?.maxPages ?? DEFAULT_MAX_PAGES));
+    const timeout = Math.max(MIN_TIMEOUT, Math.floor(options?.timeout ?? DEFAULT_TIMEOUT));
+    const concurrency = Math.max(1, Math.floor(options?.concurrency ?? DEFAULT_CONCURRENCY));
+    const onPageCrawled = options?.onPageCrawled;
+    const normalizedStart = normalizeCrawlUrl(startUrl);
+    if (!normalizedStart) {
+        return {
+            pages,
+            totalPagesFound: 0,
+            totalPagesCrawled: 0,
+            crawlDuration: 0,
+            errors: [{ url: startUrl, error: 'Invalid start URL' }],
+        };
+    }
+    const startUrlObject = new URL(normalizedStart);
+    const allowedHostname = startUrlObject.hostname;
+    let baseOrigin = startUrlObject.origin;
+    const queue = [{ url: normalizedStart, depth: 0 }];
+    enqueued.add(normalizedStart);
+    discovered.add(normalizedStart);
+    const enqueueLink = (url, depth) => {
+        if (depth > MAX_DEPTH)
+            return;
+        const normalized = normalizeCrawlUrl(url);
+        if (!normalized)
+            return;
+        if (new URL(normalized).origin !== baseOrigin)
+            return;
+        if (enqueued.has(normalized) || visited.has(normalized))
+            return;
+        enqueued.add(normalized);
+        discovered.add(normalized);
+        queue.push({ url: normalized, depth });
+    };
+    const processPage = async (item) => {
+        if (visited.has(item.url)) {
+            return;
+        }
+        visited.add(item.url);
+        try {
+            const result = await fetchPage(item.url, timeout);
+            if (!isSameHostname(result.finalUrl, allowedHostname)) {
+                errors.push({ url: item.url, error: 'Redirected outside allowed origin' });
+                return;
+            }
+            const finalOrigin = new URL(result.finalUrl).origin;
+            if (item.depth === 0 && finalOrigin !== baseOrigin) {
+                baseOrigin = finalOrigin;
+            }
+            else if (finalOrigin !== baseOrigin) {
+                errors.push({ url: item.url, error: 'Redirected to different origin' });
+                return;
+            }
+            const normalizedFinalUrl = normalizeCrawlUrl(result.finalUrl) ?? item.url;
+            if (normalizedFinalUrl !== item.url) {
+                enqueued.add(normalizedFinalUrl);
+                discovered.add(normalizedFinalUrl);
+                visited.add(normalizedFinalUrl);
+            }
+            const internalLinks = extractInternalLinks(result.html, result.finalUrl);
+            const title = extractTitle(result.html);
+            if (pages.length >= maxPages) {
+                return;
+            }
+            const page = {
+                url: normalizedFinalUrl,
+                depth: item.depth,
+                html: result.html,
+                title,
+                internalLinks,
+                statusCode: result.statusCode,
+                loadTime: result.loadTime,
+            };
+            pages.push(page);
+            if (onPageCrawled) {
+                try {
+                    onPageCrawled(page, { done: pages.length, total: maxPages });
+                }
+                catch (callbackError) {
+                    errors.push({
+                        url: item.url,
+                        error: `onPageCrawled callback failed: ${callbackError instanceof Error ? callbackError.message : String(callbackError)}`,
+                    });
+                }
+            }
+            if (item.depth < MAX_DEPTH) {
+                for (const link of internalLinks) {
+                    enqueueLink(link, item.depth + 1);
+                }
+            }
+        }
+        catch (error) {
+            const message = error instanceof Error ? error.message : String(error);
+            errors.push({ url: item.url, error: message });
+        }
+    };
+    const inFlight = new Set();
+    await new Promise((resolve) => {
+        let finished = false;
+        const finish = () => {
+            if (finished)
+                return;
+            finished = true;
+            resolve();
+        };
+        const schedule = () => {
+            if (pages.length >= maxPages) {
+                queue.length = 0;
+            }
+            while (inFlight.size < concurrency &&
+                queue.length > 0 &&
+                pages.length + inFlight.size < maxPages) {
+                const next = queue.shift();
+                if (!next)
+                    break;
+                const task = processPage(next)
+                    .catch((error) => {
+                    const message = error instanceof Error ? error.message : String(error);
+                    errors.push({ url: next.url, error: message });
+                })
+                    .finally(() => {
+                    inFlight.delete(task);
+                    if (queue.length === 0 && inFlight.size === 0) {
+                        finish();
+                    }
+                    else {
+                        schedule();
+                    }
+                });
+                inFlight.add(task);
+            }
+            if (queue.length === 0 && inFlight.size === 0) {
+                finish();
+            }
+        };
+        schedule();
+    });
+    return {
+        pages,
+        totalPagesFound: discovered.size,
+        totalPagesCrawled: pages.length,
+        crawlDuration: Date.now() - crawlStart,
+        errors,
+    };
+}
+//# sourceMappingURL=depth-crawler.js.map

package/dist/crawler/depth-crawler.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"depth-crawler.js","sourceRoot":"","sources":["../../src/crawler/depth-crawler.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AACnC,OAAO,EAAE,oBAAoB,EAAE,MAAM,qBAAqB,CAAC;AAuC3D,MAAM,iBAAiB,GAAG,EAAE,CAAC;AAC7B,MAAM,eAAe,GAAG,MAAM,CAAC;AAC/B,MAAM,mBAAmB,GAAG,CAAC,CAAC;AAC9B,MAAM,SAAS,GAAG,CAAC,CAAC;AACpB,MAAM,WAAW,GAAG,KAAK,CAAC;AAC1B,MAAM,UAAU,GAAG,mDAAmD,CAAC;AAEvE,SAAS,iBAAiB,CAAC,GAAW;IACpC,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAC5B,MAAM,CAAC,IAAI,GAAG,EAAE,CAAC;QACjB,MAAM,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC;QAC3B,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;YAChE,MAAM,CAAC,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QACjD,CAAC;QACD,OAAO,MAAM,CAAC,IAAI,CAAC;IACrB,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,SAAS,YAAY,CAAC,IAAY;IAChC,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC7B,OAAO,CAAC,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;AAC1C,CAAC;AAED,SAAS,cAAc,CAAC,GAAW,EAAE,QAAgB;IACnD,IAAI,CAAC;QACH,OAAO,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,KAAK,QAAQ,CAAC;IAC5C,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED,KAAK,UAAU,SAAS,CAAC,GAAW,EAAE,OAAe;IACnD,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;IACzC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,OAAO,CAAC,CAAC;IAC5D,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAE3B,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;YAChC,MAAM,EAAE,UAAU,CAAC,MAAM;YACzB,QAAQ,EAAE,QAAQ;YAClB,OAAO,EAAE;gBACP,YAAY,EAAE,UAAU;gBACxB,MAAM,EAAE,iEAAiE;aAC1E;SACF,CAAC,CAAC;QAEH,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;QACnC,OAAO;YACL,IAAI;YACJ,UAAU,EAAE,QAAQ,CAAC,MAAM;YAC3B,QAAQ,EAAE,QAAQ,CAAC,GAAG;YACtB,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO;SAC/B,CAAC;IACJ,CAAC;YAAS,CAAC;QACT,YAAY,CAAC,KAAK,CAAC,CAAC;IACtB,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,QAAgB,EAAE,OAAsB;IACxE,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC9B,MAAM,KAAK,GAAgB,EAAE,CAAC;IAC9B,MAAM,MAAM,GAA0C,EAAE,CAAC;IACzD,MAAM,UAAU,GAAG,IAAI,GAAG,EAAU,CAAC;IACrC,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAU,CAAC;IACnC,MAAM,OAAO,GAAG,IAAI,GAAG,EAAU,CAAC;IAElC,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,QAAQ,IAAI,iBAAiB,CAAC,CAAC,CAAC;IACjF,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,OAAO,IAAI,eAAe,CAAC,CAAC,CAAC;IACvF,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,WAAW,IAAI,mBAAmB,CAAC,CAAC,CAAC;IACzF,MAAM,aAAa,GAAG,OAAO,EAAE,aAAa,CAAC;IAE7C,MAAM,eAAe,GAAG,iBAAiB,CAAC,QAAQ,CAAC,CAAC;IACpD,IAAI,CAAC,eAAe,EAAE,CAAC;QACrB,OAAO;YACL,KAAK;YACL,eAAe,EAAE,CAAC;YAClB,iBAAiB,EAAE,CAAC;YACpB,aAAa,EAAE,CAAC;YAChB,MAAM,EAAE,CAAC,EAAE,GAAG,EAAE,QAAQ,EAAE,KAAK,EAAE,mBAAmB,EAAE,CAAC;SACxD,CAAC;IACJ,CAAC;IAED,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC,eAAe,CAAC,CAAC;IAChD,MAAM,eAAe,GAAG,cAAc,CAAC,QAAQ,CAAC;IAChD,IAAI,UAAU,GAAG,cAAc,CAAC,MAAM,CAAC;IAEvC,MAAM,KAAK,GAAgB,CAAC,EAAE,GAAG,EAAE,eAAe,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;IAChE,QAAQ,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC;IAC9B,UAAU,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC;IAEhC,MAAM,WAAW,GAAG,CAAC,GAAW,EAAE,KAAa,EAAQ,EAAE;QACvD,IAAI,KAAK,GAAG,SAAS;YAAE,OAAO;QAC9B,MAAM,UAAU,GAAG,iBAAiB,CAAC,GAAG,CAAC,CAAC;QAC1C,IAAI,CAAC,UAAU;YAAE,OAAO;QACxB,IAAI,IAAI,GAAG,CAAC,UAAU,CAAC,CAAC,MAAM,KAAK,UAAU;YAAE,OAAO;QACtD,IAAI,QAAQ,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC;YAAE,OAAO;QAEhE,QAAQ,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QACzB,UAAU,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QAC3B,KAAK,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC,CAAC;IACzC,CAAC,CAAC;IAEF,MAAM,WAAW,GAAG,KAAK,EAAE,IAAe,EAAiB,EAAE;QAC3D,IAAI,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;YAC1B,OAAO;QACT,CAAC;QACD,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAEtB,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;YAElD,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,QAAQ,EAAE,eAAe,CAAC,EAAE,CAAC;gBACtD,MAAM,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,KAAK,EAAE,mCAAmC,EAAE,CAAC,CAAC;gBAC3E,OAAO;YACT,CAAC;YAED,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC;YACpD,IAAI,IAAI,CAAC,KAAK,KAAK,CAAC,IAAI,WAAW,KAAK,UAAU,EAAE,CAAC;gBACnD,UAAU,GAAG,WAAW,CAAC;YAC3B,CAAC;iBAAM,IAAI,WAAW,KAAK,UAAU,EAAE,CAAC;gBACtC,MAAM,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,KAAK,EAAE,gCAAgC,EAAE,CAAC,CAAC;gBACxE,OAAO;YACT,CAAC;YAED,MAAM,kBAAkB,GAAG,iBAAiB,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC,GAAG,CAAC;YAC1E,IAAI,kBAAkB,KAAK,IAAI,CAAC,GAAG,EAAE,CAAC;gBACpC,QAAQ,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAC;gBACjC,UAAU,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAC;gBACnC,OAAO,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAC;YAClC,CAAC;YACD,MAAM,aAAa,GAAG,oBAAoB,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,QAAQ,CAAC,CAAC;YACzE,MAAM,KAAK,GAAG,YAAY,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;YAExC,IAAI,KAAK,CAAC,MAAM,IAAI,QAAQ,EAAE,CAAC;gBAC7B,OAAO;YACT,CAAC;YAED,MAAM,IAAI,GAAc;gBACtB,GAAG,EAAE,kBAAkB;gBACvB,KAAK,EAAE,IAAI,CAAC,KAAK;gBACjB,IAAI,EAAE,MAAM,CAAC,IAAI;gBACjB,KAAK;gBACL,aAAa;gBACb,UAAU,EAAE,MAAM,CAAC,UAAU;gBAC7B,QAAQ,EAAE,MAAM,CAAC,QAAQ;aAC1B,CAAC;YAEF,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAEjB,IAAI,aAAa,EAAE,CAAC;gBAClB,IAAI,CAAC;oBACH,aAAa,CAAC,IAAI,EAAE,EAAE,IAAI,EAAE,KAAK,CAAC,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC,CAAC;gBAC/D,CAAC;gBAAC,OAAO,aAAa,EAAE,CAAC;oBACvB,MAAM,CAAC,IAAI,CAAC;wBACV,GAAG,EAAE,IAAI,CAAC,GAAG;wBACb,KAAK,EAAE,kCAAkC,aAAa,YAAY,KAAK,CAAC,CAAC,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE;qBAC1H,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;YAED,IAAI,IAAI,CAAC,KAAK,GAAG,SAAS,EAAE,CAAC;gBAC3B,KAAK,MAAM,IAAI,IAAI,aAAa,EAAE,CAAC;oBACjC,WAAW,CAAC,IAAI,EAAE,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;gBACpC,CAAC;YACH,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACvE,MAAM,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,CAAC;QACjD,CAAC;IACH,CAAC,CAAC;IAEF,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAiB,CAAC;IAE1C,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;QAClC,IAAI,QAAQ,GAAG,KAAK,CAAC;QAErB,MAAM,MAAM,GAAG,GAAS,EAAE;YACxB,IAAI,QAAQ;gBAAE,OAAO;YACrB,QAAQ,GAAG,IAAI,CAAC;YAChB,OAAO,EAAE,CAAC;QACZ,CAAC,CAAC;QAEF,MAAM,QAAQ,GAAG,GAAS,EAAE;YAC1B,IAAI,KAAK,CAAC,MAAM,IAAI,QAAQ,EAAE,CAAC;gBAC7B,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC;YACnB,CAAC;YAED,OACE,QAAQ,CAAC,IAAI,GAAG,WAAW;gBAC3B,KAAK,CAAC,MAAM,GAAG,CAAC;gBAChB,KAAK,CAAC,MAAM,GAAG,QAAQ,CAAC,IAAI,GAAG,QAAQ,EACvC,CAAC;gBACD,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,EAAE,CAAC;gBAC3B,IAAI,CAAC,IAAI;oBAAE,MAAM;gBAEjB,MAAM,IAAI,GAAG,WAAW,CAAC,IAAI,CAAC;qBAC3B,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;oBACf,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;oBACvE,MAAM,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,CAAC;gBACjD,CAAC,CAAC;qBACD,OAAO,CAAC,GAAG,EAAE;oBACZ,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;oBACtB,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,IAAI,QAAQ,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;wBAC9C,MAAM,EAAE,CAAC;oBACX,CAAC;yBAAM,CAAC;wBACN,QAAQ,EAAE,CAAC;oBACb,CAAC;gBACH,CAAC,CAAC,CAAC;gBAEL,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YACrB,CAAC;YAED,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,IAAI,QAAQ,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;gBAC9C,MAAM,EAAE,CAAC;YACX,CAAC;QACH,CAAC,CAAC;QAEF,QAAQ,EAAE,CAAC;IACb,CAAC,CAAC,CAAC;IAEH,OAAO;QACL,KAAK;QACL,eAAe,EAAE,UAAU,CAAC,IAAI;QAChC,iBAAiB,EAAE,KAAK,CAAC,MAAM;QAC/B,aAAa,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,UAAU;QACtC,MAAM;KACP,CAAC;AACJ,CAAC"}

package/dist/crawler/index.d.ts

@@ -0,0 +1,2 @@
+export * from './link-extractor.js';
+export * from './depth-crawler.js';

package/dist/crawler/index.js

@@ -0,0 +1,3 @@
+export * from './link-extractor.js';
+export * from './depth-crawler.js';
+//# sourceMappingURL=index.js.map

package/dist/crawler/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/crawler/index.ts"],"names":[],"mappings":"AAAA,cAAc,qBAAqB,CAAC;AACpC,cAAc,oBAAoB,CAAC"}

package/dist/crawler/link-extractor.d.ts

@@ -0,0 +1 @@
+export declare function extractInternalLinks(html: string, baseUrl: string): string[];