webmcp-cli 1.2.2 → 1.2.3

package/dist/recon/robots.js

@@ -0,0 +1,158 @@
+/**
+ * Robots.txt Parser
+ *
+ * Fetches and parses robots.txt. Extracts allowed/disallowed paths,
+ * crawl-delay, and sitemap references. Handles missing robots.txt gracefully.
+ */
+/** Fetch timeout (ms) */
+const FETCH_TIMEOUT_MS = 10_000;
+/**
+ * Fetch with timeout using AbortController
+ */
+async function fetchWithTimeout(url, timeoutMs = FETCH_TIMEOUT_MS) {
+    const controller = new AbortController();
+    const timer = setTimeout(() => controller.abort(), timeoutMs);
+    try {
+        return await fetch(url, {
+            signal: controller.signal,
+            headers: { 'User-Agent': 'WebMCP-CLI/1.0 (robots-parser)' },
+            redirect: 'follow',
+        });
+    }
+    finally {
+        clearTimeout(timer);
+    }
+}
+/**
+ * Parse robots.txt content into structured directives
+ */
+function parseRobotsTxt(raw) {
+    const directives = [];
+    const sitemapUrls = [];
+    let current = null;
+    for (const rawLine of raw.split('\n')) {
+        // Strip comments and whitespace
+        const commentIdx = rawLine.indexOf('#');
+        const line = (commentIdx !== -1 ? rawLine.substring(0, commentIdx) : rawLine).trim();
+        if (!line)
+            continue;
+        const colonIdx = line.indexOf(':');
+        if (colonIdx === -1)
+            continue;
+        const field = line.substring(0, colonIdx).trim().toLowerCase();
+        const value = line.substring(colonIdx + 1).trim();
+        if (field === 'user-agent') {
+            // Start a new directive block
+            current = {
+                userAgent: value,
+                allow: [],
+                disallow: [],
+            };
+            directives.push(current);
+            continue;
+        }
+        if (field === 'sitemap') {
+            // Sitemap directives are global, not per user-agent
+            if (value)
+                sitemapUrls.push(value);
+            continue;
+        }
+        if (!current)
+            continue;
+        switch (field) {
+            case 'allow':
+                if (value)
+                    current.allow.push(value);
+                break;
+            case 'disallow':
+                if (value)
+                    current.disallow.push(value);
+                break;
+            case 'crawl-delay': {
+                const delay = parseFloat(value);
+                if (!Number.isNaN(delay) && delay >= 0) {
+                    current.crawlDelay = delay;
+                }
+                break;
+            }
+        }
+    }
+    return { directives, sitemapUrls };
+}
+/**
+ * Check whether a path is allowed by the robots.txt directives.
+ * Uses wildcard user-agent (*) rules if no specific match.
+ */
+export function isPathAllowed(path, directives, userAgent = '*') {
+    // Find matching directive (specific UA first, then wildcard)
+    const specific = directives.find((d) => d.userAgent.toLowerCase() === userAgent.toLowerCase());
+    const wildcard = directives.find((d) => d.userAgent === '*');
+    const directive = specific ?? wildcard;
+    if (!directive)
+        return true; // No rules = allowed
+    // Check disallow rules — longest match wins
+    let longestDisallow = 0;
+    let longestAllow = 0;
+    for (const rule of directive.disallow) {
+        if (path.startsWith(rule) && rule.length > longestDisallow) {
+            longestDisallow = rule.length;
+        }
+    }
+    for (const rule of directive.allow) {
+        if (path.startsWith(rule) && rule.length > longestAllow) {
+            longestAllow = rule.length;
+        }
+    }
+    // If allow is longer or equal, path is allowed
+    if (longestAllow >= longestDisallow)
+        return true;
+    return longestDisallow === 0;
+}
+/**
+ * Fetch and parse robots.txt for a site
+ */
+export async function parseRobots(baseUrl) {
+    const url = new URL('/robots.txt', baseUrl).href;
+    try {
+        const response = await fetchWithTimeout(url);
+        if (!response.ok) {
+            return {
+                found: false,
+                directives: [],
+                sitemapUrls: [],
+                raw: '',
+                error: `HTTP ${response.status}`,
+            };
+        }
+        const contentType = response.headers.get('content-type') ?? '';
+        // Robots.txt should be text/plain; skip if HTML (common 404 page)
+        if (contentType.includes('text/html')) {
+            return {
+                found: false,
+                directives: [],
+                sitemapUrls: [],
+                raw: '',
+                error: 'Response was HTML, not robots.txt',
+            };
+        }
+        const raw = await response.text();
+        const { directives, sitemapUrls } = parseRobotsTxt(raw);
+        return {
+            found: true,
+            directives,
+            sitemapUrls,
+            raw,
+        };
+    }
+    catch (error) {
+        const message = error instanceof Error ? error.message : 'Unknown error';
+        return {
+            found: false,
+            directives: [],
+            sitemapUrls: [],
+            raw: '',
+            error: message,
+        };
+    }
+}
+//# sourceMappingURL=robots.js.map

package/dist/recon/robots.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"robots.js","sourceRoot":"","sources":["../../src/recon/robots.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAIH,yBAAyB;AACzB,MAAM,gBAAgB,GAAG,MAAM,CAAC;AAEhC;;GAEG;AACH,KAAK,UAAU,gBAAgB,CAC7B,GAAW,EACX,YAAoB,gBAAgB;IAEpC,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;IACzC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,SAAS,CAAC,CAAC;IAC9D,IAAI,CAAC;QACH,OAAO,MAAM,KAAK,CAAC,GAAG,EAAE;YACtB,MAAM,EAAE,UAAU,CAAC,MAAM;YACzB,OAAO,EAAE,EAAE,YAAY,EAAE,gCAAgC,EAAE;YAC3D,QAAQ,EAAE,QAAQ;SACnB,CAAC,CAAC;IACL,CAAC;YAAS,CAAC;QACT,YAAY,CAAC,KAAK,CAAC,CAAC;IACtB,CAAC;AACH,CAAC;AAED;;GAEG;AACH,SAAS,cAAc,CACrB,GAAW;IAEX,MAAM,UAAU,GAAsB,EAAE,CAAC;IACzC,MAAM,WAAW,GAAa,EAAE,CAAC;IAEjC,IAAI,OAAO,GAA2B,IAAI,CAAC;IAE3C,KAAK,MAAM,OAAO,IAAI,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;QACtC,gCAAgC;QAChC,MAAM,UAAU,GAAG,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QACxC,MAAM,IAAI,GAAG,CAAC,UAAU,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC;QACrF,IAAI,CAAC,IAAI;YAAE,SAAS;QAEpB,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QACnC,IAAI,QAAQ,KAAK,CAAC,CAAC;YAAE,SAAS;QAE9B,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QAC/D,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAElD,IAAI,KAAK,KAAK,YAAY,EAAE,CAAC;YAC3B,8BAA8B;YAC9B,OAAO,GAAG;gBACR,SAAS,EAAE,KAAK;gBAChB,KAAK,EAAE,EAAE;gBACT,QAAQ,EAAE,EAAE;aACb,CAAC;YACF,UAAU,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACzB,SAAS;QACX,CAAC;QAED,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;YACxB,oDAAoD;YACpD,IAAI,KAAK;gBAAE,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACnC,SAAS;QACX,CAAC;QAED,IAAI,CAAC,OAAO;YAAE,SAAS;QAEvB,QAAQ,KAAK,EAAE,CAAC;YACd,KAAK,OAAO;gBACV,IAAI,KAAK;oBAAE,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBACrC,MAAM;YACR,KAAK,UAAU;gBACb,IAAI,KAAK;oBAAE,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBACxC,MAAM;YACR,KAAK,aAAa,CAAC,CAAC,CAAC;gBACnB,MAAM,KAAK,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC;gBAChC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,KAAK,IAAI,CAAC,EAAE,CAAC;oBACvC,OAAO,CAAC,UAAU,GAAG,KAAK,CAAC;gBAC7B,CAAC;gBACD,MAAM;YACR,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,EAAE,UAAU,EAAE,WAAW,EAAE,CAAC;AACrC,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,aAAa,CAC3B,IAAY,EACZ,UAA6B,EAC7B,YAAoB,GAAG;IAEvB,6DAA6D;IAC7D,MAAM,QAAQ,GAAG,UAAU,CAAC,IAAI,CAC9B,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,WAAW,EAAE,KAAK,SAAS,CAAC,WAAW,EAAE,CAC7D,CAAC;IACF,MAAM,QAAQ,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,KAAK,GAAG,CAAC,CAAC;IAC7D,MAAM,SAAS,GAAG,QAAQ,IAAI,QAAQ,CAAC;IAEvC,IAAI,CAAC,SAAS;QAAE,OAAO,IAAI,CAAC,CAAC,qBAAqB;IAElD,4CAA4C;IAC5C,IAAI,eAAe,GAAG,CAAC,CAAC;IACxB,IAAI,YAAY,GAAG,CAAC,CAAC;IAErB,KAAK,MAAM,IAAI,IAAI,SAAS,CAAC,QAAQ,EAAE,CAAC;QACtC,IAAI,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,MAAM,GAAG,eAAe,EAAE,CAAC;YAC3D,eAAe,GAAG,IAAI,CAAC,MAAM,CAAC;QAChC,CAAC;IACH,CAAC;IAED,KAAK,MAAM,IAAI,IAAI,SAAS,CAAC,KAAK,EAAE,CAAC;QACnC,IAAI,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,MAAM,GAAG,YAAY,EAAE,CAAC;YACxD,YAAY,GAAG,IAAI,CAAC,MAAM,CAAC;QAC7B,CAAC;IACH,CAAC;IAED,+CAA+C;IAC/C,IAAI,YAAY,IAAI,eAAe;QAAE,OAAO,IAAI,CAAC;IACjD,OAAO,eAAe,KAAK,CAAC,CAAC;AAC/B,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,OAAe;IAC/C,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,aAAa,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC;IAEjD,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,MAAM,gBAAgB,CAAC,GAAG,CAAC,CAAC;QAE7C,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACjB,OAAO;gBACL,KAAK,EAAE,KAAK;gBACZ,UAAU,EAAE,EAAE;gBACd,WAAW,EAAE,EAAE;gBACf,GAAG,EAAE,EAAE;gBACP,KAAK,EAAE,QAAQ,QAAQ,CAAC,MAAM,EAAE;aACjC,CAAC;QACJ,CAAC;QAED,MAAM,WAAW,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC;QAC/D,kEAAkE;QAClE,IAAI,WAAW,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;YACtC,OAAO;gBACL,KAAK,EAAE,KAAK;gBACZ,UAAU,EAAE,EAAE;gBACd,WAAW,EAAE,EAAE;gBACf,GAAG,EAAE,EAAE;gBACP,KAAK,EAAE,mCAAmC;aAC3C,CAAC;QACJ,CAAC;QAED,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;QAClC,MAAM,EAAE,UAAU,EAAE,WAAW,EAAE,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC;QAExD,OAAO;YACL,KAAK,EAAE,IAAI;YACX,UAAU;YACV,WAAW;YACX,GAAG;SACJ,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,CAAC;QACzE,OAAO;YACL,KAAK,EAAE,KAAK;YACZ,UAAU,EAAE,EAAE;YACd,WAAW,EAAE,EAAE;YACf,GAAG,EAAE,EAAE;YACP,KAAK,EAAE,OAAO;SACf,CAAC;IACJ,CAAC;AACH,CAAC"}

package/dist/recon/route-discovery.d.ts

@@ -0,0 +1,25 @@
+/**
+ * Route Discovery
+ *
+ * Builds a route graph from all recon sources: sitemap, HTML links,
+ * form actions. Deduplicates and assigns preliminary priorities using
+ * the scoring algorithm from the technical plan.
+ */
+import type { Route, SitemapEntry, RobotsDirective } from '../core/types/recon.js';
+export interface RouteDiscoveryInput {
+    /** Base URL of the site */
+    baseUrl: string;
+    /** Root page HTML for link extraction */
+    rootHtml: string;
+    /** Sitemap entries (if any) */
+    sitemapEntries: SitemapEntry[];
+    /** Robots.txt directives for filtering */
+    robotsDirectives: RobotsDirective[];
+    /** Whether to respect robots.txt disallow rules */
+    respectRobotsTxt: boolean;
+}
+/**
+ * Discover all routes from sitemap, HTML, and form actions.
+ * Deduplicates, classifies, scores, and sorts by priority.
+ */
+export declare function discoverRoutes(input: RouteDiscoveryInput): Route[];

package/dist/recon/route-discovery.js

@@ -0,0 +1,303 @@
+/**
+ * Route Discovery
+ *
+ * Builds a route graph from all recon sources: sitemap, HTML links,
+ * form actions. Deduplicates and assigns preliminary priorities using
+ * the scoring algorithm from the technical plan.
+ */
+import * as cheerio from 'cheerio';
+import { isPathAllowed } from './robots.js';
+/**
+ * Page type heuristics based on URL patterns
+ */
+const PAGE_TYPE_PATTERNS = [
+    { pattern: /\/(search|find|browse|discover)\b/i, type: 'search' },
+    { pattern: /[?&]q=/i, type: 'search' },
+    { pattern: /\/(results|listings|products|catalog)\b/i, type: 'listing-results' },
+    { pattern: /\/(product|item|detail)\//i, type: 'detail-page' },
+    { pattern: /\/(cart|basket)\b/i, type: 'checkout-cart' },
+    { pattern: /\/(checkout|pay|payment|purchase)\b/i, type: 'checkout-payment' },
+    { pattern: /\/(account|profile|my-?account|user)\b/i, type: 'account-management' },
+    { pattern: /\/(settings|preferences|config)\b/i, type: 'settings' },
+    { pattern: /\/(login|signin|sign-in|auth|register|signup|sign-up)\b/i, type: 'authentication' },
+    { pattern: /\/(dashboard|admin|overview)\b/i, type: 'dashboard' },
+    { pattern: /\/(contact|contact-us|reach-us)\b/i, type: 'contact' },
+    { pattern: /\/(help|support|faq|docs|documentation|knowledge-?base)\b/i, type: 'help-support' },
+    { pattern: /\/(blog|article|post|news|press)\b/i, type: 'content-article' },
+    { pattern: /\/(about|team|careers|company)\b/i, type: 'content-article' },
+];
+/**
+ * Page type scores for priority calculation
+ */
+const PAGE_TYPE_SCORES = {
+    search: 25,
+    'checkout-payment': 25,
+    'checkout-cart': 20,
+    'listing-results': 20,
+    homepage: 20,
+    'detail-page': 15,
+    'account-management': 15,
+    'form-submission': 15,
+    settings: 10,
+    authentication: 10,
+    dashboard: 10,
+    contact: 10,
+    'help-support': 5,
+    'content-article': 3,
+    unknown: 10,
+};
+/**
+ * Classify a URL into a page type based on URL patterns
+ */
+function classifyPageType(url) {
+    const pathname = new URL(url).pathname;
+    // Root path = homepage
+    if (pathname === '/' || pathname === '')
+        return 'homepage';
+    for (const { pattern, type } of PAGE_TYPE_PATTERNS) {
+        if (pattern.test(pathname))
+            return type;
+    }
+    return 'unknown';
+}
+/**
+ * Calculate priority score for a route (higher = more important)
+ */
+function calculatePriorityScore(route) {
+    let score = 0;
+    // Forms are the #1 signal for agentic value
+    if (route.hasForm)
+        score += 30;
+    // Page type scores
+    score += PAGE_TYPE_SCORES[route.estimatedPageType] ?? 0;
+    // Depth penalty (deeper = less important)
+    score -= route.depth * 3;
+    // Interactive elements bonus
+    if (route.hasInteractiveElements)
+        score += 10;
+    return Math.max(0, score);
+}
+/**
+ * Convert priority score to priority label
+ */
+function scoreToPriority(score) {
+    if (score >= 40)
+        return 'critical';
+    if (score >= 25)
+        return 'high';
+    if (score >= 15)
+        return 'medium';
+    if (score >= 5)
+        return 'low';
+    return 'skip';
+}
+/**
+ * Check if a URL belongs to the same origin
+ */
+function isSameOrigin(url, baseOrigin) {
+    try {
+        return new URL(url).origin === baseOrigin;
+    }
+    catch {
+        return false;
+    }
+}
+/**
+ * Normalize a URL for deduplication (strip trailing slash, hash, sort params)
+ */
+function normalizeUrl(url) {
+    try {
+        const parsed = new URL(url);
+        // Remove hash
+        parsed.hash = '';
+        // Sort search params for consistent comparison
+        parsed.searchParams.sort();
+        // Remove trailing slash (but keep root /)
+        let pathname = parsed.pathname;
+        if (pathname.length > 1 && pathname.endsWith('/')) {
+            pathname = pathname.slice(0, -1);
+        }
+        parsed.pathname = pathname;
+        return parsed.href;
+    }
+    catch {
+        return url;
+    }
+}
+/**
+ * Extract links from HTML that are same-origin
+ */
+function extractHtmlLinks(html, baseUrl) {
+    const $ = cheerio.load(html);
+    const baseOrigin = new URL(baseUrl).origin;
+    const links = [];
+    // Navigation links (<a> in nav, header)
+    $('nav a[href], header a[href]').each((_i, el) => {
+        const href = $(el).attr('href');
+        if (!href)
+            return;
+        try {
+            const resolved = new URL(href, baseUrl).href;
+            if (isSameOrigin(resolved, baseOrigin)) {
+                links.push({ url: resolved, source: 'navigation', hasForm: false });
+            }
+        }
+        catch {
+            // Invalid URL — skip
+        }
+    });
+    // All other links
+    $('a[href]').each((_i, el) => {
+        const href = $(el).attr('href');
+        if (!href)
+            return;
+        // Skip anchor-only, javascript:, mailto:, tel:
+        if (href.startsWith('#') || href.startsWith('javascript:') ||
+            href.startsWith('mailto:') || href.startsWith('tel:'))
+            return;
+        try {
+            const resolved = new URL(href, baseUrl).href;
+            if (isSameOrigin(resolved, baseOrigin)) {
+                links.push({ url: resolved, source: 'link', hasForm: false });
+            }
+        }
+        catch {
+            // Invalid URL — skip
+        }
+    });
+    // Form actions
+    $('form[action]').each((_i, el) => {
+        const action = $(el).attr('action');
+        if (!action)
+            return;
+        try {
+            const resolved = new URL(action, baseUrl).href;
+            if (isSameOrigin(resolved, baseOrigin)) {
+                links.push({ url: resolved, source: 'form-action', hasForm: true });
+            }
+        }
+        catch {
+            // Invalid URL — skip
+        }
+    });
+    return links;
+}
+/**
+ * Check if HTML contains form or interactive elements at a URL
+ * This is a lightweight heuristic — actual forms are detected during audit
+ */
+function htmlHasInteractiveElements(html) {
+    const $ = cheerio.load(html);
+    // Check for interactive elements beyond forms
+    return ($('button').length > 0 ||
+        $('[role="button"]').length > 0 ||
+        $('[onclick]').length > 0 ||
+        $('[data-action]').length > 0 ||
+        $('details').length > 0 ||
+        $('[role="tab"]').length > 0);
+}
+/**
+ * Discover all routes from sitemap, HTML, and form actions.
+ * Deduplicates, classifies, scores, and sorts by priority.
+ */
+export function discoverRoutes(input) {
+    const { baseUrl, rootHtml, sitemapEntries, robotsDirectives, respectRobotsTxt } = input;
+    const baseOrigin = new URL(baseUrl).origin;
+    // Map: normalizedUrl -> Route
+    const routeMap = new Map();
+    /**
+     * Add or update a route in the map. First source wins,
+     * but some properties are merged.
+     */
+    function addRoute(url, source, depth, extras) {
+        const normalized = normalizeUrl(url);
+        // Skip non-same-origin
+        if (!isSameOrigin(normalized, baseOrigin))
+            return;
+        // Respect robots.txt
+        if (respectRobotsTxt && robotsDirectives.length > 0) {
+            const pathname = new URL(normalized).pathname;
+            if (!isPathAllowed(pathname, robotsDirectives))
+                return;
+        }
+        // Skip known non-page resources
+        const pathname = new URL(normalized).pathname.toLowerCase();
+        if (/\.(jpg|jpeg|png|gif|svg|webp|ico|css|js|woff|woff2|ttf|eot|pdf|zip|tar|gz)$/i.test(pathname)) {
+            return;
+        }
+        const existing = routeMap.get(normalized);
+        if (existing) {
+            // Merge: use lower depth, merge sitemap data, merge hasForm
+            if (depth < existing.depth)
+                existing.depth = depth;
+            if (extras?.hasForm)
+                existing.hasForm = true;
+            if (extras?.lastmod && !existing.lastmod)
+                existing.lastmod = extras.lastmod;
+            if (extras?.changefreq && !existing.changefreq)
+                existing.changefreq = extras.changefreq;
+            if (extras?.sitemapPriority != null && existing.sitemapPriority == null) {
+                existing.sitemapPriority = extras.sitemapPriority;
+            }
+            return;
+        }
+        const estimatedPageType = classifyPageType(normalized);
+        const hasForm = extras?.hasForm ?? false;
+        const route = {
+            url: normalized,
+            source,
+            depth,
+            hasForm,
+            hasInteractiveElements: false, // Updated later from HTML analysis
+            estimatedPageType,
+            priority: 'medium', // Calculated after all properties set
+            priorityScore: 0,
+            lastmod: extras?.lastmod,
+            changefreq: extras?.changefreq,
+            sitemapPriority: extras?.sitemapPriority,
+        };
+        routeMap.set(normalized, route);
+    }
+    // 1. Add root URL
+    addRoute(baseUrl, 'navigation', 0, { hasForm: false });
+    // 2. Add sitemap entries
+    for (const entry of sitemapEntries) {
+        try {
+            const entryUrl = new URL(entry.loc, baseUrl).href;
+            addRoute(entryUrl, 'sitemap', 1, {
+                lastmod: entry.lastmod,
+                changefreq: entry.changefreq,
+                sitemapPriority: entry.priority,
+            });
+        }
+        catch {
+            // Invalid sitemap URL — skip
+        }
+    }
+    // 3. Extract links from root HTML
+    const htmlLinks = extractHtmlLinks(rootHtml, baseUrl);
+    for (const link of htmlLinks) {
+        addRoute(link.url, link.source, 1, { hasForm: link.hasForm });
+    }
+    // 4. Check root HTML for interactive elements
+    const rootHasInteractive = htmlHasInteractiveElements(rootHtml);
+    const rootNormalized = normalizeUrl(baseUrl);
+    const rootRoute = routeMap.get(rootNormalized);
+    if (rootRoute) {
+        rootRoute.hasInteractiveElements = rootHasInteractive;
+        // Root page likely has forms if the HTML has them
+        const $ = cheerio.load(rootHtml);
+        if ($('form').length > 0) {
+            rootRoute.hasForm = true;
+        }
+    }
+    // 5. Calculate priority scores for all routes
+    for (const route of routeMap.values()) {
+        route.priorityScore = calculatePriorityScore(route);
+        route.priority = scoreToPriority(route.priorityScore);
+    }
+    // 6. Sort by priority score (descending)
+    const routes = [...routeMap.values()].sort((a, b) => b.priorityScore - a.priorityScore);
+    return routes;
+}
+//# sourceMappingURL=route-discovery.js.map

package/dist/recon/route-discovery.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"route-discovery.js","sourceRoot":"","sources":["../../src/recon/route-discovery.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AASnC,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAE5C;;GAEG;AACH,MAAM,kBAAkB,GAA0C;IAChE,EAAE,OAAO,EAAE,oCAAoC,EAAE,IAAI,EAAE,QAAQ,EAAE;IACjE,EAAE,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE,QAAQ,EAAE;IACtC,EAAE,OAAO,EAAE,0CAA0C,EAAE,IAAI,EAAE,iBAAiB,EAAE;IAChF,EAAE,OAAO,EAAE,4BAA4B,EAAE,IAAI,EAAE,aAAa,EAAE;IAC9D,EAAE,OAAO,EAAE,oBAAoB,EAAE,IAAI,EAAE,eAAe,EAAE;IACxD,EAAE,OAAO,EAAE,sCAAsC,EAAE,IAAI,EAAE,kBAAkB,EAAE;IAC7E,EAAE,OAAO,EAAE,yCAAyC,EAAE,IAAI,EAAE,oBAAoB,EAAE;IAClF,EAAE,OAAO,EAAE,oCAAoC,EAAE,IAAI,EAAE,UAAU,EAAE;IACnE,EAAE,OAAO,EAAE,0DAA0D,EAAE,IAAI,EAAE,gBAAgB,EAAE;IAC/F,EAAE,OAAO,EAAE,iCAAiC,EAAE,IAAI,EAAE,WAAW,EAAE;IACjE,EAAE,OAAO,EAAE,oCAAoC,EAAE,IAAI,EAAE,SAAS,EAAE;IAClE,EAAE,OAAO,EAAE,4DAA4D,EAAE,IAAI,EAAE,cAAc,EAAE;IAC/F,EAAE,OAAO,EAAE,qCAAqC,EAAE,IAAI,EAAE,iBAAiB,EAAE;IAC3E,EAAE,OAAO,EAAE,mCAAmC,EAAE,IAAI,EAAE,iBAAiB,EAAE;CAC1E,CAAC;AAEF;;GAEG;AACH,MAAM,gBAAgB,GAA6B;IACjD,MAAM,EAAE,EAAE;IACV,kBAAkB,EAAE,EAAE;IACtB,eAAe,EAAE,EAAE;IACnB,iBAAiB,EAAE,EAAE;IACrB,QAAQ,EAAE,EAAE;IACZ,aAAa,EAAE,EAAE;IACjB,oBAAoB,EAAE,EAAE;IACxB,iBAAiB,EAAE,EAAE;IACrB,QAAQ,EAAE,EAAE;IACZ,cAAc,EAAE,EAAE;IAClB,SAAS,EAAE,EAAE;IACb,OAAO,EAAE,EAAE;IACX,cAAc,EAAE,CAAC;IACjB,iBAAiB,EAAE,CAAC;IACpB,OAAO,EAAE,EAAE;CACZ,CAAC;AAEF;;GAEG;AACH,SAAS,gBAAgB,CAAC,GAAW;IACnC,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC;IAEvC,uBAAuB;IACvB,IAAI,QAAQ,KAAK,GAAG,IAAI,QAAQ,KAAK,EAAE;QAAE,OAAO,UAAU,CAAC;IAE3D,KAAK,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,kBAAkB,EAAE,CAAC;QACnD,IAAI,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC;YAAE,OAAO,IAAI,CAAC;IAC1C,CAAC;IAED,OAAO,SAAS,CAAC;AACnB,CAAC;AAED;;GAEG;AACH,SAAS,sBAAsB,CAAC,KAAwF;IACtH,IAAI,KAAK,GAAG,CAAC,CAAC;IAEd,4CAA4C;IAC5C,IAAI,KAAK,CAAC,OAAO;QAAE,KAAK,IAAI,EAAE,CAAC;IAE/B,mBAAmB;IACnB,KAAK,IAAI,gBAAgB,CAAC,KAAK,CAAC,iBAAiB,CAAC,IAAI,CAAC,CAAC;IAExD,0CAA0C;IAC1C,KAAK,IAAI,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC;IAEzB,6BAA6B;IAC7B,IAAI,KAAK,CAAC,sBAAsB;QAAE,KAAK,IAAI,EAAE,CAAC;IAE9C,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;AAC5B,CAAC;AAED;;GAEG;AACH,SAAS,eAAe,CAAC,KAAa;IACpC,IAAI,KAAK,IAAI,EAAE;QAAE,OAAO,UAAU,CAAC;IACnC,IAAI,KAAK,IAAI,EAAE;QAAE,OAAO,MAAM,CAAC;IAC/B,IAAI,KAAK,IAAI,EAAE;QAAE,OAAO,QAAQ,CAAC;IACjC,IAAI,KAAK,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC;IAC7B,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;GAEG;AACH,SAAS,YAAY,CAAC,GAAW,EAAE,UAAkB;IACnD,IAAI,CAAC;QACH,OAAO,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,MAAM,KAAK,UAAU,CAAC;IAC5C,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED;;GAEG;AACH,SAAS,YAAY,CAAC,GAAW;IAC/B,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAC5B,cAAc;QACd,MAAM,CAAC,IAAI,GAAG,EAAE,CAAC;QACjB,+CAA+C;QAC/C,MAAM,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC;QAC3B,0CAA0C;QAC1C,IAAI,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC;QAC/B,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;YAClD,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QACnC,CAAC;QACD,MAAM,CAAC,QAAQ,GAAG,QAAQ,CAAC;QAC3B,OAAO,MAAM,CAAC,IAAI,CAAC;IACrB,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,GAAG,CAAC;IACb,CAAC;AACH,CAAC;AAED;;GAEG;AACH,SAAS,gBAAgB,CACvB,IAAY,EACZ,OAAe;IAEf,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC7B,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC;IAC3C,MAAM,KAAK,GAA6D,EAAE,CAAC;IAE3E,wCAAwC;IACxC,CAAC,CAAC,6BAA6B,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE;QAC/C,MAAM,IAAI,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAChC,IAAI,CAAC,IAAI;YAAE,OAAO;QAClB,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC;YAC7C,IAAI,YAAY,CAAC,QAAQ,EAAE,UAAU,CAAC,EAAE,CAAC;gBACvC,KAAK,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,QAAQ,EAAE,MAAM,EAAE,YAAY,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC;YACtE,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,qBAAqB;QACvB,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,kBAAkB;IAClB,CAAC,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE;QAC3B,MAAM,IAAI,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAChC,IAAI,CAAC,IAAI;YAAE,OAAO;QAClB,+CAA+C;QAC/C,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC;YACtD,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC;YAAE,OAAO;QAClE,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC;YAC7C,IAAI,YAAY,CAAC,QAAQ,EAAE,UAAU,CAAC,EAAE,CAAC;gBACvC,KAAK,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC;YAChE,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,qBAAqB;QACvB,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,eAAe;IACf,CAAC,CAAC,cAAc,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE;QAChC,MAAM,MAAM,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACpC,IAAI,CAAC,MAAM;YAAE,OAAO;QACpB,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC;YAC/C,IAAI,YAAY,CAAC,QAAQ,EAAE,UAAU,CAAC,EAAE,CAAC;gBACvC,KAAK,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,QAAQ,EAAE,MAAM,EAAE,aAAa,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;YACtE,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,qBAAqB;QACvB,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;GAGG;AACH,SAAS,0BAA0B,CAAC,IAAY;IAC9C,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC7B,8CAA8C;IAC9C,OAAO,CACL,CAAC,CAAC,QAAQ,CAAC,CAAC,MAAM,GAAG,CAAC;QACtB,CAAC,CAAC,iBAAiB,CAAC,CAAC,MAAM,GAAG,CAAC;QAC/B,CAAC,CAAC,WAAW,CAAC,CAAC,MAAM,GAAG,CAAC;QACzB,CAAC,CAAC,eAAe,CAAC,CAAC,MAAM,GAAG,CAAC;QAC7B,CAAC,CAAC,SAAS,CAAC,CAAC,MAAM,GAAG,CAAC;QACvB,CAAC,CAAC,cAAc,CAAC,CAAC,MAAM,GAAG,CAAC,CAC7B,CAAC;AACJ,CAAC;AAmBD;;;GAGG;AACH,MAAM,UAAU,cAAc,CAAC,KAA0B;IACvD,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,cAAc,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,GAAG,KAAK,CAAC;IACxF,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC;IAE3C,8BAA8B;IAC9B,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAiB,CAAC;IAE1C;;;OAGG;IACH,SAAS,QAAQ,CACf,GAAW,EACX,MAAmB,EACnB,KAAa,EACb,MAKC;QAED,MAAM,UAAU,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC;QAErC,uBAAuB;QACvB,IAAI,CAAC,YAAY,CAAC,UAAU,EAAE,UAAU,CAAC;YAAE,OAAO;QAElD,qBAAqB;QACrB,IAAI,gBAAgB,IAAI,gBAAgB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACpD,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,CAAC,QAAQ,CAAC;YAC9C,IAAI,CAAC,aAAa,CAAC,QAAQ,EAAE,gBAAgB,CAAC;gBAAE,OAAO;QACzD,CAAC;QAED,gCAAgC;QAChC,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;QAC5D,IAAI,8EAA8E,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;YAClG,OAAO;QACT,CAAC;QAED,MAAM,QAAQ,GAAG,QAAQ,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QAC1C,IAAI,QAAQ,EAAE,CAAC;YACb,4DAA4D;YAC5D,IAAI,KAAK,GAAG,QAAQ,CAAC,KAAK;gBAAE,QAAQ,CAAC,KAAK,GAAG,KAAK,CAAC;YACnD,IAAI,MAAM,EAAE,OAAO;gBAAE,QAAQ,CAAC,OAAO,GAAG,IAAI,CAAC;YAC7C,IAAI,MAAM,EAAE,OAAO,IAAI,CAAC,QAAQ,CAAC,OAAO;gBAAE,QAAQ,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC;YAC5E,IAAI,MAAM,EAAE,UAAU,IAAI,CAAC,QAAQ,CAAC,UAAU;gBAAE,QAAQ,CAAC,UAAU,GAAG,MAAM,CAAC,UAAU,CAAC;YACxF,IAAI,MAAM,EAAE,eAAe,IAAI,IAAI,IAAI,QAAQ,CAAC,eAAe,IAAI,IAAI,EAAE,CAAC;gBACxE,QAAQ,CAAC,eAAe,GAAG,MAAM,CAAC,eAAe,CAAC;YACpD,CAAC;YACD,OAAO;QACT,CAAC;QAED,MAAM,iBAAiB,GAAG,gBAAgB,CAAC,UAAU,CAAC,CAAC;QACvD,MAAM,OAAO,GAAG,MAAM,EAAE,OAAO,IAAI,KAAK,CAAC;QAEzC,MAAM,KAAK,GAAU;YACnB,GAAG,EAAE,UAAU;YACf,MAAM;YACN,KAAK;YACL,OAAO;YACP,sBAAsB,EAAE,KAAK,EAAE,mCAAmC;YAClE,iBAAiB;YACjB,QAAQ,EAAE,QAAQ,EAAE,sCAAsC;YAC1D,aAAa,EAAE,CAAC;YAChB,OAAO,EAAE,MAAM,EAAE,OAAO;YACxB,UAAU,EAAE,MAAM,EAAE,UAAU;YAC9B,eAAe,EAAE,MAAM,EAAE,eAAe;SACzC,CAAC;QAEF,QAAQ,CAAC,GAAG,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC;IAClC,CAAC;IAED,kBAAkB;IAClB,QAAQ,CAAC,OAAO,EAAE,YAAY,EAAE,CAAC,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC;IAEvD,yBAAyB;IACzB,KAAK,MAAM,KAAK,IAAI,cAAc,EAAE,CAAC;QACnC,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC;YAClD,QAAQ,CAAC,QAAQ,EAAE,SAAS,EAAE,CAAC,EAAE;gBAC/B,OAAO,EAAE,KAAK,CAAC,OAAO;gBACtB,UAAU,EAAE,KAAK,CAAC,UAAU;gBAC5B,eAAe,EAAE,KAAK,CAAC,QAAQ;aAChC,CAAC,CAAC;QACL,CAAC;QAAC,MAAM,CAAC;YACP,6BAA6B;QAC/B,CAAC;IACH,CAAC;IAED,kCAAkC;IAClC,MAAM,SAAS,GAAG,gBAAgB,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IACtD,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;QAC7B,QAAQ,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;IAChE,CAAC;IAED,8CAA8C;IAC9C,MAAM,kBAAkB,GAAG,0BAA0B,CAAC,QAAQ,CAAC,CAAC;IAChE,MAAM,cAAc,GAAG,YAAY,CAAC,OAAO,CAAC,CAAC;IAC7C,MAAM,SAAS,GAAG,QAAQ,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;IAC/C,IAAI,SAAS,EAAE,CAAC;QACd,SAAS,CAAC,sBAAsB,GAAG,kBAAkB,CAAC;QACtD,kDAAkD;QAClD,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACjC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACzB,SAAS,CAAC,OAAO,GAAG,IAAI,CAAC;QAC3B,CAAC;IACH,CAAC;IAED,8CAA8C;IAC9C,KAAK,MAAM,KAAK,IAAI,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAC;QACtC,KAAK,CAAC,aAAa,GAAG,sBAAsB,CAAC,KAAK,CAAC,CAAC;QACpD,KAAK,CAAC,QAAQ,GAAG,eAAe,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;IACxD,CAAC;IAED,yCAAyC;IACzC,MAAM,MAAM,GAAG,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,CACxC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,aAAa,GAAG,CAAC,CAAC,aAAa,CAC5C,CAAC;IAEF,OAAO,MAAM,CAAC;AAChB,CAAC"}

package/dist/recon/sitemap.d.ts

@@ -0,0 +1,12 @@
+/**
+ * Sitemap Parser
+ *
+ * Fetches and parses sitemap.xml and sitemap index files.
+ * Uses native fetch. Handles missing/malformed sitemaps gracefully.
+ */
+import type { SitemapResult } from '../core/types/recon.js';
+/**
+ * Fetch and parse a sitemap from a URL.
+ * Handles sitemap index files by recursively fetching children.
+ */
+export declare function parseSitemap(baseUrl: string, sitemapUrl?: string): Promise<SitemapResult>;