webmcp-cli 1.2.2 → 1.2.3

package/dist/recon/index.js

@@ -0,0 +1,143 @@
+/**
+ * Reconnaissance Module
+ *
+ * Orchestrates all recon sub-tasks: sitemap, robots.txt, manifest,
+ * meta extraction, and route discovery. Runs as many tasks in parallel
+ * as possible to minimize wall-clock time.
+ */
+import { parseSitemap } from './sitemap.js';
+import { parseRobots } from './robots.js';
+import { parseManifest } from './manifest.js';
+import { extractMeta } from './meta-extractor.js';
+import { discoverRoutes } from './route-discovery.js';
+/** Fetch timeout (ms) */
+const ROOT_FETCH_TIMEOUT_MS = 15_000;
+/**
+ * Fetch the root page HTML via plain HTTP (no browser needed).
+ * Returns the resolved URL (after redirects), status, response time, and HTML.
+ */
+async function fetchRootPage(url) {
+    const controller = new AbortController();
+    const timer = setTimeout(() => controller.abort(), ROOT_FETCH_TIMEOUT_MS);
+    const start = Date.now();
+    try {
+        const response = await fetch(url, {
+            signal: controller.signal,
+            headers: {
+                'User-Agent': 'Mozilla/5.0 (compatible; WebMCP-CLI/1.0; +https://webmcp.org)',
+                Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+            },
+            redirect: 'follow',
+        });
+        const responseTime = Date.now() - start;
+        const html = await response.text();
+        return {
+            resolvedUrl: response.url,
+            statusCode: response.status,
+            responseTime,
+            html,
+        };
+    }
+    finally {
+        clearTimeout(timer);
+    }
+}
+/**
+ * Run full reconnaissance on a target URL.
+ *
+ * Gathers sitemap, robots.txt, manifest, meta tags, and discovers
+ * routes — all without launching a browser.
+ */
+export async function runReconnaissance(url) {
+    const errors = [];
+    // Step 1: Fetch root page HTML + robots + sitemap in parallel
+    // (robots.txt is needed for route filtering, and may reference sitemaps)
+    const [rootResult, robotsResult] = await Promise.allSettled([
+        fetchRootPage(url),
+        parseRobots(url),
+    ]);
+    // Handle root page failure (fatal)
+    if (rootResult.status === 'rejected') {
+        const errMsg = rootResult.reason instanceof Error
+            ? rootResult.reason.message
+            : 'Failed to fetch root page';
+        return {
+            url,
+            resolvedUrl: url,
+            statusCode: 0,
+            responseTime: 0,
+            sitemap: null,
+            robots: null,
+            manifest: null,
+            meta: null,
+            discoveredRoutes: [],
+            rootHtml: '',
+            errors: [errMsg],
+        };
+    }
+    const { resolvedUrl, statusCode, responseTime, html } = rootResult.value;
+    const robots = robotsResult.status === 'fulfilled'
+        ? robotsResult.value
+        : null;
+    if (robotsResult.status === 'rejected') {
+        errors.push(`robots.txt: ${robotsResult.reason instanceof Error ? robotsResult.reason.message : 'Failed'}`);
+    }
+    // Step 2: Determine sitemap URL(s)
+    // Check robots.txt for sitemap references first
+    const sitemapUrlsFromRobots = robots?.sitemapUrls ?? [];
+    // Step 3: Fetch sitemap + manifest + extract meta in parallel
+    const [sitemapResult, manifestResult] = await Promise.allSettled([
+        sitemapUrlsFromRobots.length > 0
+            ? parseSitemap(url, sitemapUrlsFromRobots[0])
+            : parseSitemap(url),
+        parseManifest(url, html),
+    ]);
+    const sitemap = sitemapResult.status === 'fulfilled'
+        ? sitemapResult.value
+        : null;
+    const manifest = manifestResult.status === 'fulfilled'
+        ? manifestResult.value
+        : null;
+    if (sitemapResult.status === 'rejected') {
+        errors.push(`sitemap: ${sitemapResult.reason instanceof Error ? sitemapResult.reason.message : 'Failed'}`);
+    }
+    if (manifestResult.status === 'rejected') {
+        errors.push(`manifest: ${manifestResult.reason instanceof Error ? manifestResult.reason.message : 'Failed'}`);
+    }
+    // Step 4: Extract meta (synchronous cheerio operation)
+    let meta = null;
+    try {
+        meta = extractMeta(html, resolvedUrl);
+    }
+    catch (err) {
+        errors.push(`meta: ${err instanceof Error ? err.message : 'Failed'}`);
+    }
+    // Step 5: Discover and score routes
+    const discoveredRoutes = discoverRoutes({
+        baseUrl: resolvedUrl,
+        rootHtml: html,
+        sitemapEntries: sitemap?.entries ?? [],
+        robotsDirectives: robots?.directives ?? [],
+        respectRobotsTxt: true,
+    });
+    return {
+        url,
+        resolvedUrl,
+        statusCode,
+        responseTime,
+        sitemap,
+        robots,
+        manifest,
+        meta,
+        discoveredRoutes,
+        rootHtml: html,
+        errors,
+    };
+}
+// Re-export sub-modules for direct use
+export { parseSitemap } from './sitemap.js';
+export { parseRobots, isPathAllowed } from './robots.js';
+export { parseManifest } from './manifest.js';
+export { extractMeta } from './meta-extractor.js';
+export { discoverRoutes } from './route-discovery.js';
+//# sourceMappingURL=index.js.map

package/dist/recon/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/recon/index.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAGH,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAC1C,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAC9C,OAAO,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAClD,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAEtD,yBAAyB;AACzB,MAAM,qBAAqB,GAAG,MAAM,CAAC;AAErC;;;GAGG;AACH,KAAK,UAAU,aAAa,CAAC,GAAW;IAMtC,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;IACzC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,qBAAqB,CAAC,CAAC;IAE1E,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACzB,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;YAChC,MAAM,EAAE,UAAU,CAAC,MAAM;YACzB,OAAO,EAAE;gBACP,YAAY,EACV,+DAA+D;gBACjE,MAAM,EACJ,iEAAiE;aACpE;YACD,QAAQ,EAAE,QAAQ;SACnB,CAAC,CAAC;QACH,MAAM,YAAY,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC;QACxC,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;QAEnC,OAAO;YACL,WAAW,EAAE,QAAQ,CAAC,GAAG;YACzB,UAAU,EAAE,QAAQ,CAAC,MAAM;YAC3B,YAAY;YACZ,IAAI;SACL,CAAC;IACJ,CAAC;YAAS,CAAC;QACT,YAAY,CAAC,KAAK,CAAC,CAAC;IACtB,CAAC;AACH,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,GAAW;IACjD,MAAM,MAAM,GAAa,EAAE,CAAC;IAE5B,8DAA8D;IAC9D,yEAAyE;IACzE,MAAM,CAAC,UAAU,EAAE,YAAY,CAAC,GAAG,MAAM,OAAO,CAAC,UAAU,CAAC;QAC1D,aAAa,CAAC,GAAG,CAAC;QAClB,WAAW,CAAC,GAAG,CAAC;KACjB,CAAC,CAAC;IAEH,mCAAmC;IACnC,IAAI,UAAU,CAAC,MAAM,KAAK,UAAU,EAAE,CAAC;QACrC,MAAM,MAAM,GAAG,UAAU,CAAC,MAAM,YAAY,KAAK;YAC/C,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC,OAAO;YAC3B,CAAC,CAAC,2BAA2B,CAAC;QAChC,OAAO;YACL,GAAG;YACH,WAAW,EAAE,GAAG;YAChB,UAAU,EAAE,CAAC;YACb,YAAY,EAAE,CAAC;YACf,OAAO,EAAE,IAAI;YACb,MAAM,EAAE,IAAI;YACZ,QAAQ,EAAE,IAAI;YACd,IAAI,EAAE,IAAI;YACV,gBAAgB,EAAE,EAAE;YACpB,QAAQ,EAAE,EAAE;YACZ,MAAM,EAAE,CAAC,MAAM,CAAC;SACjB,CAAC;IACJ,CAAC;IAED,MAAM,EAAE,WAAW,EAAE,UAAU,EAAE,YAAY,EAAE,IAAI,EAAE,GAAG,UAAU,CAAC,KAAK,CAAC;IACzE,MAAM,MAAM,GAAG,YAAY,CAAC,MAAM,KAAK,WAAW;QAChD,CAAC,CAAC,YAAY,CAAC,KAAK;QACpB,CAAC,CAAC,IAAI,CAAC;IAET,IAAI,YAAY,CAAC,MAAM,KAAK,UAAU,EAAE,CAAC;QACvC,MAAM,CAAC,IAAI,CAAC,eAAe,YAAY,CAAC,MAAM,YAAY,KAAK,CAAC,CAAC,CAAC,YAAY,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC;IAC9G,CAAC;IAED,mCAAmC;IACnC,gDAAgD;IAChD,MAAM,qBAAqB,GAAG,MAAM,EAAE,WAAW,IAAI,EAAE,CAAC;IAExD,8DAA8D;IAC9D,MAAM,CAAC,aAAa,EAAE,cAAc,CAAC,GAAG,MAAM,OAAO,CAAC,UAAU,CAAC;QAC/D,qBAAqB,CAAC,MAAM,GAAG,CAAC;YAC9B,CAAC,CAAC,YAAY,CAAC,GAAG,EAAE,qBAAqB,CAAC,CAAC,CAAC,CAAC;YAC7C,CAAC,CAAC,YAAY,CAAC,GAAG,CAAC;QACrB,aAAa,CAAC,GAAG,EAAE,IAAI,CAAC;KACzB,CAAC,CAAC;IAEH,MAAM,OAAO,GAAG,aAAa,CAAC,MAAM,KAAK,WAAW;QAClD,CAAC,CAAC,aAAa,CAAC,KAAK;QACrB,CAAC,CAAC,IAAI,CAAC;IACT,MAAM,QAAQ,GAAG,cAAc,CAAC,MAAM,KAAK,WAAW;QACpD,CAAC,CAAC,cAAc,CAAC,KAAK;QACtB,CAAC,CAAC,IAAI,CAAC;IAET,IAAI,aAAa,CAAC,MAAM,KAAK,UAAU,EAAE,CAAC;QACxC,MAAM,CAAC,IAAI,CAAC,YAAY,aAAa,CAAC,MAAM,YAAY,KAAK,CAAC,CAAC,CAAC,aAAa,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC;IAC7G,CAAC;IACD,IAAI,cAAc,CAAC,MAAM,KAAK,UAAU,EAAE,CAAC;QACzC,MAAM,CAAC,IAAI,CAAC,aAAa,cAAc,CAAC,MAAM,YAAY,KAAK,CAAC,CAAC,CAAC,cAAc,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC;IAChH,CAAC;IAED,uDAAuD;IACvD,IAAI,IAAI,GAAG,IAAI,CAAC;IAChB,IAAI,CAAC;QACH,IAAI,GAAG,WAAW,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC;IACxC,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,CAAC,IAAI,CAAC,SAAS,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC;IACxE,CAAC;IAED,oCAAoC;IACpC,MAAM,gBAAgB,GAAG,cAAc,CAAC;QACtC,OAAO,EAAE,WAAW;QACpB,QAAQ,EAAE,IAAI;QACd,cAAc,EAAE,OAAO,EAAE,OAAO,IAAI,EAAE;QACtC,gBAAgB,EAAE,MAAM,EAAE,UAAU,IAAI,EAAE;QAC1C,gBAAgB,EAAE,IAAI;KACvB,CAAC,CAAC;IAEH,OAAO;QACL,GAAG;QACH,WAAW;QACX,UAAU;QACV,YAAY;QACZ,OAAO;QACP,MAAM;QACN,QAAQ;QACR,IAAI;QACJ,gBAAgB;QAChB,QAAQ,EAAE,IAAI;QACd,MAAM;KACP,CAAC;AACJ,CAAC;AAED,uCAAuC;AACvC,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAAE,WAAW,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AACzD,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAC9C,OAAO,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAClD,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC"}

package/dist/recon/manifest.d.ts

@@ -0,0 +1,16 @@
+/**
+ * Web App Manifest Parser
+ *
+ * Extracts and parses the web app manifest from <link rel="manifest">
+ * or the default /manifest.json path. Handles missing manifests gracefully.
+ */
+import type { ManifestResult } from '../core/types/recon.js';
+/**
+ * Fetch and parse the web app manifest for a site.
+ *
+ * Strategy:
+ * 1. If rootHtml provided, look for <link rel="manifest"> href
+ * 2. Otherwise try /manifest.json
+ * 3. Try /manifest.webmanifest as fallback
+ */
+export declare function parseManifest(baseUrl: string, rootHtml?: string): Promise<ManifestResult>;

package/dist/recon/manifest.js

@@ -0,0 +1,108 @@
+/**
+ * Web App Manifest Parser
+ *
+ * Extracts and parses the web app manifest from <link rel="manifest">
+ * or the default /manifest.json path. Handles missing manifests gracefully.
+ */
+import * as cheerio from 'cheerio';
+/** Fetch timeout (ms) */
+const FETCH_TIMEOUT_MS = 10_000;
+/**
+ * Fetch with timeout using AbortController
+ */
+async function fetchWithTimeout(url, timeoutMs = FETCH_TIMEOUT_MS) {
+    const controller = new AbortController();
+    const timer = setTimeout(() => controller.abort(), timeoutMs);
+    try {
+        return await fetch(url, {
+            signal: controller.signal,
+            headers: { 'User-Agent': 'WebMCP-CLI/1.0 (manifest-parser)' },
+            redirect: 'follow',
+        });
+    }
+    finally {
+        clearTimeout(timer);
+    }
+}
+/**
+ * Extract manifest URL from HTML <link rel="manifest"> tag
+ */
+function extractManifestUrl(html, baseUrl) {
+    const $ = cheerio.load(html);
+    const href = $('link[rel="manifest"]').attr('href');
+    if (!href)
+        return null;
+    try {
+        return new URL(href, baseUrl).href;
+    }
+    catch {
+        return null;
+    }
+}
+/**
+ * Fetch and parse the web app manifest for a site.
+ *
+ * Strategy:
+ * 1. If rootHtml provided, look for <link rel="manifest"> href
+ * 2. Otherwise try /manifest.json
+ * 3. Try /manifest.webmanifest as fallback
+ */
+export async function parseManifest(baseUrl, rootHtml) {
+    // Build list of candidate URLs to try
+    const candidates = [];
+    if (rootHtml) {
+        const fromLink = extractManifestUrl(rootHtml, baseUrl);
+        if (fromLink)
+            candidates.push(fromLink);
+    }
+    candidates.push(new URL('/manifest.json', baseUrl).href, new URL('/manifest.webmanifest', baseUrl).href);
+    // Deduplicate
+    const uniqueCandidates = [...new Set(candidates)];
+    for (const url of uniqueCandidates) {
+        try {
+            const response = await fetchWithTimeout(url);
+            if (!response.ok)
+                continue;
+            const contentType = response.headers.get('content-type') ?? '';
+            // Skip HTML responses (custom 404 pages)
+            if (contentType.includes('text/html'))
+                continue;
+            const text = await response.text();
+            let data;
+            try {
+                data = JSON.parse(text);
+            }
+            catch {
+                continue; // Not valid JSON
+            }
+            return {
+                found: true,
+                name: typeof data['name'] === 'string' ? data['name'] : undefined,
+                shortName: typeof data['short_name'] === 'string'
+                    ? data['short_name']
+                    : undefined,
+                description: typeof data['description'] === 'string'
+                    ? data['description']
+                    : undefined,
+                startUrl: typeof data['start_url'] === 'string'
+                    ? data['start_url']
+                    : undefined,
+                display: typeof data['display'] === 'string' ? data['display'] : undefined,
+                themeColor: typeof data['theme_color'] === 'string'
+                    ? data['theme_color']
+                    : undefined,
+                source: url,
+                raw: data,
+            };
+        }
+        catch {
+            // Try next candidate
+            continue;
+        }
+    }
+    return {
+        found: false,
+        error: 'No manifest found',
+    };
+}
+//# sourceMappingURL=manifest.js.map

package/dist/recon/manifest.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"manifest.js","sourceRoot":"","sources":["../../src/recon/manifest.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AAGnC,yBAAyB;AACzB,MAAM,gBAAgB,GAAG,MAAM,CAAC;AAEhC;;GAEG;AACH,KAAK,UAAU,gBAAgB,CAC7B,GAAW,EACX,YAAoB,gBAAgB;IAEpC,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;IACzC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,SAAS,CAAC,CAAC;IAC9D,IAAI,CAAC;QACH,OAAO,MAAM,KAAK,CAAC,GAAG,EAAE;YACtB,MAAM,EAAE,UAAU,CAAC,MAAM;YACzB,OAAO,EAAE,EAAE,YAAY,EAAE,kCAAkC,EAAE;YAC7D,QAAQ,EAAE,QAAQ;SACnB,CAAC,CAAC;IACL,CAAC;YAAS,CAAC;QACT,YAAY,CAAC,KAAK,CAAC,CAAC;IACtB,CAAC;AACH,CAAC;AAED;;GAEG;AACH,SAAS,kBAAkB,CAAC,IAAY,EAAE,OAAe;IACvD,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC7B,MAAM,IAAI,GAAG,CAAC,CAAC,sBAAsB,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACpD,IAAI,CAAC,IAAI;QAAE,OAAO,IAAI,CAAC;IAEvB,IAAI,CAAC;QACH,OAAO,IAAI,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC;IACrC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,OAAe,EACf,QAAiB;IAEjB,sCAAsC;IACtC,MAAM,UAAU,GAAa,EAAE,CAAC;IAEhC,IAAI,QAAQ,EAAE,CAAC;QACb,MAAM,QAAQ,GAAG,kBAAkB,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QACvD,IAAI,QAAQ;YAAE,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IAC1C,CAAC;IAED,UAAU,CAAC,IAAI,CACb,IAAI,GAAG,CAAC,gBAAgB,EAAE,OAAO,CAAC,CAAC,IAAI,EACvC,IAAI,GAAG,CAAC,uBAAuB,EAAE,OAAO,CAAC,CAAC,IAAI,CAC/C,CAAC;IAEF,cAAc;IACd,MAAM,gBAAgB,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC;IAElD,KAAK,MAAM,GAAG,IAAI,gBAAgB,EAAE,CAAC;QACnC,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,gBAAgB,CAAC,GAAG,CAAC,CAAC;YAC7C,IAAI,CAAC,QAAQ,CAAC,EAAE;gBAAE,SAAS;YAE3B,MAAM,WAAW,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC;YAC/D,yCAAyC;YACzC,IAAI,WAAW,CAAC,QAAQ,CAAC,WAAW,CAAC;gBAAE,SAAS;YAEhD,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;YACnC,IAAI,IAA6B,CAAC;YAClC,IAAI,CAAC;gBACH,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAA4B,CAAC;YACrD,CAAC;YAAC,MAAM,CAAC;gBACP,SAAS,CAAC,iBAAiB;YAC7B,CAAC;YAED,OAAO;gBACL,KAAK,EAAE,IAAI;gBACX,IAAI,EAAE,OAAO,IAAI,CAAC,MAAM,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,SAAS;gBACjE,SAAS,EACP,OAAO,IAAI,CAAC,YAAY,CAAC,KAAK,QAAQ;oBACpC,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC;oBACpB,CAAC,CAAC,SAAS;gBACf,WAAW,EACT,OAAO,IAAI,CAAC,aAAa,CAAC,KAAK,QAAQ;oBACrC,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC;oBACrB,CAAC,CAAC,SAAS;gBACf,QAAQ,EACN,OAAO,IAAI,CAAC,WAAW,CAAC,KAAK,QAAQ;oBACnC,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC;oBACnB,CAAC,CAAC,SAAS;gBACf,OAAO,EACL,OAAO,IAAI,CAAC,SAAS,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS;gBACnE,UAAU,EACR,OAAO,IAAI,CAAC,aAAa,CAAC,KAAK,QAAQ;oBACrC,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC;oBACrB,CAAC,CAAC,SAAS;gBACf,MAAM,EAAE,GAAG;gBACX,GAAG,EAAE,IAAI;aACV,CAAC;QACJ,CAAC;QAAC,MAAM,CAAC;YACP,qBAAqB;YACrB,SAAS;QACX,CAAC;IACH,CAAC;IAED,OAAO;QACL,KAAK,EAAE,KAAK;QACZ,KAAK,EAAE,mBAAmB;KAC3B,CAAC;AACJ,CAAC"}

package/dist/recon/meta-extractor.d.ts

@@ -0,0 +1,11 @@
+/**
+ * Meta Tag & Schema.org Extractor
+ *
+ * Extracts OpenGraph, Twitter Card, Schema.org JSON-LD, and generic
+ * meta tags from root page HTML. Classifies the site type using cheerio.
+ */
+import type { MetaExtractResult } from '../core/types/recon.js';
+/**
+ * Extract meta tags, OpenGraph, Twitter Card, Schema.org, and classify site
+ */
+export declare function extractMeta(html: string, pageUrl: string): MetaExtractResult;

package/dist/recon/meta-extractor.js

@@ -0,0 +1,276 @@
+/**
+ * Meta Tag & Schema.org Extractor
+ *
+ * Extracts OpenGraph, Twitter Card, Schema.org JSON-LD, and generic
+ * meta tags from root page HTML. Classifies the site type using cheerio.
+ */
+import * as cheerio from 'cheerio';
+/**
+ * Site classification keywords mapped to category
+ */
+const CLASSIFICATION_SIGNALS = {
+    'e-commerce': [
+        /shop|store|product|cart|checkout|buy|price|add.to.cart/i,
+        /e-?commerce|retail|merchant|catalog/i,
+    ],
+    travel: [
+        /travel|flight|hotel|booking|reservation|trip|tour|airline/i,
+        /destination|cruise|vacation|itinerary/i,
+    ],
+    healthcare: [
+        /health|medical|doctor|patient|hospital|clinic|pharma/i,
+        /diagnosis|treatment|symptom|appointment/i,
+    ],
+    finance: [
+        /bank|finance|invest|trading|loan|mortgage|insurance|credit/i,
+        /portfolio|stock|fund|payment/i,
+    ],
+    education: [
+        /education|course|learn|school|university|student|teacher/i,
+        /curriculum|enrollment|lecture|tutorial/i,
+    ],
+    'news-media': [
+        /news|article|blog|press|journal|media|publish|editorial/i,
+        /breaking|headline|reporter|column/i,
+    ],
+    social: [
+        /social|community|profile|friend|follow|post|feed|message/i,
+        /network|connect|share|like|comment/i,
+    ],
+    saas: [
+        /dashboard|analytics|platform|subscription|api|integration/i,
+        /workspace|team|workflow|automation|saas/i,
+    ],
+    government: [
+        /government|gov\.|public.service|citizen|municipality|permit/i,
+        /regulation|compliance|legislation|agency/i,
+    ],
+    entertainment: [
+        /entertainment|game|movie|music|stream|watch|play|video/i,
+        /podcast|episode|series|concert/i,
+    ],
+    'food-restaurant': [
+        /restaurant|food|menu|order|delivery|recipe|cuisine/i,
+        /dining|chef|meal|reservation/i,
+    ],
+    'real-estate': [
+        /real.estate|property|listing|rent|lease|apartment|house/i,
+        /mortgage|realtor|condo|housing/i,
+    ],
+    general: [], // fallback
+};
+/**
+ * Schema.org type to site classification mapping
+ */
+const SCHEMA_TYPE_MAP = {
+    Product: 'e-commerce',
+    Offer: 'e-commerce',
+    ShoppingCenter: 'e-commerce',
+    Store: 'e-commerce',
+    Airline: 'travel',
+    Flight: 'travel',
+    Hotel: 'travel',
+    TouristAttraction: 'travel',
+    LodgingBusiness: 'travel',
+    TravelAgency: 'travel',
+    Hospital: 'healthcare',
+    MedicalClinic: 'healthcare',
+    Physician: 'healthcare',
+    Pharmacy: 'healthcare',
+    BankOrCreditUnion: 'finance',
+    FinancialService: 'finance',
+    InsuranceAgency: 'finance',
+    EducationalOrganization: 'education',
+    Course: 'education',
+    School: 'education',
+    University: 'education',
+    NewsArticle: 'news-media',
+    NewsMediaOrganization: 'news-media',
+    BlogPosting: 'news-media',
+    SocialMediaPosting: 'social',
+    SoftwareApplication: 'saas',
+    WebApplication: 'saas',
+    GovernmentOrganization: 'government',
+    GovernmentService: 'government',
+    Movie: 'entertainment',
+    MusicRecording: 'entertainment',
+    VideoGame: 'entertainment',
+    Restaurant: 'food-restaurant',
+    FoodEstablishment: 'food-restaurant',
+    Menu: 'food-restaurant',
+    RealEstateAgent: 'real-estate',
+    Apartment: 'real-estate',
+    House: 'real-estate',
+};
+/**
+ * Extract all meta tags from HTML
+ */
+function extractMetaTags($) {
+    const tags = [];
+    $('meta').each((_i, el) => {
+        const $el = $(el);
+        const property = $el.attr('property') ?? '';
+        const name = $el.attr('name') ?? '';
+        const content = $el.attr('content') ?? '';
+        if (!content)
+            return;
+        if (property.startsWith('og:')) {
+            tags.push({ name: property, content, category: 'og' });
+        }
+        else if (property.startsWith('twitter:') ||
+            name.startsWith('twitter:')) {
+            tags.push({
+                name: property || name,
+                content,
+                category: 'twitter',
+            });
+        }
+        else if (name) {
+            tags.push({ name, content, category: 'generic' });
+        }
+    });
+    return tags;
+}
+/**
+ * Extract OpenGraph data as a flat record
+ */
+function extractOpenGraph(tags) {
+    const og = {};
+    for (const tag of tags) {
+        if (tag.category === 'og') {
+            og[tag.name] = tag.content;
+        }
+    }
+    return og;
+}
+/**
+ * Extract Twitter Card data as a flat record
+ */
+function extractTwitterCard(tags) {
+    const twitter = {};
+    for (const tag of tags) {
+        if (tag.category === 'twitter') {
+            twitter[tag.name] = tag.content;
+        }
+    }
+    return twitter;
+}
+/**
+ * Extract Schema.org JSON-LD entries from <script type="application/ld+json">
+ */
+function extractSchemaOrg($) {
+    const schemas = [];
+    $('script[type="application/ld+json"]').each((_i, el) => {
+        const raw = $(el).html();
+        if (!raw)
+            return;
+        try {
+            const data = JSON.parse(raw);
+            // Handle @graph arrays
+            if (Array.isArray(data['@graph'])) {
+                for (const item of data['@graph']) {
+                    if (item &&
+                        typeof item === 'object' &&
+                        '@type' in item) {
+                        schemas.push(item);
+                    }
+                }
+            }
+            else if (data['@type']) {
+                schemas.push(data);
+            }
+        }
+        catch {
+            // Invalid JSON-LD — skip
+        }
+    });
+    return schemas;
+}
+/**
+ * Classify the site type based on all available signals
+ */
+function classifySite(metaTags, schemaOrg, pageTitle, html) {
+    const scores = {
+        'e-commerce': 0,
+        travel: 0,
+        healthcare: 0,
+        finance: 0,
+        education: 0,
+        'news-media': 0,
+        social: 0,
+        saas: 0,
+        government: 0,
+        entertainment: 0,
+        'food-restaurant': 0,
+        'real-estate': 0,
+        general: 0,
+    };
+    // Signal 1: Schema.org @type (strongest signal)
+    for (const schema of schemaOrg) {
+        const schemaType = schema['@type'];
+        if (typeof schemaType === 'string') {
+            const mapped = SCHEMA_TYPE_MAP[schemaType];
+            if (mapped)
+                scores[mapped] += 10;
+        }
+    }
+    // Signal 2: Meta tags + title + HTML content
+    const textSignal = [
+        pageTitle,
+        ...metaTags.map((t) => t.content),
+        // Sample a small portion of the HTML for keyword signals
+        html.substring(0, 5000),
+    ].join(' ');
+    for (const [category, patterns] of Object.entries(CLASSIFICATION_SIGNALS)) {
+        for (const pattern of patterns) {
+            if (pattern.test(textSignal)) {
+                scores[category] += 3;
+            }
+        }
+    }
+    // Find highest score
+    let bestCategory = 'general';
+    let bestScore = 0;
+    for (const [category, score] of Object.entries(scores)) {
+        if (score > bestScore) {
+            bestScore = score;
+            bestCategory = category;
+        }
+    }
+    return bestScore >= 3 ? bestCategory : 'general';
+}
+/**
+ * Extract meta tags, OpenGraph, Twitter Card, Schema.org, and classify site
+ */
+export function extractMeta(html, pageUrl) {
+    const $ = cheerio.load(html);
+    const metaTags = extractMetaTags($);
+    const openGraph = extractOpenGraph(metaTags);
+    const twitterCard = extractTwitterCard(metaTags);
+    const schemaOrg = extractSchemaOrg($);
+    const pageTitle = $('title').text().trim();
+    const descriptionTag = metaTags.find((t) => t.name === 'description' && t.category === 'generic');
+    const pageDescription = descriptionTag?.content ?? openGraph['og:description'] ?? '';
+    const canonicalEl = $('link[rel="canonical"]').attr('href');
+    let canonicalUrl;
+    if (canonicalEl) {
+        try {
+            canonicalUrl = new URL(canonicalEl, pageUrl).href;
+        }
+        catch {
+            // Invalid canonical
+        }
+    }
+    const siteClassification = classifySite(metaTags, schemaOrg, pageTitle, html);
+    return {
+        metaTags,
+        openGraph,
+        twitterCard,
+        schemaOrg,
+        siteClassification,
+        pageTitle,
+        pageDescription,
+        canonicalUrl,
+    };
+}
+//# sourceMappingURL=meta-extractor.js.map

package/dist/recon/meta-extractor.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"meta-extractor.js","sourceRoot":"","sources":["../../src/recon/meta-extractor.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AAQnC;;GAEG;AACH,MAAM,sBAAsB,GAAyC;IACnE,YAAY,EAAE;QACZ,yDAAyD;QACzD,sCAAsC;KACvC;IACD,MAAM,EAAE;QACN,4DAA4D;QAC5D,wCAAwC;KACzC;IACD,UAAU,EAAE;QACV,uDAAuD;QACvD,0CAA0C;KAC3C;IACD,OAAO,EAAE;QACP,6DAA6D;QAC7D,+BAA+B;KAChC;IACD,SAAS,EAAE;QACT,2DAA2D;QAC3D,yCAAyC;KAC1C;IACD,YAAY,EAAE;QACZ,0DAA0D;QAC1D,oCAAoC;KACrC;IACD,MAAM,EAAE;QACN,2DAA2D;QAC3D,qCAAqC;KACtC;IACD,IAAI,EAAE;QACJ,4DAA4D;QAC5D,0CAA0C;KAC3C;IACD,UAAU,EAAE;QACV,8DAA8D;QAC9D,2CAA2C;KAC5C;IACD,aAAa,EAAE;QACb,yDAAyD;QACzD,iCAAiC;KAClC;IACD,iBAAiB,EAAE;QACjB,qDAAqD;QACrD,+BAA+B;KAChC;IACD,aAAa,EAAE;QACb,0DAA0D;QAC1D,iCAAiC;KAClC;IACD,OAAO,EAAE,EAAE,EAAE,WAAW;CACzB,CAAC;AAEF;;GAEG;AACH,MAAM,eAAe,GAAuC;IAC1D,OAAO,EAAE,YAAY;IACrB,KAAK,EAAE,YAAY;IACnB,cAAc,EAAE,YAAY;IAC5B,KAAK,EAAE,YAAY;IACnB,OAAO,EAAE,QAAQ;IACjB,MAAM,EAAE,QAAQ;IAChB,KAAK,EAAE,QAAQ;IACf,iBAAiB,EAAE,QAAQ;IAC3B,eAAe,EAAE,QAAQ;IACzB,YAAY,EAAE,QAAQ;IACtB,QAAQ,EAAE,YAAY;IACtB,aAAa,EAAE,YAAY;IAC3B,SAAS,EAAE,YAAY;IACvB,QAAQ,EAAE,YAAY;IACtB,iBAAiB,EAAE,SAAS;IAC5B,gBAAgB,EAAE,SAAS;IAC3B,eAAe,EAAE,SAAS;IAC1B,uBAAuB,EAAE,WAAW;IACpC,MAAM,EAAE,WAAW;IACnB,MAAM,EAAE,WAAW;IACnB,UAAU,EAAE,WAAW;IACvB,WAAW,EAAE,YAAY;IACzB,qBAAqB,EAAE,YAAY;IACnC,WAAW,EAAE,YAAY;IACzB,kBAAkB,EAAE,QAAQ;IAC5B,mBAAmB,EAAE,MAAM;IAC3B,cAAc,EAAE,MAAM;IACtB,sBAAsB,EAAE,YAAY;IACpC,iBAAiB,EAAE,YAAY;IAC/B,KAAK,EAAE,eAAe;IACtB,cAAc,EAAE,eAAe;IAC/B,SAAS,EAAE,eAAe;IAC1B,UAAU,EAAE,iBAAiB;IAC7B,iBAAiB,EAAE,iBAAiB;IACpC,IAAI,EAAE,iBAAiB;IACvB,eAAe,EAAE,aAAa;IAC9B,SAAS,EAAE,aAAa;IACxB,KAAK,EAAE,aAAa;CACrB,CAAC;AAEF;;GAEG;AACH,SAAS,eAAe,CAAC,CAAqB;IAC5C,MAAM,IAAI,GAAoB,EAAE,CAAC;IAEjC,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE;QACxB,MAAM,GAAG,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC;QAClB,MAAM,QAAQ,GAAG,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC;QAC5C,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;QACpC,MAAM,OAAO,GAAG,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC;QAE1C,IAAI,CAAC,OAAO;YAAE,OAAO;QAErB,IAAI,QAAQ,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;YAC/B,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC;QACzD,CAAC;aAAM,IACL,QAAQ,CAAC,UAAU,CAAC,UAAU,CAAC;YAC/B,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,EAC3B,CAAC;YACD,IAAI,CAAC,IAAI,CAAC;gBACR,IAAI,EAAE,QAAQ,IAAI,IAAI;gBACtB,OAAO;gBACP,QAAQ,EAAE,SAAS;aACpB,CAAC,CAAC;QACL,CAAC;aAAM,IAAI,IAAI,EAAE,CAAC;YAChB,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,CAAC,CAAC;QACpD,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;GAEG;AACH,SAAS,gBAAgB,CAAC,IAAqB;IAC7C,MAAM,EAAE,GAA2B,EAAE,CAAC;IACtC,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,IAAI,GAAG,CAAC,QAAQ,KAAK,IAAI,EAAE,CAAC;YAC1B,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,GAAG,CAAC,OAAO,CAAC;QAC7B,CAAC;IACH,CAAC;IACD,OAAO,EAAE,CAAC;AACZ,CAAC;AAED;;GAEG;AACH,SAAS,kBAAkB,CAAC,IAAqB;IAC/C,MAAM,OAAO,GAA2B,EAAE,CAAC;IAC3C,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,IAAI,GAAG,CAAC,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC/B,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,GAAG,CAAC,OAAO,CAAC;QAClC,CAAC;IACH,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED;;GAEG;AACH,SAAS,gBAAgB,CAAC,CAAqB;IAC7C,MAAM,OAAO,GAAoB,EAAE,CAAC;IAEpC,CAAC,CAAC,oCAAoC,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE;QACtD,MAAM,GAAG,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QACzB,IAAI,CAAC,GAAG;YAAE,OAAO;QAEjB,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAA4B,CAAC;YAExD,uBAAuB;YACvB,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC;gBAClC,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;oBAClC,IACE,IAAI;wBACJ,OAAO,IAAI,KAAK,QAAQ;wBACxB,OAAO,IAAK,IAAgC,EAC5C,CAAC;wBACD,OAAO,CAAC,IAAI,CAAC,IAAqB,CAAC,CAAC;oBACtC,CAAC;gBACH,CAAC;YACH,CAAC;iBAAM,IAAI,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;gBACzB,OAAO,CAAC,IAAI,CAAC,IAAgC,CAAC,CAAC;YACjD,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,yBAAyB;QAC3B,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,OAAO,OAAO,CAAC;AACjB,CAAC;AAED;;GAEG;AACH,SAAS,YAAY,CACnB,QAAyB,EACzB,SAA0B,EAC1B,SAAiB,EACjB,IAAY;IAEZ,MAAM,MAAM,GAAuC;QACjD,YAAY,EAAE,CAAC;QACf,MAAM,EAAE,CAAC;QACT,UAAU,EAAE,CAAC;QACb,OAAO,EAAE,CAAC;QACV,SAAS,EAAE,CAAC;QACZ,YAAY,EAAE,CAAC;QACf,MAAM,EAAE,CAAC;QACT,IAAI,EAAE,CAAC;QACP,UAAU,EAAE,CAAC;QACb,aAAa,EAAE,CAAC;QAChB,iBAAiB,EAAE,CAAC;QACpB,aAAa,EAAE,CAAC;QAChB,OAAO,EAAE,CAAC;KACX,CAAC;IAEF,gDAAgD;IAChD,KAAK,MAAM,MAAM,IAAI,SAAS,EAAE,CAAC;QAC/B,MAAM,UAAU,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC;QACnC,IAAI,OAAO,UAAU,KAAK,QAAQ,EAAE,CAAC;YACnC,MAAM,MAAM,GAAG,eAAe,CAAC,UAAU,CAAC,CAAC;YAC3C,IAAI,MAAM;gBAAE,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;QACnC,CAAC;IACH,CAAC;IAED,6CAA6C;IAC7C,MAAM,UAAU,GAAG;QACjB,SAAS;QACT,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC;QACjC,yDAAyD;QACzD,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,IAAI,CAAC;KACxB,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAEZ,KAAK,MAAM,CAAC,QAAQ,EAAE,QAAQ,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,sBAAsB,CAAC,EAAE,CAAC;QAC1E,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC/B,IAAI,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC;gBAC7B,MAAM,CAAC,QAA8B,CAAC,IAAI,CAAC,CAAC;YAC9C,CAAC;QACH,CAAC;IACH,CAAC;IAED,qBAAqB;IACrB,IAAI,YAAY,GAAuB,SAAS,CAAC;IACjD,IAAI,SAAS,GAAG,CAAC,CAAC;IAElB,KAAK,MAAM,CAAC,QAAQ,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;QACvD,IAAI,KAAK,GAAG,SAAS,EAAE,CAAC;YACtB,SAAS,GAAG,KAAK,CAAC;YAClB,YAAY,GAAG,QAA8B,CAAC;QAChD,CAAC;IACH,CAAC;IAED,OAAO,SAAS,IAAI,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,SAAS,CAAC;AACnD,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,WAAW,CACzB,IAAY,EACZ,OAAe;IAEf,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAE7B,MAAM,QAAQ,GAAG,eAAe,CAAC,CAAC,CAAC,CAAC;IACpC,MAAM,SAAS,GAAG,gBAAgB,CAAC,QAAQ,CAAC,CAAC;IAC7C,MAAM,WAAW,GAAG,kBAAkB,CAAC,QAAQ,CAAC,CAAC;IACjD,MAAM,SAAS,GAAG,gBAAgB,CAAC,CAAC,CAAC,CAAC;IAEtC,MAAM,SAAS,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;IAC3C,MAAM,cAAc,GAAG,QAAQ,CAAC,IAAI,CAClC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,aAAa,IAAI,CAAC,CAAC,QAAQ,KAAK,SAAS,CAC5D,CAAC;IACF,MAAM,eAAe,GACnB,cAAc,EAAE,OAAO,IAAI,SAAS,CAAC,gBAAgB,CAAC,IAAI,EAAE,CAAC;IAE/D,MAAM,WAAW,GAAG,CAAC,CAAC,uBAAuB,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAC5D,IAAI,YAAgC,CAAC;IACrC,IAAI,WAAW,EAAE,CAAC;QAChB,IAAI,CAAC;YACH,YAAY,GAAG,IAAI,GAAG,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC;QACpD,CAAC;QAAC,MAAM,CAAC;YACP,oBAAoB;QACtB,CAAC;IACH,CAAC;IAED,MAAM,kBAAkB,GAAG,YAAY,CAAC,QAAQ,EAAE,SAAS,EAAE,SAAS,EAAE,IAAI,CAAC,CAAC;IAE9E,OAAO;QACL,QAAQ;QACR,SAAS;QACT,WAAW;QACX,SAAS;QACT,kBAAkB;QAClB,SAAS;QACT,eAAe;QACf,YAAY;KACb,CAAC;AACJ,CAAC"}

package/dist/recon/robots.d.ts

@@ -0,0 +1,16 @@
+/**
+ * Robots.txt Parser
+ *
+ * Fetches and parses robots.txt. Extracts allowed/disallowed paths,
+ * crawl-delay, and sitemap references. Handles missing robots.txt gracefully.
+ */
+import type { RobotsDirective, RobotsResult } from '../core/types/recon.js';
+/**
+ * Check whether a path is allowed by the robots.txt directives.
+ * Uses wildcard user-agent (*) rules if no specific match.
+ */
+export declare function isPathAllowed(path: string, directives: RobotsDirective[], userAgent?: string): boolean;
+/**
+ * Fetch and parse robots.txt for a site
+ */
+export declare function parseRobots(baseUrl: string): Promise<RobotsResult>;