penthera 1.1.0

package/lib/compliance/readiness.js

@@ -0,0 +1,229 @@
+/**
+ * NIS2 readiness report, the "forms/data they need" core.
+ *
+ * Combines what the scan can measure (technical measures) with a local
+ * self-assessment (the organisational measures a scanner can't test) into a
+ * per-measure met / partial / gap / n-a report, with proportionality notes and
+ * the jurisdiction's incident-reporting duties.
+ *
+ * Provenance is first-class: every piece of evidence carries
+ * `{ source, method, collected_at, confidence }`, so the whole report is
+ * reproducible and defensible. Nothing here asserts compliance, it is
+ * readiness and self-assessment tooling only.
+ */
+import { buildComplianceCoverage } from "./index.js";
+import { normalizeAnswers, questionsForMeasure } from "./self-assessment.js";
+
+const PROPORTIONALITY = {
+  a: "A short written policy plus a basic risk list is proportionate for a small entity.",
+  b: "A one-page runbook with roles, contacts, and the NIS2 deadlines is enough to start.",
+  c: "Regular backups plus one real restore test per year.",
+  d: "A supplier list plus basic security checks or contractual clauses.",
+  e: "A patch cadence and a vulnerability-reporting channel; Penthera covers the technical scan.",
+  f: "Periodic re-scan and incident review; baseline diffs are your evidence.",
+  g: "A short onboarding briefing and occasional reminders.",
+  h: "Encryption in transit (scanned) plus encryption at rest for sensitive stores.",
+  i: "Least privilege, a joiner/leaver process, and a simple asset list.",
+  j: "MFA on email and admin/privileged accounts is the highest-value control here.",
+};
+
+function scanStatusOf(coverageStatus) {
+  // Maps the scan's per-measure coverage to a readiness contribution. Note that
+  // a measure the scanner can't test ("out-of-scope"/"evidence") returns "none",
+  // NOT "n/a": it still matters to the company and is driven by attestation. A
+  // true N/A only comes from the user attesting "na".
+  switch (coverageStatus) {
+    case "tested-clean": return "met";
+    case "tested-findings": return "partial";
+    default: return "none"; // evidence / planned / not-run / out-of-scope
+  }
+}
+
+function attestStatusOf(measure, answers) {
+  const vals = questionsForMeasure(measure).map((q) => answers[q.id]?.answer).filter(Boolean);
+  if (vals.length === 0) return "none";
+  const real = vals.filter((v) => v !== "na");
+  if (real.length === 0) return "na";
+  if (real.includes("no")) return "gap";
+  if (real.includes("partial")) return "partial";
+  return "met";
+}
+
+function combine(scanS, attestS) {
+  const known = [];
+  if (scanS === "met" || scanS === "partial") known.push(scanS);
+  if (["met", "partial", "gap"].includes(attestS)) known.push(attestS);
+  if (known.length === 0) return attestS === "na" ? "n/a" : "gap";
+  if (known.includes("gap")) return "gap";
+  if (known.includes("partial")) return "partial";
+  return "met";
+}
+
+/**
+ * Build a readiness report from a scan result and optional self-assessment.
+ * @param {object} scanResult - merged scan result (may be near-empty for attestation-only)
+ * @param {object} opts - { answers, jurisdiction, generatedAt, assessedAt }
+ */
+export function buildReadiness(scanResult, opts = {}) {
+  const answers = normalizeAnswers(opts.answers || {});
+  const coverage = buildComplianceCoverage(scanResult, "nis2") || { measures: [] };
+  const scanTs = scanResult?.timestamp || null;
+
+  const measures = coverage.measures.map((m) => {
+    const scanS = scanStatusOf(m.status);
+    const attestS = attestStatusOf(m.id, answers);
+    const status = combine(scanS, attestS);
+
+    const evidence = [];
+    for (const p of m.probesRun || []) {
+      evidence.push({ source: "scan", method: `probe:${p}`, collected_at: scanTs, confidence: "measured" });
+    }
+    for (const q of questionsForMeasure(m.id)) {
+      const a = answers[q.id];
+      if (a) {
+        evidence.push({
+          source: "self-assessment",
+          method: "attestation",
+          collected_at: opts.assessedAt || null,
+          confidence: "attested",
+          question: q.text,
+          answer: a.answer,
+          note: a.note || "",
+        });
+      }
+    }
+
+    const gaps = [];
+    for (const q of questionsForMeasure(m.id)) {
+      const a = answers[q.id];
+      if (!a) gaps.push({ type: "unanswered", question: q.text });
+      else if (a.answer === "no") gaps.push({ type: "not-in-place", question: q.text, note: a.note || "" });
+      else if (a.answer === "partial") gaps.push({ type: "partial", question: q.text, note: a.note || "" });
+    }
+
+    return {
+      id: m.id,
+      title: m.title,
+      status,
+      basis: { scan: scanS, attestation: attestS },
+      findings: m.findings,
+      evidence,
+      gaps,
+      proportionality: PROPORTIONALITY[m.id] || "",
+    };
+  });
+
+  const summary = { met: 0, partial: 0, gap: 0, "n/a": 0 };
+  for (const m of measures) summary[m.status] = (summary[m.status] || 0) + 1;
+  const assessed = measures.filter((m) => m.basis.attestation !== "none" || m.basis.scan !== "none").length;
+
+  const j = opts.jurisdiction || null;
+  return {
+    framework: "nis2",
+    jurisdiction: j && {
+      id: j.id, name: j.name, law: j.law, authorities: j.authorities,
+      incidentDeadlines: j.incidentDeadlines, penalties: j.penalties, disclaimer: j.disclaimer,
+    },
+    generated_at: opts.generatedAt || null,
+    measures,
+    summary,
+    assessedCount: assessed,
+    totalMeasures: measures.length,
+  };
+}
+
+const STATUS_LABEL = { met: "Met", partial: "Partial", gap: "Gap", "n/a": "N/A" };
+
+/** Render a readiness report as Markdown. */
+export function formatReadinessMarkdown(readiness) {
+  if (!readiness) return "";
+  const L = [];
+  const j = readiness.jurisdiction;
+  L.push(`# NIS2 readiness report${j ? `, ${j.name}` : ""}`);
+  L.push("");
+  if (j?.law) L.push(`> Framework: **${j.law}** (EU NIS2, Article 21 measures).`);
+  L.push("> **Readiness, evidence, and self-assessment tooling, not legal advice, not a certification, and not a guarantee of compliance.**");
+  if (j?.disclaimer) { L.push(">"); L.push(`> ${j.disclaimer}`); }
+  L.push("");
+
+  const s = readiness.summary;
+  L.push(`**Summary:** ${s.met} met · ${s.partial} partial · ${s.gap} gap · ${s["n/a"]} n/a, across ${readiness.totalMeasures} Article 21 measures.`);
+  L.push("");
+  L.push("Status combines what Penthera measured (technical) with your self-assessment (organisational). *Proportionality applies: for a small entity, a short policy or process that genuinely exists counts.*");
+  L.push("");
+
+  L.push("| Art. 21(2) | Measure | Status | Basis |");
+  L.push("|-----------|---------|--------|-------|");
+  for (const m of readiness.measures) {
+    const basis = [
+      m.basis.scan !== "none" && m.basis.scan !== "na" ? "scan" : null,
+      ["met", "partial", "gap"].includes(m.basis.attestation) ? "attestation" : null,
+    ].filter(Boolean).join(" + ") || "not assessed";
+    L.push(`| **(${m.id})** | ${m.title} | ${STATUS_LABEL[m.status]} | ${basis} |`);
+  }
+  L.push("");
+
+  // Per-measure detail
+  L.push("## Measures in detail");
+  L.push("");
+  for (const m of readiness.measures) {
+    L.push(`### (${m.id}) ${m.title}, ${STATUS_LABEL[m.status]}`);
+    L.push("");
+    if (m.evidence.length) {
+      L.push("**Evidence**");
+      for (const e of m.evidence) {
+        if (e.source === "scan") {
+          L.push(`- scan · \`${e.method}\` · ${e.collected_at || "-"} · confidence: ${e.confidence}`);
+        } else {
+          L.push(`- self-assessment · "${e.question}" → **${e.answer}**${e.note ? ` (${e.note})` : ""}`);
+        }
+      }
+      L.push("");
+    }
+    if (m.gaps.length) {
+      L.push("**Gaps / to confirm**");
+      for (const g of m.gaps) {
+        if (g.type === "unanswered") L.push(`- ❓ Not yet assessed: ${g.question}`);
+        else if (g.type === "not-in-place") L.push(`- ❌ Not in place: ${g.question}${g.note ? ` (${g.note})` : ""}`);
+        else L.push(`- ⚠️ Partial: ${g.question}${g.note ? ` (${g.note})` : ""}`);
+      }
+      L.push("");
+    }
+    if (m.proportionality) { L.push(`*Proportionate baseline:* ${m.proportionality}`); L.push(""); }
+  }
+
+  // Incident reporting duties (measure b, practical)
+  if (j?.incidentDeadlines?.length) {
+    L.push("## Incident-reporting duties");
+    L.push("");
+    L.push(`Report significant incidents to **${j.authorities?.csirt || "your CSIRT"}** (supervisor: ${j.authorities?.supervisor || "your supervisor"}):`);
+    L.push("");
+    L.push("| Deadline | When | What |");
+    L.push("|----------|------|------|");
+    for (const d of j.incidentDeadlines) {
+      L.push(`| **${d.within}** | ${d.of} | ${d.content} |`);
+    }
+    L.push("");
+    if (j.penalties) L.push(`> ${j.penalties}`);
+    L.push("");
+  }
+
+  // What only you can confirm
+  const unanswered = readiness.measures.flatMap((m) => m.gaps.filter((g) => g.type === "unanswered").map((g) => g.question));
+  if (unanswered.length) {
+    L.push("## What only you can confirm");
+    L.push("");
+    L.push("These are attestations a scan cannot make. Run the self-assessment to record them:");
+    L.push("");
+    unanswered.forEach((q) => L.push(`- ${q}`));
+    L.push("");
+    L.push("Start with: `penthera --assessment-init assessment.json`, fill it in, then re-run with `--assessment assessment.json`.");
+    L.push("");
+  }
+
+  L.push("---");
+  L.push("");
+  L.push(`*Generated by Penthera${readiness.generated_at ? ` at ${readiness.generated_at}` : ""}. Every item above is provenance-linked (source, method, collected_at, confidence) and reproducible by re-running the scan and self-assessment.*`);
+  L.push("");
+  return L.join("\n");
+}

package/lib/compliance/self-assessment.js

@@ -0,0 +1,66 @@
+/**
+ * Local self-assessment for the organisational half of NIS2 Article 21, the
+ * measures a scanner cannot test and must capture as an honest attestation.
+ *
+ * Runs entirely offline. The question bank is data; answers are recorded to a
+ * JSON file that the readiness report consumes. Every answer becomes an evidence
+ * record with provenance (source: self-assessment / attestation).
+ *
+ * Proportionality is the governing principle: for a small entity, a two-page
+ * policy that genuinely exists counts. The questions ask whether something
+ * exists and is used, not whether it is enterprise-grade.
+ */
+
+// One or more attestation questions per Article 21(2) measure. `id` is stable
+// (used as the answer key and in provenance). Answers: yes | partial | no | na.
+export const QUESTIONS = [
+  { id: "a1", measure: "a", text: "Do you have a written information-security policy (even one to two pages)?", guidance: "A short, real policy that staff can find counts. It doesn't need to be long." },
+  { id: "a2", measure: "a", text: "Have you done a basic risk assessment of your most important systems and data?", guidance: "Identify what would hurt most if lost/breached, and the top risks to it." },
+  { id: "b1", measure: "b", text: "Do you have a documented incident-response process (who does what, who to call)?", guidance: "Even a one-page runbook with roles and contacts." },
+  { id: "b2", measure: "b", text: "Do you know your NIS2 reporting deadlines (24h / 72h / 1 month) and who you would notify?", guidance: "See the incident-report helper; the authority is your CSIRT / RDI." },
+  { id: "c1", measure: "c", text: "Do you take regular backups of important data and systems?", guidance: "Automated, off-site or immutable backups are best." },
+  { id: "c2", measure: "c", text: "Have you successfully restored from a backup in the last 12 months?", guidance: "An untested backup is not a backup. A single real restore test counts." },
+  { id: "d1", measure: "d", text: "Do you keep a list of your key suppliers/vendors (especially those with access to your data or systems)?", guidance: "A simple inventory is enough to start." },
+  { id: "d2", measure: "d", text: "Do you assess your suppliers' security (questionnaire, certification, or contractual clauses)?", guidance: "Penthera's passive supplier rating can help for the technical part." },
+  { id: "e1", measure: "e", text: "Do you have a process for applying security updates and handling reported vulnerabilities?", guidance: "Patching cadence + a way for people to report issues (e.g. a SECURITY.md)." },
+  { id: "f1", measure: "f", text: "Do you periodically review whether your security measures are actually working (re-scan, review incidents)?", guidance: "Penthera baseline diffs and re-scans provide this evidence." },
+  { id: "g1", measure: "g", text: "Do staff receive basic security-awareness guidance (phishing, passwords, reporting)?", guidance: "A short onboarding briefing and occasional reminders count." },
+  { id: "h1", measure: "h", text: "Is sensitive data encrypted at rest (databases, backups, laptops)?", guidance: "Penthera checks encryption in transit (TLS); at-rest is your attestation." },
+  { id: "i1", measure: "i", text: "Do you have an access-control approach (least privilege, and removing access when people leave)?", guidance: "A joiner/leaver checklist and role-based access are proportionate for most SMEs." },
+  { id: "i2", measure: "i", text: "Do you keep an inventory of your assets (devices, systems, key data)?", guidance: "A spreadsheet is a fine starting point." },
+  { id: "j1", measure: "j", text: "Is multi-factor authentication (MFA) enforced for admin/privileged accounts and email?", guidance: "MFA on email and admin panels is the single highest-value control here." },
+];
+
+const VALID_ANSWERS = new Set(["yes", "partial", "no", "na"]);
+
+/** A blank assessment template the user or agent fills in. */
+export function blankAssessment() {
+  const answers = {};
+  for (const q of QUESTIONS) {
+    answers[q.id] = { answer: "", note: "" };
+  }
+  return {
+    _about: "Penthera NIS2 self-assessment. Answer each: yes | partial | no | na, with an optional note. Offline; no data leaves your machine. See lib/compliance/self-assessment.js for the questions.",
+    framework: "nis2",
+    answers,
+  };
+}
+
+/** Normalise a loaded answers object; ignores unknown ids and invalid values. */
+export function normalizeAnswers(loaded) {
+  const raw = loaded?.answers || loaded || {};
+  const out = {};
+  for (const q of QUESTIONS) {
+    const a = raw[q.id];
+    const answer = typeof a === "string" ? a : a?.answer;
+    if (answer && VALID_ANSWERS.has(String(answer).toLowerCase())) {
+      out[q.id] = { answer: String(answer).toLowerCase(), note: (a && a.note) || "" };
+    }
+  }
+  return out;
+}
+
+/** Questions for a given measure id (a-j). */
+export function questionsForMeasure(measure) {
+  return QUESTIONS.filter((q) => q.measure === measure);
+}

package/lib/crawler.js

@@ -0,0 +1,257 @@
+/**
+ * Runtime Endpoint Discovery (inspired by Katana/httpx)
+ *
+ * Crawls a live target to discover API endpoints, forms, and links.
+ * Compares findings with filesystem-discovered routes to find "shadow" endpoints.
+ *
+ * Techniques borrowed from:
+ *   - Katana (ProjectDiscovery), recursive crawl with scope control
+ *   - httpx, tech fingerprinting from response headers
+ *   - ffuf, wordlist-based endpoint brute-forcing
+ */
+
+const DEFAULT_TIMEOUT = 8_000;
+
+async function safeFetch(url, opts = {}) {
+  const controller = new AbortController();
+  const timeout = setTimeout(() => controller.abort(), opts.timeout || DEFAULT_TIMEOUT);
+  try {
+    return await fetch(url, { ...opts, signal: controller.signal, redirect: "manual" });
+  } catch {
+    return null;
+  } finally {
+    clearTimeout(timeout);
+  }
+}
+
+/**
+ * Discover endpoints by brute-forcing common API paths.
+ * Like ffuf but built-in, no external tool needed.
+ *
+ * @param {string} baseUrl - Target base URL
+ * @param {object} opts - { concurrency, wordlist, timeout }
+ * @returns {Array<{ path, status, size, tech }>} Discovered endpoints
+ */
+export async function discoverEndpoints(baseUrl, opts = {}) {
+  const concurrency = opts.concurrency || 10;
+  const timeout = opts.timeout || DEFAULT_TIMEOUT;
+
+  // Common API paths, inspired by Nikto's 8k+ check database
+  // and common Next.js / Node.js patterns
+  const wordlist = opts.wordlist || [
+    // Health / status
+    "/api/health", "/api/status", "/api/ping", "/api/version",
+    "/health", "/healthz", "/ready", "/readyz", "/.well-known/health",
+    // Auth
+    "/api/auth/login", "/api/auth/register", "/api/auth/logout",
+    "/api/auth/me", "/api/auth/session", "/api/auth/refresh",
+    "/api/auth/forgot-password", "/api/auth/reset-password",
+    "/api/auth/verify-email", "/api/auth/callback",
+    "/api/login", "/api/register", "/api/signup",
+    // User
+    "/api/user", "/api/user/profile", "/api/user/settings",
+    "/api/users", "/api/me", "/api/account",
+    // Admin
+    "/api/admin", "/api/admin/users", "/api/admin/settings",
+    "/api/admin/logs", "/api/admin/stats", "/api/admin/dashboard",
+    // CRUD common
+    "/api/posts", "/api/comments", "/api/items", "/api/products",
+    "/api/orders", "/api/invoices", "/api/payments",
+    "/api/projects", "/api/tasks", "/api/events",
+    "/api/messages", "/api/notifications", "/api/uploads",
+    // Common utility
+    "/api/search", "/api/contact", "/api/feedback", "/api/subscribe",
+    "/api/unsubscribe", "/api/webhook", "/api/webhooks",
+    "/api/email", "/api/chat", "/api/ai", "/api/generate",
+    // Cron / background
+    "/api/cron", "/api/cron/cleanup", "/api/cron/digest",
+    "/api/cron/sync", "/api/cron/reindex",
+    // Stripe / billing
+    "/api/stripe/checkout", "/api/stripe/webhook",
+    "/api/billing", "/api/subscription", "/api/pricing",
+    // Third-party integrations
+    "/api/oauth/callback", "/api/oauth/google", "/api/oauth/github",
+    "/api/integrations", "/api/connect",
+    // Export / import
+    "/api/export", "/api/import", "/api/download",
+    // GraphQL
+    "/graphql", "/api/graphql",
+    // Documentation / debug (should be disabled in prod)
+    "/docs", "/redoc", "/openapi.json",
+    "/api/docs", "/api/swagger", "/api/openapi",
+    "/api/debug", "/api/test", "/api/_debug",
+    // Sensitive files
+    "/.env", "/.env.local", "/.env.production",
+    "/.git/HEAD", "/.git/config",
+    "/package.json", "/tsconfig.json",
+    "/next.config.js", "/next.config.mjs",
+    "/firebase.json", "/firestore.rules",
+    "/serviceAccountKey.json", "/service-account.json",
+    "/robots.txt", "/sitemap.xml",
+    "/.well-known/security.txt",
+    // Next.js internals
+    "/_next/static/chunks/main.js.map",
+    "/_next/data/build-id/index.json",
+    "/api/__nextjs_original-stack-frame",
+    // Common CMS / frameworks
+    "/wp-admin", "/wp-login.php", "/administrator",
+    "/.htaccess", "/web.config",
+  ];
+
+  const discovered = [];
+
+  // Batch requests with concurrency control (like httpx)
+  for (let i = 0; i < wordlist.length; i += concurrency) {
+    const batch = wordlist.slice(i, i + concurrency);
+    const results = await Promise.allSettled(
+      batch.map(async (path) => {
+        const res = await safeFetch(`${baseUrl}${path}`, { timeout });
+        if (!res) return null;
+        // Measure the actual body length rather than the Content-Length header,
+        // which is often absent on chunked responses. This matches the unit the
+        // calibration baseline uses (text.length) so the noise filter in
+        // autoCalibrate compares like with like instead of header-vs-body.
+        const body = await res.text().catch(() => "");
+        return {
+          path,
+          status: res.status,
+          size: body.length,
+          contentType: res.headers.get("content-type") || "",
+          server: res.headers.get("server") || "",
+          poweredBy: res.headers.get("x-powered-by") || "",
+          redirectTo: res.headers.get("location") || null,
+        };
+      }),
+    );
+
+    for (const r of results) {
+      if (r.status === "fulfilled" && r.value) {
+        discovered.push(r.value);
+      }
+    }
+  }
+
+  return discovered;
+}
+
+/**
+ * Fingerprint the target technology stack.
+ * Inspired by httpx / Wappalyzer.
+ *
+ * @param {string} baseUrl - Target base URL
+ * @returns {object} Technology fingerprint
+ */
+export async function fingerprint(baseUrl) {
+  const tech = {
+    framework: null,
+    server: null,
+    cdn: null,
+    security: [],
+    headers: {},
+  };
+
+  // Fetch the homepage
+  const res = await safeFetch(baseUrl);
+  if (!res) return tech;
+
+  tech.headers = Object.fromEntries(res.headers.entries());
+
+  // Server detection
+  const server = res.headers.get("server") || "";
+  const poweredBy = res.headers.get("x-powered-by") || "";
+  if (server.includes("Vercel")) tech.server = "Vercel";
+  else if (server.includes("cloudflare")) { tech.server = "Cloudflare"; tech.cdn = "Cloudflare"; }
+  else if (server.includes("nginx")) tech.server = "nginx";
+  else if (server.includes("apache")) tech.server = "Apache";
+  else if (server) tech.server = server;
+
+  // Framework detection
+  if (poweredBy.includes("Next.js")) tech.framework = "Next.js";
+  else if (poweredBy.includes("Express")) tech.framework = "Express";
+  else if (poweredBy.includes("Nuxt")) tech.framework = "Nuxt";
+
+  // CDN / platform detection from headers
+  if (res.headers.has("cf-ray")) tech.cdn = "Cloudflare";
+  if (res.headers.has("x-vercel-id")) { tech.server = "Vercel"; tech.cdn = "Vercel Edge"; }
+  if (res.headers.has("x-amz-cf-id")) tech.cdn = "AWS CloudFront";
+
+  // Next.js detection from HTML
+  try {
+    const html = await res.text();
+    if (html.includes("/_next/static") || html.includes("__NEXT_DATA__")) {
+      tech.framework = "Next.js";
+    }
+    if (html.includes("__nuxt")) tech.framework = "Nuxt";
+    if (html.includes("ng-version")) tech.framework = "Angular";
+    if (html.includes("data-reactroot") || html.includes("__react")) tech.framework = "React";
+  } catch { /* ignore */ }
+
+  // Security header audit
+  const securityHeaders = [
+    "strict-transport-security", "content-security-policy",
+    "x-frame-options", "x-content-type-options",
+    "x-xss-protection", "referrer-policy",
+    "permissions-policy",
+  ];
+  for (const h of securityHeaders) {
+    if (res.headers.has(h)) tech.security.push(h);
+  }
+
+  return tech;
+}
+
+/**
+ * Auto-calibrate response filtering (inspired by ffuf -ac).
+ * Sends baseline requests and identifies the "default" response signature.
+ * Returns a filter function that rejects matching responses.
+ *
+ * @param {string} baseUrl
+ * @returns {function} Filter function: (response) => boolean (true = interesting, false = noise)
+ */
+export async function autoCalibrate(baseUrl) {
+  // Send requests to paths that definitely don't exist
+  const baselines = [];
+  for (const garbage of [
+    `/api/${Date.now()}-nonexistent-calibration-a`,
+    `/api/${Date.now()}-nonexistent-calibration-b`,
+    `/api/${Date.now()}-nonexistent-calibration-c`,
+  ]) {
+    const res = await safeFetch(`${baseUrl}${garbage}`);
+    if (res) {
+      const text = await res.text().catch(() => "");
+      baselines.push({
+        status: res.status,
+        size: text.length,
+        contentType: res.headers.get("content-type") || "",
+      });
+    }
+  }
+
+  // Find common baseline signature
+  if (baselines.length === 0) return () => true; // can't calibrate, accept everything
+
+  const baselineStatus = baselines[0].status;
+  const baselineSize = baselines.reduce((sum, b) => sum + b.size, 0) / baselines.length;
+  const sizeTolerance = baselineSize * 0.1 + 50; // 10% + 50 bytes tolerance
+
+  return (response) => {
+    // If response matches the baseline signature, it's noise
+    if (response.status === baselineStatus &&
+        Math.abs(response.size - baselineSize) < sizeTolerance) {
+      return false; // noise, filter out
+    }
+    return true; // interesting, keep
+  };
+}
+
+/**
+ * Compare crawled endpoints with filesystem-discovered routes.
+ * Returns "shadow" endpoints, routes that exist at runtime but aren't in source.
+ */
+export function findShadowEndpoints(crawledPaths, filesystemRoutes) {
+  const knownPaths = new Set(filesystemRoutes.map(r => r.url));
+  return crawledPaths.filter(ep => {
+    const normalized = ep.path.replace(/\/$/, "");
+    return !knownPaths.has(normalized) && ep.status !== 404;
+  });
+}