penthera 1.1.0

package/lib/audit-loop.js

@@ -0,0 +1,170 @@
+/**
+ * Remediation throughput / audit-loop tracking (NIS2 measure f).
+ *
+ * More findings is not more security, what matters is closing them. This module
+ * turns repeated scans into an audit-loop artifact: time-to-fix, ageing of open
+ * items, and drift since last scan. It persists a small history keyed by a
+ * STABLE finding fingerprint (severity- and count-independent) so a re-worded or
+ * re-counted finding isn't seen as "resolved + new" churn.
+ *
+ * Pure functions take `now` (ISO string) so they're deterministic and testable.
+ */
+
+const DAY_MS = 24 * 3600 * 1000;
+
+/**
+ * Stable fingerprint for a finding: source + category + normalized url + title,
+ * with digit runs collapsed so ids/counts (e.g. "5 active params", "/users/1")
+ * don't create spurious new/resolved churn. Severity is intentionally excluded.
+ */
+export function fingerprintFinding(f) {
+  // Collapse only path segments that are entirely numeric (IDOR object ids like
+  // /users/1 → /users/#), keeping version-like segments (v1, v2) distinct.
+  const normUrl = (u) => String(u || "").toLowerCase().trim()
+    .split("/").map((seg) => (/^\d+$/.test(seg) ? "#" : seg)).join("/");
+  // Keep digits in the title: versions, ciphers, CVEs, protocols, and IP:port
+  // pairs are exactly what tell otherwise-similar findings apart.
+  const normTitle = (t) => String(t || "").toLowerCase().replace(/\s+/g, " ").trim();
+  // Dependency-CVE findings can share one title (the CVE lives in the
+  // description), so fold in a short description discriminator for those.
+  const disc = (f.category === "cve" || f.category === "js-vulnerability")
+    ? "|" + String(f.description || "").toLowerCase().replace(/\s+/g, " ").slice(0, 80)
+    : "";
+  return [f.source || "", f.category || "", normUrl(f.url), normTitle(f.title)].join("|") + disc;
+}
+
+/** A fresh, empty history store. */
+export function emptyHistory() {
+  return { version: 1, findings: {}, scans: [] };
+}
+
+/**
+ * Fold the current findings into the history. Returns a new history object.
+ * - a finding seen now that wasn't open before → firstSeen = now
+ * - a finding previously resolved but seen again → reopened (resolvedAt cleared)
+ * - a tracked finding not seen now → resolvedAt = now
+ */
+export function updateHistory(history, findings, now) {
+  // Deep-copy records so we never mutate the caller's history object.
+  const h = {
+    version: 1,
+    findings: Object.fromEntries(
+      Object.entries(history?.findings || {}).map(([k, v]) => [k, { ...v }]),
+    ),
+    scans: [...(history?.scans || [])],
+  };
+  const current = new Map();
+  for (const f of findings || []) {
+    if (f.severity === "info") continue; // track actionable findings only
+    current.set(fingerprintFinding(f), f);
+  }
+
+  let newCount = 0;
+  let reopened = 0;
+  for (const [fp, f] of current) {
+    const rec = h.findings[fp];
+    if (!rec) {
+      h.findings[fp] = {
+        firstSeen: now, lastSeen: now, resolvedAt: null,
+        severity: f.severity, title: f.title, url: f.url || null, category: f.category || null,
+      };
+      newCount++;
+    } else {
+      rec.lastSeen = now;
+      rec.severity = f.severity; // keep latest severity
+      if (rec.resolvedAt) { rec.resolvedAt = null; reopened++; } // reappeared
+    }
+  }
+
+  let resolvedCount = 0;
+  for (const [fp, rec] of Object.entries(h.findings)) {
+    if (!current.has(fp) && !rec.resolvedAt) {
+      rec.resolvedAt = now;
+      resolvedCount++;
+    }
+  }
+
+  const open = Object.values(h.findings).filter((r) => !r.resolvedAt).length;
+  h.scans.push({ at: now, open, new: newCount, resolved: resolvedCount, reopened });
+  return h;
+}
+
+function daysBetween(aIso, bIso) {
+  const a = new Date(aIso).getTime();
+  const b = new Date(bIso).getTime();
+  if (Number.isNaN(a) || Number.isNaN(b)) return null;
+  return Math.max(0, Math.round((b - a) / DAY_MS));
+}
+
+function median(nums) {
+  if (!nums.length) return null;
+  const s = [...nums].sort((a, b) => a - b);
+  const mid = Math.floor(s.length / 2);
+  return s.length % 2 ? s[mid] : Math.round((s[mid - 1] + s[mid]) / 2);
+}
+
+/** Compute the audit-loop artifact from a history store. */
+export function buildAuditLoop(history, now) {
+  const recs = Object.values(history?.findings || {});
+  const open = recs.filter((r) => !r.resolvedAt);
+  const resolved = recs.filter((r) => r.resolvedAt);
+
+  const openBySeverity = { critical: 0, high: 0, medium: 0, low: 0 };
+  for (const r of open) if (openBySeverity[r.severity] != null) openBySeverity[r.severity]++;
+
+  const ageOf = (r) => daysBetween(r.firstSeen, now);
+  const ttf = resolved.map((r) => daysBetween(r.firstSeen, r.resolvedAt)).filter((n) => n != null);
+
+  const ageing = { "0-7": 0, "8-30": 0, "31-90": 0, "90+": 0 };
+  for (const r of open) {
+    const d = ageOf(r) ?? 0;
+    if (d <= 7) ageing["0-7"]++;
+    else if (d <= 30) ageing["8-30"]++;
+    else if (d <= 90) ageing["31-90"]++;
+    else ageing["90+"]++;
+  }
+
+  const openList = open
+    .map((r) => ({ title: r.title, url: r.url, severity: r.severity, ageDays: ageOf(r), firstSeen: r.firstSeen }))
+    .sort((a, b) => (b.ageDays ?? 0) - (a.ageDays ?? 0));
+
+  return {
+    generated_at: now,
+    scansRecorded: (history?.scans || []).length,
+    openCount: open.length,
+    resolvedCount: resolved.length,
+    openBySeverity,
+    medianTimeToFixDays: median(ttf),
+    oldestOpenDays: openList.length ? (openList[0].ageDays ?? 0) : 0,
+    ageing,
+    open: openList,
+    lastScan: (history?.scans || []).slice(-1)[0] || null,
+  };
+}
+
+/** Render the audit loop as a Markdown section. */
+export function formatAuditLoopMarkdown(a) {
+  if (!a) return "";
+  const L = [];
+  L.push("## Remediation & audit loop (measure f)");
+  L.push("");
+  L.push(`Across **${a.scansRecorded}** recorded scan${a.scansRecorded === 1 ? "" : "s"}: **${a.openCount}** open · **${a.resolvedCount}** resolved.`);
+  L.push("");
+  L.push(`- **Open by severity:** ${a.openBySeverity.critical} critical · ${a.openBySeverity.high} high · ${a.openBySeverity.medium} medium · ${a.openBySeverity.low} low`);
+  L.push(`- **Median time-to-fix:** ${a.medianTimeToFixDays == null ? "-" : `${a.medianTimeToFixDays} day(s)`}`);
+  L.push(`- **Oldest open finding:** ${a.oldestOpenDays} day(s)`);
+  L.push(`- **Ageing of open items:** 0-7d: ${a.ageing["0-7"]} · 8-30d: ${a.ageing["8-30"]} · 31-90d: ${a.ageing["31-90"]} · 90d+: ${a.ageing["90+"]}`);
+  if (a.lastScan) L.push(`- **Since last scan:** ${a.lastScan.new} new · ${a.lastScan.resolved} resolved${a.lastScan.reopened ? ` · ${a.lastScan.reopened} reopened` : ""}`);
+  L.push("");
+  if (a.open.length) {
+    L.push("| Open finding | Severity | Age (days) |");
+    L.push("|--------------|----------|-----------|");
+    for (const o of a.open.slice(0, 20)) {
+      L.push(`| ${o.title}${o.url ? ` (${o.url})` : ""} | ${o.severity} | ${o.ageDays ?? "-"} |`);
+    }
+    L.push("");
+  }
+  L.push("> Measure f is about closing the loop, not lengthening the backlog. Track time-to-fix and ageing over time; re-run on a cadence.");
+  L.push("");
+  return L.join("\n");
+}

package/lib/blackbox/adaptive-scan.js

@@ -0,0 +1,137 @@
+/**
+ * Adaptive black-box probes for live URL scans.
+ * Uses OpenAPI paths and discovered endpoints as the attack surface.
+ */
+import { SecurityKnowledgeGraph } from "../knowledge-graph.js";
+import {
+  probeAuthEscalation,
+  probeMethodConfusion,
+  probeInfoLeakage,
+  probeCors,
+  probePrototypePollution,
+  probeContentTypeConfusion,
+  probeTimingAnalysis,
+} from "../adaptive-probes.js";
+import { normalizeBaseUrl } from "../../src/utils/url.js";
+
+function graphFindingToPenthera(base, finding) {
+  return {
+    severity: finding.severity,
+    title: finding.title,
+    description: finding.detail || finding.title,
+    url: finding.routeUrl ? `${base}${finding.routeUrl}` : base,
+    category: finding.category,
+    source: "adaptive-probe",
+  };
+}
+
+/** Build route objects from OpenAPI paths. */
+export function routesFromOpenApi(spec, paths = []) {
+  if (!spec?.paths && paths.length === 0) return [];
+
+  const pathMap = spec?.paths || {};
+  const entries = paths.length > 0 ? paths : Object.keys(pathMap);
+
+  return entries.map((path) => {
+    const ops = pathMap[path] || {};
+    const methods = Object.keys(ops)
+      .filter((m) => ["get", "post", "put", "patch", "delete", "options"].includes(m))
+      .map((m) => m.toUpperCase());
+    const security = ops.get?.security || ops.post?.security || spec?.security;
+    const auth = security?.length ? ["user"] : ["public"];
+
+    return {
+      url: path.startsWith("/") ? path : `/${path}`,
+      methods: methods.length ? methods : ["GET"],
+      auth,
+      inputs: [],
+      externalCalls: [],
+      securityPatterns: [],
+    };
+  });
+}
+
+/** Build routes from discovered endpoint list. */
+export function routesFromEndpoints(endpoints = []) {
+  return endpoints
+    .filter((ep) => ep.path && ep.status !== 404)
+    .slice(0, 25)
+    .map((ep) => ({
+      url: ep.path.startsWith("/") ? ep.path : `/${ep.path}`,
+      methods: ["GET"],
+      auth: ["public"],
+      inputs: [],
+      externalCalls: [],
+      securityPatterns: [],
+    }));
+}
+
+/**
+ * Run adaptive probe chains against a live target.
+ *
+ * @param {string} target
+ * @param {object} opts - { spec, paths, endpoints, maxRoutes, onPhase }
+ */
+export async function runAdaptiveProbes(target, opts = {}) {
+  const base = normalizeBaseUrl(target);
+  const progress = opts.onPhase || (() => {});
+  const graph = new SecurityKnowledgeGraph();
+
+  let routes = routesFromOpenApi(opts.spec, opts.paths);
+  if (routes.length === 0) {
+    routes = routesFromEndpoints(opts.endpoints);
+  }
+
+  const maxRoutes = opts.maxRoutes || 15;
+  routes = routes.slice(0, maxRoutes);
+
+  if (routes.length === 0) {
+    return [];
+  }
+
+  progress(`Running adaptive probes on ${routes.length} routes...`);
+
+  const allFindings = [];
+  const probes = [
+    probeAuthEscalation,
+    probeMethodConfusion,
+    probeInfoLeakage,
+    probeCors,
+    probePrototypePollution,
+    probeContentTypeConfusion,
+    probeTimingAnalysis,
+  ];
+
+  for (const route of routes) {
+    graph.addRoute(route);
+    for (const probe of probes) {
+      try {
+        const findings = await probe(base, route, graph);
+        allFindings.push(...findings.map((f) => graphFindingToPenthera(base, f)));
+      } catch {
+        // Skip failing probe on this route
+      }
+    }
+  }
+
+  // Surface correlated attack chains the knowledge graph found (e.g. an auth
+  // bypass that combines with an IDOR into cross-user access). This was computed
+  // nowhere before; it's the highest-value output of the graph. Best-effort.
+  try {
+    for (const chain of graph.discoverChains()) {
+      allFindings.push({
+        severity: chain.severity,
+        title: `Attack chain: ${chain.type}`,
+        description: chain.description,
+        url: base,
+        category: "attack-chain",
+        source: "adaptive-probe",
+        chain: chain.path || [],
+      });
+    }
+  } catch {
+    // Chain discovery is best-effort; never fail the scan over it.
+  }
+
+  return allFindings;
+}

package/lib/blackbox/client-auth.js

@@ -0,0 +1,60 @@
+/**
+ * Detect client-side-only authentication (UI hides data but API is open).
+ */
+import { joinUrl } from "../../src/utils/url.js";
+import { probeFetch } from "../../src/utils/http.js";
+
+const SCRIPT_CANDIDATES = ["/script.js", "/app.js", "/main.js", "/static/script.js"];
+
+/**
+ * @param {string} baseUrl
+ */
+export async function probeClientSideAuth(baseUrl, opts = {}) {
+  const timeout = opts.timeout || 8_000;
+  const findings = [];
+
+  for (const path of SCRIPT_CANDIDATES) {
+    const res = await probeFetch(joinUrl(baseUrl, path), { timeout });
+    if (!res || res.status !== 200 || !/function|const|fetch/i.test(res.body)) continue;
+
+    const loadsDataOnBoot = /DOMContentLoaded|initApp|document\.ready/i.test(res.body)
+      && /loadAndRender|fetch\s*\(\s*['"][^'"]+\.json/i.test(res.body);
+    const hasLoginGate = /doLogin|loginScreen|Sign In/i.test(res.body);
+
+    if (loadsDataOnBoot && hasLoginGate) {
+      findings.push({
+        severity: "high",
+        title: "Client-side-only authentication",
+        description:
+          `${path} loads dashboard data on page init before login completes. ` +
+          "If API endpoints are unauthenticated, all data is accessible without credentials.",
+        url: joinUrl(baseUrl, path),
+        status: 200,
+        category: "auth",
+        source: "client-auth-probe",
+      });
+    }
+
+    // Extract JSON data URLs referenced in script
+    const dataUrls = [...res.body.matchAll(/['"]([^'"]+\.json)['"]/g)].map((m) => m[1]);
+    for (const rel of [...new Set(dataUrls)].slice(0, 5)) {
+      const dataPath = rel.startsWith("/") ? rel : `/${rel}`;
+      const dataRes = await probeFetch(joinUrl(baseUrl, dataPath), { timeout });
+      if (dataRes && dataRes.status === 200 && dataRes.body.length > 10) {
+        findings.push({
+          severity: "high",
+          title: `Sensitive data exposed: GET ${dataPath}`,
+          description: "JSON data endpoint responds without authentication.",
+          url: joinUrl(baseUrl, dataPath),
+          status: 200,
+          category: "auth",
+          source: "client-auth-probe",
+        });
+      }
+    }
+
+    break;
+  }
+
+  return findings;
+}

package/lib/blackbox/email-dns.js

@@ -0,0 +1,135 @@
+/**
+ * Email authentication posture, SPF, DMARC, and MX via DNS.
+ *
+ * Spoofed email is one of the most common attack vectors, and every security
+ * questionnaire asks about it, yet most vibecoded apps ship without SPF or
+ * DMARC. These are passive DNS lookups against the domain you're scanning, no
+ * traffic to the app itself.
+ *
+ * DKIM needs an organisation-specific selector, which can't be enumerated
+ * reliably, so we flag it as a manual follow-up rather than guessing selectors.
+ *
+ * The resolver is injectable so the evaluation logic is unit-testable without
+ * live DNS.
+ */
+import { resolveTxt as dnsResolveTxt, resolveMx as dnsResolveMx } from "node:dns/promises";
+
+const DEFAULT_RESOLVER = { resolveTxt: dnsResolveTxt, resolveMx: dnsResolveMx };
+
+async function safeTxt(resolver, name) {
+  try {
+    const records = await resolver.resolveTxt(name);
+    // resolveTxt returns string[][]; each record's chunks must be joined.
+    return records.map((chunks) => (Array.isArray(chunks) ? chunks.join("") : String(chunks)));
+  } catch {
+    return [];
+  }
+}
+
+async function safeMx(resolver, name) {
+  try {
+    return await resolver.resolveMx(name);
+  } catch {
+    return [];
+  }
+}
+
+/** Evaluate SPF from a domain's root TXT records. */
+export function evaluateSpf(txtRecords) {
+  const spf = (txtRecords || []).find((r) => /^v=spf1\b/i.test(r.trim()));
+  if (!spf) {
+    return { present: false, record: null, finding: {
+      severity: "medium",
+      title: "No SPF record",
+      description: "No `v=spf1` TXT record found. Anyone can send email that appears to come from this domain.",
+      category: "email-auth",
+      source: "email-dns-probe",
+    } };
+  }
+  if (/[+]all\b/i.test(spf)) {
+    return { present: true, record: spf, finding: {
+      severity: "high",
+      title: "SPF allows all senders (+all)",
+      description: `SPF record ends in \`+all\`, which authorises any server to send as this domain: \`${spf.slice(0, 140)}\``,
+      category: "email-auth",
+      source: "email-dns-probe",
+    } };
+  }
+  if (/[-~]all\b/i.test(spf) || /\bredirect=/i.test(spf)) {
+    return { present: true, record: spf, finding: null }; // -all / ~all / delegated
+  }
+  return { present: true, record: spf, finding: {
+    severity: "medium",
+    title: "SPF does not restrict senders",
+    description: `SPF record has no \`-all\` or \`~all\` mechanism, so spoofing is not restricted: \`${spf.slice(0, 140)}\``,
+    category: "email-auth",
+    source: "email-dns-probe",
+  } };
+}
+
+/** Evaluate DMARC from the `_dmarc.<domain>` TXT records. */
+export function evaluateDmarc(txtRecords) {
+  const dmarc = (txtRecords || []).find((r) => /^v=DMARC1\b/i.test(r.trim()));
+  if (!dmarc) {
+    return { present: false, record: null, policy: null, finding: {
+      severity: "medium",
+      title: "No DMARC record",
+      description: "No `v=DMARC1` record at `_dmarc.<domain>`. Without DMARC, receivers can't act on or report spoofed mail.",
+      category: "email-auth",
+      source: "email-dns-probe",
+    } };
+  }
+  const policy = (dmarc.match(/\bp=(none|quarantine|reject)\b/i)?.[1] || "none").toLowerCase();
+  if (policy === "none") {
+    return { present: true, record: dmarc, policy, finding: {
+      severity: "low",
+      title: "DMARC policy is monitoring-only (p=none)",
+      description: "DMARC is published but `p=none` only monitors, it does not block spoofed mail. Move to `quarantine` or `reject` once your legitimate senders are aligned.",
+      category: "email-auth",
+      source: "email-dns-probe",
+    } };
+  }
+  return { present: true, record: dmarc, policy, finding: null };
+}
+
+/**
+ * Probe a domain's email authentication posture.
+ * @param {string} domain
+ * @param {object} opts - { resolver }
+ */
+export async function probeEmailDns(domain, opts = {}) {
+  const resolver = opts.resolver || DEFAULT_RESOLVER;
+  const findings = [];
+
+  const [rootTxt, dmarcTxt, mx] = await Promise.all([
+    safeTxt(resolver, domain),
+    safeTxt(resolver, `_dmarc.${domain}`),
+    safeMx(resolver, domain),
+  ]);
+
+  const spf = evaluateSpf(rootTxt);
+  const dmarc = evaluateDmarc(dmarcTxt);
+  if (spf.finding) findings.push(spf.finding);
+  if (dmarc.finding) findings.push(dmarc.finding);
+
+  // DKIM: selector is provider-specific and can't be enumerated. Only nudge when
+  // the domain clearly handles mail (has MX) and email auth is already imperfect.
+  if (mx.length > 0 && (spf.finding || dmarc.finding)) {
+    findings.push({
+      severity: "info",
+      title: "Verify DKIM is signing outbound mail",
+      description: "This domain has MX records (it handles mail) and its SPF/DMARC is incomplete. Also confirm DKIM signing, the selector is provider-specific, so Penthera does not auto-check it.",
+      category: "email-auth",
+      source: "email-dns-probe",
+    });
+  }
+
+  return {
+    domain,
+    spf: spf.record,
+    dmarc: dmarc.record,
+    dmarcPolicy: dmarc.policy || null,
+    mxCount: mx.length,
+    findings,
+  };
+}

package/lib/blackbox/headers.js

@@ -0,0 +1,91 @@
+/**
+ * HTTP security header audit.
+ */
+
+const HEADER_CHECKS = [
+  {
+    header: "x-frame-options",
+    severity: "low",
+    title: "Missing X-Frame-Options",
+    description: "Page can be embedded in iframes (clickjacking risk).",
+    category: "headers",
+  },
+  {
+    header: "content-security-policy",
+    severity: "medium",
+    title: "Missing Content-Security-Policy",
+    description: "No CSP, inline scripts and third-party resources are unrestricted.",
+    category: "headers",
+  },
+  {
+    header: "x-content-type-options",
+    severity: "low",
+    title: "Missing X-Content-Type-Options",
+    description: "Browser may MIME-sniff responses (XSS aid).",
+    category: "headers",
+  },
+  {
+    header: "referrer-policy",
+    severity: "info",
+    title: "Missing Referrer-Policy",
+    description: "Referrer headers may leak internal URLs to third parties.",
+    category: "headers",
+  },
+  {
+    header: "permissions-policy",
+    severity: "info",
+    title: "Missing Permissions-Policy",
+    description: "Browser features (camera, geolocation) are not restricted by policy header.",
+    category: "headers",
+  },
+];
+
+/**
+ * @param {Record<string, string>} headers - lowercase header map
+ * @param {string} url
+ * @param {object} opts
+ */
+export function auditSecurityHeaders(headers = {}, url, opts = {}) {
+  const findings = [];
+  const local = opts.local || false;
+  const isHttps = url.startsWith("https://");
+
+  for (const check of HEADER_CHECKS) {
+    const present = Object.keys(headers).some((h) => h.toLowerCase() === check.header);
+    if (present) continue;
+
+    findings.push({
+      severity: check.severity,
+      title: check.title,
+      description: check.description,
+      url,
+      category: check.category,
+      source: "header-audit",
+    });
+  }
+
+  if (!isHttps && !local) {
+    findings.push({
+      severity: "high",
+      title: "No HTTPS, traffic sent in cleartext",
+      description: "Credentials and session data can be intercepted on the network. Use TLS in production.",
+      url,
+      category: "transport",
+      source: "header-audit",
+    });
+  }
+
+  const server = headers.server || headers["x-powered-by"];
+  if (server && /uvicorn|gunicorn|werkzeug/i.test(server)) {
+    findings.push({
+      severity: "info",
+      title: "Server version disclosed",
+      description: `Response exposes server header: ${server}`,
+      url,
+      category: "disclosure",
+      source: "header-audit",
+    });
+  }
+
+  return findings;
+}