penthera 1.2.0 → 1.4.0

package/lib/memory/views.js

@@ -0,0 +1,198 @@
+/**
+ * Read-only views over memory events, plus the two pure helpers scans use:
+ * rankFindings() re-orders, isSuppressed() hides. Neither can add a finding,
+ * change a severity, or run anything: memory content is only ever compared and
+ * counted. Every field read from an event is type-checked, so a malformed or
+ * hostile event degrades to "ignored", never to behaviour.
+ */
+import { fingerprintFinding } from "../audit-loop.js";
+import { sha256 } from "./store.js";
+
+const SEV_ORDER = { critical: 0, high: 1, medium: 2, low: 3, info: 4 };
+const MAX_STR = 500;
+
+/** Short, stable id for a finding: sha256 of the audit-loop fingerprint. */
+export function findingId(f) {
+  return sha256(fingerprintFinding(f)).slice(0, 16);
+}
+
+const str = (x) => (typeof x === "string" && x.length > 0 && x.length <= MAX_STR ? x : null);
+const int = (x) => (Number.isInteger(x) && x >= 0 ? x : null);
+const time = (x) => { const t = Date.parse(x); return Number.isNaN(t) ? null : t; };
+const key = (x) => (str(x) ? x.toLowerCase() : "unknown");
+
+function median(nums) {
+  if (!nums.length) return null;
+  const s = [...nums].sort((a, b) => a - b);
+  const mid = Math.floor(s.length / 2);
+  return s.length % 2 ? s[mid] : (s[mid - 1] + s[mid]) / 2;
+}
+
+/**
+ * Build the views from a verified event list.
+ * @param {object[]} events  parsed events from openMemory().events()
+ * @param {{ now?: string }} opts  `now` decides which suppressions have expired
+ */
+export function buildViews(events, { now = new Date().toISOString() } = {}) {
+  const nowMs = time(now) ?? Date.now();
+  const recipes = new Map();      // `${framework}\0${recipe}` -> stats
+  const lastApplied = new Map();  // finding id -> { recipe, framework, category, ts }
+  const firstSeen = new Map();    // finding id -> ms
+  const seenCount = new Map();    // finding id -> scans it appeared in
+  const fixDurations = [];
+  const suppressions = new Map(); // finding id -> latest suppression
+  const probes = new Map();       // `${stack}\0${probe}` -> stats
+  let scans = 0;
+
+  const recipeStats = (framework, recipe) => {
+    const k = `${framework}\0${recipe}`;
+    if (!recipes.has(k)) recipes.set(k, { framework, recipe, applied: 0, verified: 0, failed: 0, categories: new Set() });
+    return recipes.get(k);
+  };
+
+  for (const ev of events || []) {
+    const d = ev?.data;
+    if (!d || typeof d !== "object") continue;
+    const ts = time(ev.ts);
+    switch (ev.type) {
+      case "scan": {
+        scans++;
+        const ids = Array.isArray(d.ids) ? d.ids.filter(str) : [];
+        for (const id of ids) {
+          if (!firstSeen.has(id) && ts != null) firstSeen.set(id, ts);
+          seenCount.set(id, (seenCount.get(id) || 0) + 1);
+        }
+        break;
+      }
+      case "fix_applied": {
+        const id = str(d.id) || str(d.fingerprint);
+        const recipe = str(d.recipe);
+        if (!recipe) break;
+        const framework = key(d.framework);
+        const s = recipeStats(framework, recipe);
+        s.applied++;
+        if (str(d.category)) s.categories.add(d.category);
+        if (id) lastApplied.set(id, { recipe, framework, category: str(d.category), ts });
+        break;
+      }
+      case "fix_verified":
+      case "fix_failed": {
+        const id = str(d.id) || str(d.fingerprint);
+        const prior = id ? lastApplied.get(id) : null;
+        const recipe = str(d.recipe) || prior?.recipe;
+        if (!recipe) break;
+        const framework = str(d.framework) ? key(d.framework) : (prior?.framework || "unknown");
+        const s = recipeStats(framework, recipe);
+        if (ev.type === "fix_verified") {
+          s.verified++;
+          if (id && ts != null && firstSeen.has(id)) fixDurations.push(Math.max(0, ts - firstSeen.get(id)));
+        } else {
+          s.failed++;
+        }
+        break;
+      }
+      case "fp_suppressed": {
+        const id = str(d.id) || (str(d.fingerprint) ? sha256(d.fingerprint).slice(0, 16) : null);
+        const reason = str(d.reason);
+        if (!id || !reason) break; // a suppression without a reason is ignored
+        const expires = str(d.expires);
+        if (expires && (time(expires) == null || time(expires) <= nowMs)) { suppressions.delete(id); break; }
+        suppressions.set(id, {
+          id, reason, actor: str(d.actor) || "unknown", since: ev.ts, expires: expires || null, source: "memory",
+        });
+        break;
+      }
+      case "probe_yield": {
+        const probe = str(d.probe);
+        const findings = int(d.findings);
+        if (!probe || findings == null) break;
+        const stack = key(d.stack);
+        const k = `${stack}\0${probe}`;
+        if (!probes.has(k)) probes.set(k, { stack, probe, runs: 0, findings: 0 });
+        const p = probes.get(k);
+        p.runs++;
+        p.findings += findings;
+        break;
+      }
+      default:
+        break;
+    }
+  }
+
+  const recipeSuccess = [...recipes.values()].map((s) => {
+    const decided = s.verified + s.failed;
+    return {
+      framework: s.framework, recipe: s.recipe, applied: s.applied, verified: s.verified, failed: s.failed,
+      rate: decided ? s.verified / decided : null, categories: [...s.categories].sort(),
+    };
+  }).sort((a, b) => a.framework.localeCompare(b.framework) || (b.rate ?? -1) - (a.rate ?? -1) || a.recipe.localeCompare(b.recipe));
+
+  const probeRanking = {};
+  for (const p of probes.values()) {
+    (probeRanking[p.stack] ||= []).push({ probe: p.probe, runs: p.runs, findings: p.findings, yield: p.findings / p.runs });
+  }
+  for (const list of Object.values(probeRanking)) {
+    list.sort((a, b) => b.yield - a.yield || b.findings - a.findings || a.probe.localeCompare(b.probe));
+  }
+
+  const med = median(fixDurations);
+  return {
+    scans,
+    recipeSuccess,
+    suppressions: [...suppressions.values()],
+    probeRanking,
+    medianTimeToFixMs: med,
+    medianTimeToFixDays: med == null ? null : Math.round((med / 86_400_000) * 10) / 10,
+    seen: Object.fromEntries(seenCount),
+  };
+}
+
+/**
+ * Add repo-file suppressions to a views object. They keep their own source so
+ * reports can say "suppressed by repo file". A local-memory suppression for the
+ * same id wins (the user decided it, not the repo).
+ */
+export function withRepoSuppressions(views, repoEntries) {
+  const have = new Set((views?.suppressions || []).map((s) => s.id));
+  const extra = (repoEntries || []).filter((s) => !have.has(s.id)).map((s) => ({ ...s, source: "repo-file" }));
+  return { ...(views || {}), suppressions: [...(views?.suppressions || []), ...extra] };
+}
+
+/**
+ * Is this finding suppressed? Returns the suppression record or null.
+ * Matches on the finding's id (or its full fingerprint, for repo files).
+ */
+export function isSuppressed(finding, views) {
+  const list = views?.suppressions;
+  if (!list?.length) return null;
+  const fp = fingerprintFinding(finding);
+  const id = sha256(fp).slice(0, 16);
+  return list.find((s) => s.id === id || (s.fingerprint && s.fingerprint === fp)) || null;
+}
+
+/**
+ * Re-order findings using what memory learned. Severity always comes first and
+ * is never changed. Within a severity:
+ *   1. findings whose category has a recipe with a proven fix rate (quick wins),
+ *   2. findings seen in more past scans (persistent problems),
+ *   3. then the original order (the sort is stable).
+ * Returns a new array with exactly the same findings.
+ */
+export function rankFindings(findings, views, { framework = null } = {}) {
+  const fw = framework ? String(framework).toLowerCase() : null;
+  const bestRate = new Map(); // category -> best verified rate
+  for (const r of views?.recipeSuccess || []) {
+    if (r.rate == null || (fw && r.framework !== fw && r.framework !== "unknown")) continue;
+    for (const c of r.categories || []) bestRate.set(c, Math.max(bestRate.get(c) ?? 0, r.rate));
+  }
+  const seen = views?.seen || {};
+  return (findings || [])
+    .map((f, i) => ({
+      f, i,
+      sev: SEV_ORDER[f.severity] ?? 5,
+      fix: bestRate.get(f.category) ?? 0,
+      seen: seen[findingId(f)] || 0,
+    }))
+    .sort((a, b) => a.sev - b.sev || b.fix - a.fix || b.seen - a.seen || a.i - b.i)
+    .map((x) => x.f);
+}

package/lib/params.js

@@ -14,20 +14,12 @@
  * Inspired by: Arjun (6.2k stars), ParamSpider
  */
 
+import { fetchText } from "../src/utils/http.js";
+
 const PARAM_TIMEOUT = 8_000;
 
-async function safeFetch(url, opts = {}) {
-  const controller = new AbortController();
-  const timeout = setTimeout(() => controller.abort(), opts.timeout || PARAM_TIMEOUT);
-  try {
-    const res = await fetch(url, { ...opts, signal: controller.signal, redirect: "manual" });
-    const body = await res.text().catch(() => "");
-    return { status: res.status, body, size: body.length, ok: true };
-  } catch {
-    return { status: 0, body: "", size: 0, ok: false };
-  } finally {
-    clearTimeout(timeout);
-  }
+function safeFetch(url, opts = {}) {
+  return fetchText(url, { timeout: PARAM_TIMEOUT, probe: "param-discovery", ...opts });
 }
 
 function simpleHash(str) {

package/lib/provenance.js

@@ -26,6 +26,7 @@ const SOURCE_CONFIDENCE = {
   "openapi-scan": "confirmed",
   "email-dns-probe": "confirmed",
   "framework-check": "likely",
+  "deps-scan": "likely",
   "static-analysis": "likely",
   "trust-boundary": "likely",
   "auth-probe": "likely",
@@ -59,6 +60,7 @@ const CATEGORY_CONFIDENCE = {
   ssrf: "likely",
   "open-redirect": "likely",
   fuzzing: "potential",
+  "dependency-risk": "likely",
 };
 
 /** Honest-confidence label for a finding. An explicit finding.confidence wins. */

package/lib/recon.js

@@ -12,19 +12,16 @@
  * Inspired by: subfinder, gau, waybackurls, ParamSpider
  */
 
+import { safeFetch } from "../src/utils/http.js";
+import { registrableDomain } from "../src/utils/url.js";
+
 const RECON_TIMEOUT = 15_000;
 
-async function safeFetch(url, opts = {}) {
-  const controller = new AbortController();
-  const timeout = setTimeout(() => controller.abort(), opts.timeout || RECON_TIMEOUT);
-  try {
-    return await fetch(url, { signal: controller.signal, ...opts });
-  } catch {
-    return null;
-  } finally {
-    clearTimeout(timeout);
-  }
-}
+/**
+ * Third-party hosts this module queries. Under --offline the scan skips recon
+ * entirely, and the HTTP client would refuse these hosts anyway.
+ */
+export const RECON_HOSTS = ["crt.sh", "web.archive.org", "otx.alienvault.com"];
 
 // ── Subdomain Discovery via Certificate Transparency ─────────────────────
 
@@ -38,7 +35,7 @@ async function safeFetch(url, opts = {}) {
 export async function discoverSubdomains(domain) {
   const res = await safeFetch(
     `https://crt.sh/?q=%25.${encodeURIComponent(domain)}&output=json`,
-    { timeout: 20_000 },
+    { timeout: 20_000, probe: "recon", redirect: "follow" },
   );
   if (!res || !res.ok) return [];
 
@@ -77,7 +74,7 @@ export async function fetchWaybackUrls(domain, opts = {}) {
   const limit = opts.limit || 1000;
   const res = await safeFetch(
     `https://web.archive.org/cdx/search/cdx?url=*.${encodeURIComponent(domain)}/*&output=json&fl=original&collapse=urlkey&limit=${limit}`,
-    { timeout: 20_000 },
+    { timeout: 20_000, probe: "recon", redirect: "follow" },
   );
   if (!res || !res.ok) return [];
 
@@ -103,6 +100,7 @@ export async function fetchOtxUrls(domain, opts = {}) {
   const limit = opts.limit || 200;
   const res = await safeFetch(
     `https://otx.alienvault.com/api/v1/indicators/domain/${encodeURIComponent(domain)}/url_list?limit=${limit}`,
+    { timeout: RECON_TIMEOUT, probe: "recon", redirect: "follow" },
   );
   if (!res || !res.ok) return [];
 
@@ -180,13 +178,7 @@ export async function runRecon(domain, opts = {}) {
  */
 export function extractDomain(urlString) {
   try {
-    const hostname = new URL(urlString).hostname;
-    const parts = hostname.split(".");
-    // Handle two-part TLDs like .co.uk, .com.au
-    if (parts.length >= 3 && parts[parts.length - 2].length <= 3) {
-      return parts.slice(-3).join(".");
-    }
-    return parts.slice(-2).join(".");
+    return registrableDomain(new URL(urlString).hostname);
   } catch {
     return urlString;
   }

package/lib/references.js

@@ -0,0 +1,31 @@
+/**
+ * Research and standards references for findings.
+ *
+ * lib/references.json maps each finding category to 1-3 keys of
+ * docs/references.bib, and carries a short record for each key so reports can
+ * cite without parsing BibTeX. Every entry was checked against the publisher or
+ * standards body before it was added; see docs/references.md.
+ */
+import { readFileSync } from "node:fs";
+
+const DATA = JSON.parse(readFileSync(new URL("./references.json", import.meta.url), "utf-8"));
+
+export const SOURCES = DATA.sources;
+export const CATEGORY_REFERENCES = DATA.categories;
+
+/** Reference keys for a finding, by category. Unknown categories get []. */
+export function referenceKeysFor(finding) {
+  return CATEGORY_REFERENCES[finding?.category] || [];
+}
+
+/** Resolved source records ({ key, cite, title, url }) for a finding. */
+export function referencesFor(finding) {
+  return referenceKeysFor(finding)
+    .map((key) => (SOURCES[key] ? { key, ...SOURCES[key] } : null))
+    .filter(Boolean);
+}
+
+/** One-line Markdown list of sources, e.g. "[Weichselbaum et al. 2016](https://doi.org/...)". */
+export function formatSourcesMarkdown(refs) {
+  return refs.map((r) => `[${r.cite}](${r.url})`).join(" · ");
+}