penthera 1.7.0 → 1.8.0

package/lib/hook/index.js

@@ -14,11 +14,11 @@
  * Offline, no telemetry. Findings use the standard shape, so ids match a full
  * scan and penthera.suppress.json works here too.
  */
-import { execFileSync, spawnSync } from "node:child_process";
-import { existsSync, mkdirSync, mkdtempSync, readFileSync, realpathSync, renameSync, rmSync, unlinkSync, writeFileSync, chmodSync } from "node:fs";
-import { tmpdir } from "node:os";
+import { existsSync, lstatSync, mkdirSync, readFileSync, renameSync, unlinkSync, writeFileSync, chmodSync } from "node:fs";
 import { join, posix, resolve } from "node:path";
 import { fileURLToPath } from "node:url";
+import { git, gitSpawn, gitToplevel } from "../git.js";
+import { withTempDir } from "../tmp.js";
 import { findSecretsInText, isSecretScanTarget, maskMatch } from "../whitebox/secrets.js";
 import { loadPolicy, normalizePolicy, POLICY_FILE } from "../install/policy.js";
 import { parseRepoSuppressions, REPO_SUPPRESS_FILE } from "../memory/repo-suppress.js";
@@ -28,73 +28,101 @@ import { globToRegExp, isAtLeast, SEVERITIES } from "./policy.js";
 
 export const HOOK_MARKER = "# penthera-hook";
 export const BACKUP_NAME = "pre-commit.pre-penthera";
-const MAX_BLOB_BYTES = 2 * 1024 * 1024;
+// Staged files larger than this are not read. They never pass silently: by
+// default they block the commit (policy "largeFiles": "warn" only warns).
+// Large enough for the package-lock.json of a big monorepo.
+export const MAX_BLOB_BYTES = 16 * 1024 * 1024;
+const REGULAR = new Set(["100644", "100755"]);
 
 // ── git plumbing ──────────────────────────────────────────────────────────
 
-function git(cwd, args, opts = {}) {
-  return execFileSync("git", args, { cwd, encoding: "utf-8", stdio: ["ignore", "pipe", "pipe"], maxBuffer: 256 * 1024 * 1024, ...opts });
+export function repoRoot(cwd) {
+  const root = gitToplevel(cwd);
+  if (!root) throw new Error(`Not inside a git repository: ${cwd}`);
+  return root;
 }
 
-export function repoRoot(cwd) {
-  try {
-    return git(cwd, ["rev-parse", "--show-toplevel"]).trim();
-  } catch {
-    throw new Error(`Not inside a git repository: ${cwd}`);
+/**
+ * Index entries this commit adds: added, copied, modified, renamed or
+ * type-changed (a symlink that became a file). `{ path, mode, oid }`, read
+ * NUL-separated so any byte in a name survives, with the object id of the
+ * staged blob, so content is always read by id and never by a path git
+ * might parse as revision syntax.
+ */
+export function stagedEntries(root) {
+  const parts = git(root, ["diff", "--cached", "--raw", "-z", "--no-abbrev", "--no-renames", "--diff-filter=ACMRT"]).split("\0");
+  const out = [];
+  for (let i = 0; i < parts.length - 1; i += 2) {
+    const m = /^:\d{6} (\d{6}) [0-9a-f]+ ([0-9a-f]+) [A-Z]\d*$/.exec(parts[i]);
+    if (!m) throw new Error(`unexpected git diff --raw output: ${JSON.stringify(parts[i].slice(0, 200))}`);
+    out.push({ path: parts[i + 1], mode: m[1], oid: m[2] });
   }
+  return out;
 }
 
-/** Paths added, copied, modified or renamed in the index (what this commit adds). */
-export function stagedFiles(root) {
-  return git(root, ["diff", "--cached", "--name-only", "-z", "--diff-filter=ACMR", "--no-renames"]).split("\0").filter(Boolean);
+function catFile(root, mode, oids) {
+  const r = gitSpawn(root, ["cat-file", mode], { input: oids.map((o) => `${o}\n`).join("") });
+  if (r.status !== 0) throw new Error(`git cat-file failed: ${String(r.stderr).trim()}`);
+  return r.stdout;
 }
 
-/** Read index blobs in one `git cat-file --batch` call. Returns Map(path → Buffer); large files are skipped. */
-export function readStaged(root, paths) {
-  const out = new Map();
-  const wanted = paths.filter((p) => !p.includes("\n"));
-  if (!wanted.length) return out;
-  const r = spawnSync("git", ["cat-file", "--batch"], { cwd: root, input: wanted.map((p) => `:${p}\n`).join(""), maxBuffer: 1024 * 1024 * 1024 });
-  if (r.status !== 0) throw new Error(`git cat-file failed: ${String(r.stderr).trim()}`);
-  const buf = r.stdout;
+/**
+ * Read staged blobs by object id. Only regular files (mode 100644/100755):
+ * a staged symlink or submodule has no content of its own to check.
+ * @param {{ path: string, mode: string, oid: string }[]} entries
+ * @returns {{ blobs: Map<string, Buffer>, oversized: { path: string, size: number }[] }}
+ */
+export function readStaged(root, entries, { maxBytes = MAX_BLOB_BYTES } = {}) {
+  const blobs = new Map();
+  const oversized = [];
+  const files = entries.filter((e) => REGULAR.has(e.mode));
+  if (!files.length) return { blobs, oversized };
+  const sizes = catFile(root, "--batch-check", files.map((e) => e.oid)).toString().split("\n");
+  const wanted = [];
+  files.forEach((e, i) => {
+    const m = /^([0-9a-f]+) blob (\d+)$/.exec(sizes[i] || "");
+    if (!m || m[1] !== e.oid) throw new Error(`git cat-file: staged blob for ${JSON.stringify(e.path)} not found`);
+    const size = Number(m[2]);
+    if (size > maxBytes) oversized.push({ path: e.path, size });
+    else wanted.push(e);
+  });
+  if (!wanted.length) return { blobs, oversized };
+  const buf = catFile(root, "--batch", wanted.map((e) => e.oid));
   let pos = 0;
-  for (const p of wanted) {
+  for (const e of wanted) {
     const nl = buf.indexOf(10, pos);
-    if (nl === -1) break;
-    const header = buf.subarray(pos, nl).toString();
-    pos = nl + 1;
-    const m = /^\S+ (\w+) (\d+)$/.exec(header);
-    if (!m) continue; // "<name> missing"
+    const m = nl === -1 ? null : /^([0-9a-f]+) blob (\d+)$/.exec(buf.subarray(pos, nl).toString());
+    if (!m || m[1] !== e.oid) throw new Error(`git cat-file: unexpected output for ${JSON.stringify(e.path)}`);
     const size = Number(m[2]);
-    if (m[1] === "blob" && size <= MAX_BLOB_BYTES) out.set(p, buf.subarray(pos, pos + size));
-    pos += size + 1;
+    blobs.set(e.path, buf.subarray(nl + 1, nl + 1 + size));
+    pos = nl + 1 + size + 1;
   }
-  return out;
+  return { blobs, oversized };
 }
 
 // ── Checks ────────────────────────────────────────────────────────────────
 
-function lineOf(text, index) {
-  let n = 1;
-  for (let i = 0; i < index; i++) if (text.charCodeAt(i) === 10) n++;
-  return n;
-}
-
 function checkSecrets(files, blobs) {
   const findings = [];
   for (const path of files) {
     if (!isSecretScanTarget(path)) continue;
     const buf = blobs.get(path);
-    if (!buf || buf.subarray(0, 8192).includes(0)) continue;
+    if (!buf) continue;
+    // Read as text even when it holds NUL bytes: a NUL does not stop a key
+    // from being a key, and the extension already says this is a text file.
     const text = buf.toString("utf-8");
-    for (const hit of findSecretsInText(text)) {
+    const hits = findSecretsInText(text).sort((a, b) => a.index - b.index);
+    let line = 1;
+    let at = 0;
+    for (const hit of hits) {
+      for (; at < hit.index; at++) if (text.charCodeAt(at) === 10) line++;
       // Same title/url/category/source as the repo scan, so finding ids match.
       findings.push({
         severity: hit.severity,
         title: `${hit.name} in source`,
         description: `Possible secret in \`${path}\`: \`${maskMatch(hit.match)}\``,
         url: path,
-        line: lineOf(text, text.indexOf(hit.match)),
+        line,
         evidence: maskMatch(hit.match),
         category: "secrets",
         source: "secret-scan",
@@ -126,9 +154,8 @@ async function checkAgents(root, staged) {
     .map((l) => /^(\d{6}) [0-9a-f]+ \d\t(.*)$/s.exec(l))
     .filter((m) => m && (m[1] === "100644" || m[1] === "100755") && isAgentConfig(m[2]))
     .map((m) => m[2]);
-  const tmp = realpathSync(mkdtempSync(join(tmpdir(), "penthera-hook-")));
-  try {
-    const r = spawnSync("git", ["checkout-index", "-z", "--stdin", `--prefix=${tmp}/`], { cwd: root, input: all.join("\0") });
+  return withTempDir("penthera-hook-", async (tmp) => {
+    const r = gitSpawn(root, ["checkout-index", "-z", "--stdin", `--prefix=${tmp}/`], { input: all.join("\0") });
     if (r.status !== 0) throw new Error(`git checkout-index failed: ${String(r.stderr).trim()}`);
     const { scanAgents } = await import("../agents/index.js");
     const result = await scanAgents(tmp, { lockPath: join(tmp, ".penthera-hook-no-lock") });
@@ -140,9 +167,7 @@ async function checkAgents(root, staged) {
         return m ? { ...f, line: Number(m[1]) } : f;
       });
     return { findings, files: touched.length };
-  } finally {
-    rmSync(tmp, { recursive: true, force: true });
-  }
+  });
 }
 
 const EMPTY_LOCK = { lockfileVersion: 3, packages: {} };
@@ -182,15 +207,15 @@ function suppressedBy(entries, f) {
 
 const bySeverity = (a, b) => SEVERITIES.indexOf(a.severity) - SEVERITIES.indexOf(b.severity) || String(a.url).localeCompare(String(b.url)) || (a.line || 0) - (b.line || 0);
 
-/**
- * Check what is staged. Never writes to the repo.
- * @param {string} cwd anywhere inside the repo
- * @param {{ policyPath?: string, now?: string }} opts
- * @returns {Promise<{ root, staged, checked, blocking, below, suppressed, blockOn, policyPath, warnings, duration }>}
- */
-/** One file as staged, or null when it is not in the index. */
+/** One regular file as staged (stage 0), or null when it is not in the index. */
 function stagedBlob(root, path) {
-  return readStaged(root, [path]).get(path) || null;
+  const line = git(root, ["--literal-pathspecs", "ls-files", "-s", "-z", "--", path]).split("\0")
+    .map((l) => /^(\d{6}) ([0-9a-f]+) 0\t(.*)$/s.exec(l))
+    .find((m) => m && m[3] === path);
+  if (!line) return null;
+  const { blobs, oversized } = readStaged(root, [{ path, mode: line[1], oid: line[2] }]);
+  if (oversized.length) throw new Error(`${path} (staged) is too large to read`);
+  return blobs.get(path) || null;
 }
 
 /** The policy: an explicit --policy file, else penthera.policy.json as staged, else defaults. */
@@ -207,18 +232,42 @@ function stagedPolicy(root, explicitPath) {
   return { policy: normalizePolicy(raw, `${POLICY_FILE} (staged)`), path: join(root, POLICY_FILE) };
 }
 
+/**
+ * Check what is staged. Never writes to the repo.
+ * @param {string} cwd anywhere inside the repo
+ * @param {{ policyPath?: string, now?: string, maxBlobBytes?: number }} opts
+ * @returns {Promise<{ root, staged, checked, blocking, below, suppressed, blockOn, policyPath, warnings, duration }>}
+ */
 export async function runHook(cwd, opts = {}) {
   const started = Date.now();
   const root = repoRoot(cwd);
   const { policy, path: policyPath } = stagedPolicy(root, opts.policyPath || null);
-  const { blockOn, checks, ignore } = policy.hook;
+  const { blockOn, checks, ignore, largeFiles } = policy.hook;
   const ignored = ignore.map(globToRegExp);
-  const staged = stagedFiles(root).filter((p) => !ignored.some((re) => re.test(p)));
-
-  const wantBlobs = staged.filter((p) => (checks.secrets && isSecretScanTarget(p)) || /(^|\/)(package-lock|npm-shrinkwrap)\.json$/.test(p));
-  const blobs = readStaged(root, wantBlobs);
+  const entries = stagedEntries(root).filter((e) => !ignored.some((re) => re.test(e.path)));
+  const staged = entries.map((e) => e.path);
+
+  const isLock = (p) => /(^|\/)(package-lock|npm-shrinkwrap)\.json$/.test(p);
+  const wantBlobs = entries.filter((e) => (checks.secrets && isSecretScanTarget(e.path)) || (checks.deps && isLock(e.path)));
+  const { blobs, oversized } = readStaged(root, wantBlobs, { maxBytes: opts.maxBlobBytes ?? MAX_BLOB_BYTES });
+  const warnings = [];
+  const unchecked = [];
+  for (const o of oversized) {
+    const limit = `${Math.round((opts.maxBlobBytes ?? MAX_BLOB_BYTES) / 1024 / 1024 * 10) / 10} MB`;
+    warnings.push(`${o.path} is ${o.size} bytes, over the ${limit} limit, so it was not checked${largeFiles === "block" ? " (this blocks the commit)" : ""}`);
+    if (largeFiles === "block") {
+      unchecked.push({
+        severity: "critical",
+        title: "Staged file too large to check",
+        description: `\`${o.path}\` is ${o.size} bytes, over the ${limit} the hook reads, so it could hold a secret nobody checked.`,
+        url: o.path,
+        category: "unchecked",
+        source: "hook",
+      });
+    }
+  }
 
-  const secrets = checks.secrets ? checkSecrets(staged, blobs) : [];
+  const secrets = checks.secrets ? [...checkSecrets(staged, blobs), ...unchecked] : unchecked;
   const agents = checks.agents ? await checkAgents(root, staged) : { findings: [], files: 0 };
   const deps = checks.deps ? await checkLockfiles(root, staged, blobs, { licensePolicy: checks.licenses ? policy.licenses : null }) : { findings: [], files: 0 };
 
@@ -248,7 +297,7 @@ export async function runHook(cwd, opts = {}) {
     blocking: kept.filter((f) => isAtLeast(f.severity, blockOn)),
     below: kept.filter((f) => !isAtLeast(f.severity, blockOn)),
     suppressed,
-    warnings: sup.warnings,
+    warnings: [...warnings, ...sup.warnings],
     duration: Date.now() - started,
   };
 }
@@ -258,8 +307,27 @@ export function fixHint(f) {
   if (f.category === "secrets") return "Load it from an env var or secret store and unstage the file; rotate it if it was ever pushed or shared.";
   if (f.category === "license") return `Replace ${f.package}, or add it to "allowPackages" in the licenses section of penthera.policy.json.`;
   if (f.category === "dependency-risk") return `Review it before committing the lockfile: \`penthera deps --base HEAD -v\` shows the evidence.`;
-  const first = String(f.description || "").replace(/\s*\[[^\]]*\]$/, "").replace(/\s*\(line \d+\)/, "").split(/(?<=\.)\s/)[0];
-  return first || "See `penthera agents` for details.";
+  if (f.category === "unchecked") return `If it holds no secrets, add it to "ignore" (or set "largeFiles": "warn") in the hook section of penthera.policy.json.`;
+  return firstSentence(f.description) || "See `penthera agents` for details.";
+}
+
+/**
+ * The description's first sentence, without a trailing `[...]` tag or a
+ * `(line N)` note. Only the first line, capped, is looked at, and nothing
+ * here backtracks: descriptions quote agent config, which anyone can write.
+ */
+function firstSentence(description) {
+  let s = String(description || "");
+  const nl = s.indexOf("\n");
+  s = (nl === -1 ? s : s.slice(0, nl)).slice(0, 2000);
+  if (s.endsWith("]")) {
+    const open = s.lastIndexOf("[");
+    if (open !== -1 && !s.slice(open, -1).includes("]")) s = s.slice(0, open).trimEnd();
+  }
+  const line = /\(line \d+\)/.exec(s);
+  if (line) s = s.slice(0, line.index).trimEnd() + s.slice(line.index + line[0].length);
+  const stop = /\.\s/.exec(s);
+  return stop ? s.slice(0, stop.index + 1) : s;
 }
 
 // ── Install / uninstall ───────────────────────────────────────────────────
@@ -288,14 +356,22 @@ exit 0
 
 /** Where git looks for hooks (honours core.hooksPath and worktrees). */
 export function hooksDir(root) {
-  return resolve(root, git(root, ["rev-parse", "--git-path", "hooks"]).trim());
+  // raw: the repo's real core.hooksPath, which the safe config overrides.
+  return resolve(root, git(root, ["rev-parse", "--git-path", "hooks"], { raw: true }).trim());
 }
 
 function hooksPathSetting(root) {
-  try { return git(root, ["config", "--get", "core.hooksPath"]).trim() || null; } catch { return null; }
+  try { return git(root, ["config", "--get", "core.hooksPath"], { raw: true }).trim() || null; } catch { return null; }
+}
+
+/** lstat, or null when nothing is there (a dangling symlink is something). */
+function lstatOrNull(path) {
+  try { return lstatSync(path); } catch (e) { if (e.code === "ENOENT") return null; throw e; }
 }
 
-const isOurs = (path) => existsSync(path) && readFileSync(path, "utf-8").includes(HOOK_MARKER);
+// Ours only when it is a regular file with our marker: a symlink is never
+// ours, even when it points at a file that carries the marker.
+const isOurs = (path) => !!lstatOrNull(path)?.isFile() && readFileSync(path, "utf-8").includes(HOOK_MARKER);
 
 /**
  * Write .git/hooks/pre-commit. An existing hook that is not ours is kept as
@@ -314,15 +390,23 @@ export function installHook(cwd, { force = false, script = hookScript() } = {})
   const path = join(dir, "pre-commit");
   const backup = join(dir, BACKUP_NAME);
   let updated = false;
-  if (existsSync(path)) {
-    if (isOurs(path)) updated = true;
-    else if (existsSync(backup)) throw new Error(`Both ${path} and ${backup} exist. Merge them by hand, then run \`penthera hook install\` again.`);
+  const st = lstatOrNull(path);
+  if (st) {
+    // Anything that is not our own regular file (another hook, a symlink to a
+    // shared or not-yet-created hook) is moved aside as it is, never written through.
+    if (isOurs(path)) {
+      updated = true;
+      unlinkSync(path);
+    } else if (st.isDirectory()) throw new Error(`${path} is a directory. Move it away, then run \`penthera hook install\` again.`);
+    else if (lstatOrNull(backup)) throw new Error(`Both ${path} and ${backup} exist. Merge them by hand, then run \`penthera hook install\` again.`);
     else renameSync(path, backup);
   }
   mkdirSync(dir, { recursive: true });
-  writeFileSync(path, script);
+  // "wx": create a new file, and fail rather than follow anything that
+  // appeared at this path in the meantime.
+  writeFileSync(path, script, { flag: "wx", mode: 0o755 });
   chmodSync(path, 0o755);
-  return { path, chained: existsSync(backup) ? backup : null, updated };
+  return { path, chained: lstatOrNull(backup) ? backup : null, updated };
 }
 
 /**
@@ -334,10 +418,10 @@ export function uninstallHook(cwd) {
   const dir = hooksDir(root);
   const path = join(dir, "pre-commit");
   const backup = join(dir, BACKUP_NAME);
-  if (!existsSync(path)) return { path, removed: false, restored: null };
+  if (!lstatOrNull(path)) return { path, removed: false, restored: null };
   if (!isOurs(path)) throw new Error(`${path} was not installed by penthera; left untouched.`);
   unlinkSync(path);
-  if (existsSync(backup)) {
+  if (lstatOrNull(backup)) {
     renameSync(backup, path);
     return { path, removed: true, restored: path };
   }

package/lib/hook/policy.js

@@ -5,7 +5,8 @@
  *   "hook": {
  *     "blockOn": "high",                 // lowest severity that blocks: critical | high | medium | low
  *     "checks": { "secrets": true, "agents": true, "deps": true, "licenses": true },
- *     "ignore": ["tests/fixtures/**"]    // staged paths the hook skips (globs)
+ *     "ignore": ["tests/fixtures/**"],   // staged paths the hook skips (globs)
+ *     "largeFiles": "block"              // a staged file too large to check: block (default) | warn
  *   }
  */
 export const SEVERITIES = ["critical", "high", "medium", "low", "info"];
@@ -15,13 +16,16 @@ export const DEFAULT_HOOK_POLICY = Object.freeze({
   blockOn: "high",
   checks: Object.freeze(Object.fromEntries(CHECKS.map((c) => [c, true]))),
   ignore: Object.freeze([]),
+  largeFiles: "block",
 });
+const LARGE_FILES = ["block", "warn"];
 
 export function normalizeHookPolicy(raw, label = "hook") {
   if (raw == null) return { ...DEFAULT_HOOK_POLICY, checks: { ...DEFAULT_HOOK_POLICY.checks }, ignore: [] };
+  const FIELDS = ["blockOn", "checks", "ignore", "largeFiles"];
   if (typeof raw !== "object" || Array.isArray(raw)) throw new Error(`${label}: expected an object`);
-  const extra = Object.keys(raw).filter((k) => !["blockOn", "checks", "ignore"].includes(k));
-  if (extra.length) throw new Error(`${label}: unknown field(s) ${extra.join(", ")} (expected blockOn, checks, ignore)`);
+  const extra = Object.keys(raw).filter((k) => !FIELDS.includes(k));
+  if (extra.length) throw new Error(`${label}: unknown field(s) ${extra.join(", ")} (expected ${FIELDS.join(", ")})`);
   const blockOn = raw.blockOn ?? "high";
   if (!SEVERITIES.slice(0, 4).includes(blockOn)) throw new Error(`${label}: "blockOn" must be one of critical, high, medium, low`);
   const checks = { ...DEFAULT_HOOK_POLICY.checks };
@@ -35,7 +39,9 @@ export function normalizeHookPolicy(raw, label = "hook") {
   }
   const ignore = raw.ignore ?? [];
   if (!Array.isArray(ignore) || ignore.some((x) => typeof x !== "string" || !x.trim())) throw new Error(`${label}: "ignore" must be an array of path globs`);
-  return { blockOn, checks, ignore: ignore.map((x) => x.trim()) };
+  const largeFiles = raw.largeFiles ?? "block";
+  if (!LARGE_FILES.includes(largeFiles)) throw new Error(`${label}: "largeFiles" must be "block" or "warn"`);
+  return { blockOn, checks, ignore: ignore.map((x) => x.trim()), largeFiles };
 }
 
 /** Git-style path glob: `**` crosses folders, `*` and `?` stay inside one; a bare name matches at any depth. */

package/lib/install/index.js

@@ -24,9 +24,9 @@
  */
 import { execFile } from "node:child_process";
 import { promisify } from "node:util";
-import { existsSync, mkdtempSync, readFileSync, writeFileSync, copyFileSync, rmSync, mkdirSync } from "node:fs";
+import { existsSync, readFileSync, writeFileSync, copyFileSync, rmSync, mkdirSync } from "node:fs";
 import { join, resolve } from "node:path";
-import { tmpdir } from "node:os";
+import { withTempDir } from "../tmp.js";
 import { scanDeps } from "../deps/index.js";
 import { parseLockfile } from "../deps/lockfile.js";
 import { sourceFromNpmCache, npmCacheDir } from "../deps/source.js";
@@ -183,8 +183,7 @@ export async function planInstall(root, specs, opts = {}) {
   const original = Object.fromEntries(COPY.map((f) => [f, readIfExists(join(root, f))]));
   const lockName = LOCKFILES.find((f) => original[f]) || "package-lock.json";
 
-  const tmp = mkdtempSync(join(tmpdir(), "penthera-install-"));
-  try {
+  return withTempDir("penthera-install-", async (tmp) => {
     const work = join(tmp, "project");
     mkdirSync(work);
     for (const f of COPY) if (original[f]) copyFileSync(join(root, f), join(work, f));
@@ -246,9 +245,7 @@ export async function planInstall(root, specs, opts = {}) {
       typosquats: result.deps.typosquats,
       evidence: result.deps.diff.evidence,
     };
-  } finally {
-    rmSync(tmp, { recursive: true, force: true });
-  }
+  });
 }
 
 /** Packages whose new or changed install scripts would run. */

package/lib/machine.js

@@ -56,19 +56,6 @@ function verifySignature(program) {
   }
 }
 
-function runFile(file, args = [], opts = {}) {
-  try {
-    return execFileSync(file, args, {
-      encoding: "utf-8",
-      timeout: opts.timeout || 30_000,
-      maxBuffer: 10 * 1024 * 1024,
-      stdio: ["pipe", "pipe", "pipe"],
-    }).trim();
-  } catch (e) {
-    return e.stdout?.trim?.() || "";
-  }
-}
-
 function which(bin) {
   try {
     const p = execSync(`which ${bin}`, { encoding: "utf-8", stdio: ["pipe", "pipe", "pipe"] }).trim();

package/lib/memory/store.js

@@ -28,7 +28,7 @@ import { homedir } from "node:os";
 
 export const EVENT_VERSION = 1;
 export const EVENT_TYPES = Object.freeze([
-  "scan", "fix_applied", "fix_verified", "fix_failed", "fp_suppressed", "probe_yield", "loop_run",
+  "scan", "fix_applied", "fix_verified", "fix_failed", "fp_suppressed", "probe_yield", "loop_run", "gave_up",
 ]);
 const MAX_LINE_BYTES = 256 * 1024;
 const LOCK_STALE_MS = 30_000;
@@ -152,7 +152,11 @@ function sleepMs(ms) {
 
 function withLock(dir, fn) {
   const lock = join(dir, "events.lock");
-  const started = Date.now();
+  // The timeout is per holder: while other writers keep taking and releasing
+  // the lock we are queued behind progress, not stuck, however long that takes
+  // on a loaded machine. Only one holder keeping it for LOCK_TIMEOUT_MS is busy.
+  let holder = null;
+  let waitingSince = Date.now();
   let fd;
   for (;;) {
     try {
@@ -161,9 +165,12 @@ function withLock(dir, fn) {
     } catch (e) {
       if (e.code !== "EEXIST") throw e;
       try {
-        if (Date.now() - statSync(lock).mtimeMs > LOCK_STALE_MS) { unlinkSync(lock); continue; }
+        const st = statSync(lock);
+        if (Date.now() - st.mtimeMs > LOCK_STALE_MS) { unlinkSync(lock); continue; }
+        const id = `${st.ino}:${st.mtimeMs}`;
+        if (id !== holder) { holder = id; waitingSince = Date.now(); }
       } catch { continue; } // lock vanished between calls: retry
-      if (Date.now() - started > LOCK_TIMEOUT_MS) throw new Error(`memory: lock busy (${lock})`);
+      if (Date.now() - waitingSince > LOCK_TIMEOUT_MS) throw new Error(`memory: lock busy (${lock})`);
       sleepMs(5 + Math.floor(Math.random() * 20));
     }
   }

package/lib/nuclei-loader.js

@@ -307,28 +307,3 @@ export async function executeTemplates(baseUrl, templates, opts = {}) {
   return findings;
 }
 
-/**
- * Get stats about a template directory.
- */
-export async function getTemplateStats(dir) {
-  const templates = await loadTemplatesFromDir(dir);
-  const bySeverity = {};
-  const byTag = {};
-
-  for (const t of templates) {
-    const sev = t.info.severity || "unknown";
-    bySeverity[sev] = (bySeverity[sev] || 0) + 1;
-    for (const tag of t.info.tags) {
-      byTag[tag] = (byTag[tag] || 0) + 1;
-    }
-  }
-
-  return {
-    total: templates.length,
-    bySeverity,
-    topTags: Object.entries(byTag)
-      .sort(([, a], [, b]) => b - a)
-      .slice(0, 20)
-      .map(([tag, count]) => ({ tag, count })),
-  };
-}

package/lib/owasp-wstg.js

@@ -34,6 +34,7 @@ export const PROBE_WSTG = [
   { id: "secret-scanning", probe: "Secret scanning", wstg: ["WSTG-CONF-02", "WSTG-CONF-05"], profiles: ["quick", "standard", "deep"], modes: ["repo"] },
   { id: "trust-boundary", probe: "Route / trust-boundary analysis", wstg: ["WSTG-ATHZ-02", "WSTG-CONF-02"], profiles: ["quick", "standard", "deep"], modes: ["repo"] },
   { id: "risky-patterns", probe: "Risky code patterns", wstg: ["WSTG-INPV-01", "WSTG-INPV-12"], profiles: ["standard", "deep"], modes: ["repo"] },
+  { id: "runtime-advisories", probe: "Runtime & server advisories (PHP, nginx, Apache, Node.js, Python, OpenSSL)", wstg: ["WSTG-INFO-02", "WSTG-INFO-08", "WSTG-CONF-01"], profiles: ["quick", "standard", "deep"], modes: ["url", "repo"] },
 ];
 
 /** Finding category/source → WSTG tags for individual findings. */
@@ -54,6 +55,7 @@ export const FINDING_WSTG = {
   fuzzing: ["WSTG-INPV-11"],
   recon: ["WSTG-INFO-01"],
   cve: ["WSTG-CONF-04"],
+  "runtime-advisory": ["WSTG-INFO-02", "WSTG-CONF-01"],
   "rate-limiting": ["WSTG-ATHN-10"],
   "code-pattern": ["WSTG-INPV-01", "WSTG-INPV-12"],
   config: ["WSTG-CONF-02"],
@@ -79,7 +81,9 @@ const WSTG_NAMES = {
   "WSTG-INFO-04": "Enumerate Applications on Webserver",
   "WSTG-INFO-06": "Identify Application Entry Points",
   "WSTG-INFO-07": "Map Execution Paths Through Application",
+  "WSTG-INFO-08": "Fingerprint Web Application Framework",
   "WSTG-INFO-09": "Fingerprint Web Application",
+  "WSTG-CONF-01": "Test Network Infrastructure Configuration",
   "WSTG-CONF-02": "Test Application Platform Configuration",
   "WSTG-CONF-04": "Review Old Backup and Unreferenced Files",
   "WSTG-CONF-05": "Enumerate Infrastructure and Application Admin Interfaces",

package/lib/plugins.js

@@ -7,7 +7,7 @@
  *   const findings = await runTemplateScan("https://app.example", templates);
  */
 import { existsSync, statSync } from "node:fs";
-import { extname, resolve } from "node:path";
+import { resolve } from "node:path";
 import {
   loadTemplatesFromDir,
   executeTemplates,

package/lib/provenance.js

@@ -27,6 +27,7 @@ const SOURCE_CONFIDENCE = {
   "email-dns-probe": "confirmed",
   "framework-check": "likely",
   "deps-scan": "likely",
+  "runtime-advisory": "likely",
   "static-analysis": "likely",
   "trust-boundary": "likely",
   "auth-probe": "likely",

package/lib/recon.js

@@ -12,10 +12,12 @@
  * Inspired by: subfinder, gau, waybackurls, ParamSpider
  */
 
-import { safeFetch } from "../src/utils/http.js";
+import { fetchJson } from "../src/utils/http.js";
 import { registrableDomain } from "../src/utils/url.js";
 
 const RECON_TIMEOUT = 15_000;
+// Certificate-transparency answers for large domains run to megabytes.
+const RECON_MAX_BODY = 20 * 1024 * 1024;
 
 /**
  * Third-party hosts this module queries. Under --offline the scan skips recon
@@ -33,18 +35,11 @@ export const RECON_HOSTS = ["crt.sh", "web.archive.org", "otx.alienvault.com"];
  * @returns {string[]} Discovered subdomains
  */
 export async function discoverSubdomains(domain) {
-  const res = await safeFetch(
+  const data = await fetchJson(
     `https://crt.sh/?q=%25.${encodeURIComponent(domain)}&output=json`,
-    { timeout: 20_000, probe: "recon", redirect: "follow" },
+    { timeout: 20_000, probe: "recon", redirect: "follow", maxBytes: RECON_MAX_BODY },
   );
-  if (!res || !res.ok) return [];
-
-  let data;
-  try {
-    data = await res.json();
-  } catch {
-    return [];
-  }
+  if (!data) return [];
 
   const subdomains = new Set();
   for (const entry of data) {
@@ -72,18 +67,11 @@ export async function discoverSubdomains(domain) {
  */
 export async function fetchWaybackUrls(domain, opts = {}) {
   const limit = opts.limit || 1000;
-  const res = await safeFetch(
+  const rows = await fetchJson(
     `https://web.archive.org/cdx/search/cdx?url=*.${encodeURIComponent(domain)}/*&output=json&fl=original&collapse=urlkey&limit=${limit}`,
-    { timeout: 20_000, probe: "recon", redirect: "follow" },
+    { timeout: 20_000, probe: "recon", redirect: "follow", maxBytes: RECON_MAX_BODY },
   );
-  if (!res || !res.ok) return [];
-
-  let rows;
-  try {
-    rows = await res.json();
-  } catch {
-    return [];
-  }
+  if (!rows) return [];
 
   // First row is the header ["original"]
   return rows.slice(1).map((r) => r[0]).filter(Boolean);
@@ -98,18 +86,11 @@ export async function fetchWaybackUrls(domain, opts = {}) {
  */
 export async function fetchOtxUrls(domain, opts = {}) {
   const limit = opts.limit || 200;
-  const res = await safeFetch(
+  const data = await fetchJson(
     `https://otx.alienvault.com/api/v1/indicators/domain/${encodeURIComponent(domain)}/url_list?limit=${limit}`,
-    { timeout: RECON_TIMEOUT, probe: "recon", redirect: "follow" },
+    { timeout: RECON_TIMEOUT, probe: "recon", redirect: "follow", maxBytes: RECON_MAX_BODY },
   );
-  if (!res || !res.ok) return [];
-
-  let data;
-  try {
-    data = await res.json();
-  } catch {
-    return [];
-  }
+  if (!data) return [];
 
   return (data.url_list || []).map((e) => e.url).filter(Boolean);
 }