prae-cli 0.5.2 → 0.6.0

package/README.md

@@ -40,6 +40,40 @@ formats plus entropy, placeholders dropped. Findings name the type, rule,
 line and confidence — structurally never the secret, so the output is safe to
 paste anywhere. With `--pack`, also prints the verdict that pack would reach.
 
+### `prae test <suite.json> [--pack <pack.json>] [--json]`
+
+Run security invariants against a pack and exit non-zero if any stops holding.
+A suite says what an agent attempts and what the pack must decide about it:
+
+```json
+{
+  "pack": "./acme.prae.json",
+  "scenarios": [
+    {
+      "name": "Reads outside the workspace are refused",
+      "given": "the agent is scoped to ./src and ./docs",
+      "when": { "tool": "read", "args": { "file_path": "/etc/shadow" } },
+      "then": { "decision": "deny", "rule": "ACME-001" }
+    }
+  ]
+}
+```
+
+`then` also takes `secretTypes`, asserting which credential classes the
+inspector found before the verdict — so a rule about secrets is tested on the
+detection as well as the decision. `when.repeat: 8` asserts the eighth call of
+that capability, which is how a session budget is written down and how the gate
+counts it.
+
+A failure prints the invariant in full — given, when, then, and what it got
+instead — because the useful thing about a red security test is the sentence
+describing what was supposed to hold. `--pack` overrides the suite's own, which
+is how one suite is run against a candidate pack before it is adopted.
+
+This evaluates the same pack through the same evaluator and the same content
+inspector the gate uses at runtime. It is not the harness: the adapter that
+turns a live tool call into facts is covered by `prae-gate`'s own suite.
+
 ### `prae mcp --root <dir> [--pack <pack.json>] [--ledger <file>]`
 
 Serve gated filesystem access over MCP stdio. Any MCP-capable IDE or agent

package/lib/bin.js

@@ -1,6 +1,6 @@
 #!/usr/bin/env node
-import "./evaluate-DMyL3qM_.js";
-import { t as main } from "./cli-DfXHsQO7.js";
+import "./evaluate-CkYySl7T.js";
+import { t as main } from "./cli-Cr0U-PFD.js";
 import "./src-CXU0_W1P.js";
 import "./views-DalPkBZZ.js";
 

package/lib/{cli-DfXHsQO7.js → cli-Cr0U-PFD.js}

@@ -1,9 +1,9 @@
-import { t as evaluate } from "./evaluate-DMyL3qM_.js";
+import { t as evaluate } from "./evaluate-CkYySl7T.js";
 import { m as seal, t as Ledger } from "./src-CXU0_W1P.js";
 import { a as renderPacks, c as renderShell, i as renderOverview, l as VERDICT_LABEL$1, n as renderApprovals, o as esc, r as renderCompliance, s as renderNav, t as navFor } from "./views-DalPkBZZ.js";
 import { existsSync, readFileSync } from "node:fs";
 import { mkdir, readFile, writeFile } from "node:fs/promises";
-import { extname, join, normalize, resolve, sep } from "node:path";
+import { dirname, extname, join, normalize, resolve, sep } from "node:path";
 import { homedir } from "node:os";
 import { composePatchFile, preflightOps, revisionOf, takeSnapshot } from "dsh-overlay-check";
 import { createServer } from "node:http";
@@ -853,6 +853,12 @@ const USAGE = `prae — operator policy for agent harnesses
       contents are returned, and every decision lands in the ledger.
       (no --pack mounts the bundled secret-catcher pack)
 
+  prae test <suite.json> [--pack <pack.json>] [--json]
+      Run security scenarios against a pack and report PASS/FAIL. Each
+      scenario says what an agent attempts and what the pack must decide;
+      the exit code is non-zero if any invariant no longer holds, so a
+      control pack can be defended in CI. --pack overrides the suite's own.
+
   prae verify <ledger.jsonl> [--key <public.pem>]
       Re-derive every hash in a decision log and report any break. With
       --key, also verify each row's signature against that pinned key.
@@ -970,7 +976,7 @@ async function scanFile(path, packPath, json) {
 * not come up inspecting nothing.
 */
 async function mcp(root, packPath, ledgerPath) {
-	const { serveMcp } = await import("./mcp-CEN48TpN.js");
+	const { serveMcp } = await import("./mcp-Bq0DNPPR.js");
 	const { resolve: resolve$1 } = await import("node:path");
 	const bundled = [join(import.meta.dirname, "secret-catcher.prae.json"), join(import.meta.dirname, "..", "..", "..", "examples", "secret-catcher.prae.json")].find((candidate) => existsSync(candidate));
 	const chosen = packPath ?? bundled;
@@ -1086,6 +1092,52 @@ async function verifyReceiptCmd(path) {
 	else console.error(line);
 	if (!result.ok) process.exit(1);
 }
+/**
+* `prae test`: assert a pack's invariants and set an exit code.
+*
+* The checking lives in `test-run.ts`. This does the three things a command
+* does — read the files, supply the inspector, and decide the exit code.
+*/
+async function test(suitePath, packOverride, asJson) {
+	let raw;
+	try {
+		raw = await readFile(suitePath, "utf8");
+	} catch {
+		return fail(`cannot read ${suitePath}`);
+	}
+	let parsed;
+	try {
+		parsed = JSON.parse(raw);
+	} catch (error) {
+		return fail(`${suitePath} is not valid JSON: ${error instanceof Error ? error.message : error}`);
+	}
+	const { parseSuite, renderSuite, runSuite } = await import("./test-run-04ubUcSb.js");
+	const suite = parseSuite(parsed);
+	if (!suite.ok) {
+		console.error(`prae: ${suitePath} is not a valid test suite`);
+		for (const message of suite.errors) console.error(`  ${message}`);
+		process.exit(1);
+	}
+	const chosen = packOverride ?? (suite.suite.pack === void 0 ? void 0 : resolve(dirname(suitePath), suite.suite.pack));
+	if (chosen === void 0) return fail("test needs a pack: set `pack` in the suite or pass --pack");
+	const pack = await loadPack(chosen);
+	const { capabilityForTool } = await import("capmark");
+	const { scan } = await import("./src-Vnr0fcpK.js");
+	const inspect = pack.rules.some((rule) => rule.when.secrets !== void 0) ? (path) => {
+		try {
+			return scan(readFileSync(path, "utf8"));
+		} catch {
+			return;
+		}
+	} : void 0;
+	const result = runSuite(pack, suite.suite, {
+		capabilityFor: (tool) => capabilityForTool(tool)?.id,
+		...inspect === void 0 ? {} : { inspect }
+	});
+	if (asJson) console.log(JSON.stringify(result, null, 2));
+	else for (const line of renderSuite(result)) console.log(line);
+	if (!result.ok) process.exit(1);
+}
 async function verify(path, keyPath) {
 	let ledger;
 	try {
@@ -1175,7 +1227,7 @@ async function report(path, out) {
 * that step is a proposal a human approves, and it starts from this list.
 */
 async function instructions(from, asJson) {
-	const { ingest } = await import("./src-C4fnUm9V.js");
+	const { ingest } = await import("./src-CraUX64d.js");
 	const result = ingest(from);
 	if (asJson) {
 		console.log(JSON.stringify(result, null, 2));
@@ -1211,7 +1263,7 @@ async function instructions(from, asJson) {
 * on purpose rather than something that happened to their policy.
 */
 async function propose(from, options) {
-	const [{ ingest }, { extract }] = await Promise.all([import("./src-C4fnUm9V.js"), import("./src-BfYsUDW9.js")]);
+	const [{ ingest }, { extract }] = await Promise.all([import("./src-CraUX64d.js"), import("./src-CQ0UmFUK.js")]);
 	const read = ingest(from);
 	if (read.files.length === 0) {
 		console.log(`No instruction files found at or above ${from}.`);
@@ -1271,8 +1323,8 @@ async function consoleOut(path, argv) {
 	const instructionsDir = flag(argv, "instructions");
 	if (instructionsDir !== void 0) {
 		const [{ ingest }, { extract }, { renderInstructions }] = await Promise.all([
-			import("./src-C4fnUm9V.js"),
-			import("./src-BfYsUDW9.js"),
+			import("./src-CraUX64d.js"),
+			import("./src-CQ0UmFUK.js"),
 			import("./instructions-D4VOn2lk.js")
 		]);
 		const read = ingest(instructionsDir);
@@ -1389,6 +1441,12 @@ async function main(argv) {
 			if (path === void 0 || seq === void 0 || key === void 0) fail("receipt needs <ledger.jsonl> <seq> --key <private.pem>");
 			return receipt(path, Number(seq), key, flag(rest, "out"));
 		}
+		case "test": {
+			const [suite] = rest;
+			if (suite === void 0) fail("test needs a suite file");
+			await test(suite, flag(argv, "pack"), argv.includes("--json"));
+			return;
+		}
 		case "verify": {
 			const path = rest[0];
 			if (path === void 0) fail("verify needs a ledger file");

package/lib/docs/secret-catcher.md

@@ -73,6 +73,22 @@ before the tool runs. The refusal is delivered to the agent as the tool result.
 — one-minute profile, the exact command, the expected refusal, and the bypasses
 to try with what each one does. Requires `prae-gate ≥ 0.5.1`.
 
+## Prove the pack still holds, in your own pipeline
+
+`prae test` runs a pack against written invariants and exits non-zero when one
+stops holding — so a pack edit that quietly stops refusing something fails a
+build instead of reaching an agent:
+
+```bash
+npx prae-cli test ./your-invariants.json --pack ./secret-catcher.prae.json
+```
+
+Each scenario names what an agent attempts and what the pack must decide about
+it, including which credential classes the inspector had to find first — so the
+detection is asserted alongside the verdict. The file format is in
+`packages/cli/README.md`, and `demo/workspace/papaya.tests.json` in this
+repository is a worked suite of fourteen. Needs `prae-cli >= 0.6.0`.
+
 ## What the ledger holds afterwards
 
 One row per intercepted read — verdict, rule, subject path, and for content

package/lib/{evaluate-DMyL3qM_.js → evaluate-CkYySl7T.js}

@@ -80,13 +80,16 @@ function segments(command) {
 	const found = [];
 	for (const tokens of scan(command)) {
 		let at = 0;
+		const wrappers = [];
 		while (at < tokens.length) {
 			const token = tokens[at];
 			if (ASSIGNMENT.test(token)) {
 				at += 1;
 				continue;
 			}
-			if (WRAPPERS.has(token)) {
+			const word = (token.split("/").pop() ?? token).toLowerCase();
+			if (WRAPPERS.has(word)) {
+				wrappers.push(word);
 				at += 1;
 				while (at < tokens.length) {
 					const option = tokens[at];
@@ -98,10 +101,19 @@ function segments(command) {
 			break;
 		}
 		const name = tokens[at];
-		if (name === void 0) continue;
+		if (name === void 0) {
+			const last = wrappers.at(-1);
+			if (last !== void 0) found.push({
+				name: last,
+				tokens: [],
+				wrappers
+			});
+			continue;
+		}
 		found.push({
 			name: (name.split("/").pop() ?? name).toLowerCase(),
-			tokens: tokens.slice(at + 1)
+			tokens: tokens.slice(at + 1),
+			wrappers
 		});
 	}
 	return found;
@@ -122,7 +134,7 @@ function commandMatches(match, command) {
 	const args = list(match.token);
 	const pipes = list(match.pipedTo);
 	if (names.length > 0) {
-		const hit = parts.filter((segment) => names.some((pattern) => matches$1(pattern, segment.name)));
+		const hit = parts.filter((segment) => names.some((pattern) => matches$1(pattern, segment.name) || segment.wrappers.some((wrapper) => matches$1(pattern, wrapper))));
 		if (hit.length === 0) return false;
 		if (args.length > 0 && !hit.some((segment) => args.every((wanted) => segment.tokens.some((token) => matches$1(wanted, token))))) return false;
 	} else if (args.length > 0) {

package/lib/index.js

@@ -1,5 +1,5 @@
-import "./evaluate-DMyL3qM_.js";
-import { a as planWrite, i as dshHome, n as applyWrite, o as praeRow, r as defaultProfile, s as profileAt, t as main } from "./cli-DfXHsQO7.js";
+import "./evaluate-CkYySl7T.js";
+import { a as planWrite, i as dshHome, n as applyWrite, o as praeRow, r as defaultProfile, s as profileAt, t as main } from "./cli-Cr0U-PFD.js";
 import "./src-CXU0_W1P.js";
 import "./views-DalPkBZZ.js";
 

package/lib/{mcp-CEN48TpN.js → mcp-Bq0DNPPR.js}

@@ -1,4 +1,4 @@
-import { t as evaluate } from "./evaluate-DMyL3qM_.js";
+import { t as evaluate } from "./evaluate-CkYySl7T.js";
 import { a as scan, n as MAX_SCAN_BYTES } from "./src-Br1tGPtS.js";
 import { readFile, readdir, stat } from "node:fs/promises";
 import { resolve, sep } from "node:path";

package/lib/playground.js

@@ -83,13 +83,16 @@
 		const found = [];
 		for (const tokens of scan(command)) {
 			let at = 0;
+			const wrappers = [];
 			while (at < tokens.length) {
 				const token = tokens[at];
 				if (ASSIGNMENT.test(token)) {
 					at += 1;
 					continue;
 				}
-				if (WRAPPERS.has(token)) {
+				const word = (token.split("/").pop() ?? token).toLowerCase();
+				if (WRAPPERS.has(word)) {
+					wrappers.push(word);
 					at += 1;
 					while (at < tokens.length) {
 						const option = tokens[at];
@@ -101,10 +104,19 @@
 				break;
 			}
 			const name = tokens[at];
-			if (name === void 0) continue;
+			if (name === void 0) {
+				const last = wrappers.at(-1);
+				if (last !== void 0) found.push({
+					name: last,
+					tokens: [],
+					wrappers
+				});
+				continue;
+			}
 			found.push({
 				name: (name.split("/").pop() ?? name).toLowerCase(),
-				tokens: tokens.slice(at + 1)
+				tokens: tokens.slice(at + 1),
+				wrappers
 			});
 		}
 		return found;
@@ -125,7 +137,7 @@
 		const args = list(match.token);
 		const pipes = list(match.pipedTo);
 		if (names.length > 0) {
-			const hit = parts.filter((segment) => names.some((pattern) => matches$1(pattern, segment.name)));
+			const hit = parts.filter((segment) => names.some((pattern) => matches$1(pattern, segment.name) || segment.wrappers.some((wrapper) => matches$1(pattern, wrapper))));
 			if (hit.length === 0) return false;
 			if (args.length > 0 && !hit.some((segment) => args.every((wanted) => segment.tokens.some((token) => matches$1(wanted, token))))) return false;
 		} else if (args.length > 0) {

package/lib/test-run-04ubUcSb.js

@@ -0,0 +1,137 @@
+import { t as evaluate } from "./evaluate-CkYySl7T.js";
+
+//#region src/test-run.ts
+/** The path argument a scenario's call carries, if any. */
+function pathOf(args) {
+	for (const key of [
+		"path",
+		"file_path",
+		"filePath",
+		"target"
+	]) {
+		const value = args[key];
+		if (typeof value === "string" && value !== "") return value;
+	}
+}
+/**
+* Reject a malformed suite by saying what is wrong with it.
+*
+* A test file that silently runs nothing is the worst possible outcome here —
+* it reports success — so every scenario must name a tool and an expected
+* decision, and anything else is an error rather than a default.
+*/
+function parseSuite(parsed) {
+	const errors = [];
+	if (typeof parsed !== "object" || parsed === null) return {
+		ok: false,
+		errors: ["expected a JSON object"]
+	};
+	const root = parsed;
+	const scenarios = root.scenarios;
+	if (!Array.isArray(scenarios) || scenarios.length === 0) return {
+		ok: false,
+		errors: ["`scenarios` must be a non-empty array"]
+	};
+	const kinds = new Set([
+		"allow",
+		"deny",
+		"ask",
+		"correct"
+	]);
+	scenarios.forEach((raw, index) => {
+		const at = `scenarios[${index}]`;
+		if (typeof raw !== "object" || raw === null) {
+			errors.push(`${at}: expected an object`);
+			return;
+		}
+		const s = raw;
+		if (typeof s.name !== "string" || s.name === "") errors.push(`${at}.name: required`);
+		const when = s.when;
+		if (typeof when !== "object" || when === null) errors.push(`${at}.when: required`);
+		else {
+			if (typeof when.tool !== "string" || when.tool === "") errors.push(`${at}.when.tool: required`);
+			if (when.repeat !== void 0 && (typeof when.repeat !== "number" || when.repeat < 1)) errors.push(`${at}.when.repeat: must be a positive number`);
+		}
+		const then = s.then;
+		if (typeof then !== "object" || then === null) errors.push(`${at}.then: required`);
+		else if (typeof then.decision !== "string" || !kinds.has(then.decision)) errors.push(`${at}.then.decision: one of allow, deny, ask, correct`);
+	});
+	if (errors.length > 0) return {
+		ok: false,
+		errors
+	};
+	return {
+		ok: true,
+		suite: {
+			...typeof root.pack === "string" ? { pack: root.pack } : {},
+			scenarios
+		}
+	};
+}
+function runSuite(pack, suite, options) {
+	const results = suite.scenarios.map((scenario) => {
+		const args = scenario.when.args ?? {};
+		const capability = options.capabilityFor(scenario.when.tool);
+		const target = pathOf(args);
+		const findings = target === void 0 || options.inspect === void 0 ? void 0 : options.inspect(target);
+		const decision = evaluate(pack, {
+			tool: scenario.when.tool,
+			...capability === void 0 ? {} : { capability },
+			args,
+			...findings !== void 0 && findings.length > 0 ? { secrets: findings } : {},
+			...capability === void 0 || scenario.when.repeat === void 0 ? {} : { counts: { [capability]: scenario.when.repeat } }
+		});
+		const secretTypes = [...new Set((findings ?? []).map((f) => f.type))];
+		const failures = [];
+		if (decision.kind !== scenario.then.decision) failures.push(`decision was ${decision.kind}, expected ${scenario.then.decision}`);
+		if (scenario.then.rule !== void 0 && decision.rule !== scenario.then.rule) failures.push(`rule was ${decision.rule ?? "none"}, expected ${scenario.then.rule}`);
+		if (scenario.then.secretTypes !== void 0) {
+			const missing = scenario.then.secretTypes.filter((type) => !secretTypes.includes(type));
+			if (missing.length > 0) failures.push(`no finding of type ${missing.join(", ")}`);
+		}
+		return {
+			name: scenario.name,
+			...scenario.given === void 0 ? {} : { given: scenario.given },
+			when: scenario.when,
+			expected: scenario.then,
+			actual: {
+				decision: decision.kind,
+				...decision.rule === void 0 ? {} : { rule: decision.rule },
+				secretTypes
+			},
+			pass: failures.length === 0,
+			failures
+		};
+	});
+	const failed = results.filter((r) => !r.pass).length;
+	return {
+		pack: `${pack.pack} ${pack.version}`,
+		passed: results.length - failed,
+		failed,
+		results,
+		ok: failed === 0
+	};
+}
+/** The report, as lines. Phrased for a reader; the command only prints it. */
+function renderSuite(result) {
+	const lines = [`prae test — ${result.pack}`, ""];
+	const width = Math.max(...result.results.map((r) => r.name.length), 20);
+	for (const r of result.results) {
+		const mark = r.pass ? "✓" : "✕";
+		const shown = r.pass ? `${r.actual.decision.toUpperCase()}${r.actual.rule === void 0 ? "" : `  ${r.actual.rule}`}` : r.failures.join("; ");
+		lines.push(`  ${mark} ${r.name.padEnd(width)}  ${shown}`);
+	}
+	for (const r of result.results.filter((one) => !one.pass)) {
+		lines.push("", `  ✕ ${r.name}`);
+		if (r.given !== void 0) lines.push(`      GIVEN  ${r.given}`);
+		lines.push(`      WHEN   ${r.when.tool}${r.when.repeat === void 0 ? "" : ` ×${r.when.repeat}`} ${JSON.stringify(r.when.args ?? {})}`);
+		lines.push(`      THEN   ${r.expected.decision}${r.expected.rule === void 0 ? "" : ` · ${r.expected.rule}`}`);
+		lines.push(`      GOT    ${r.failures.join("; ")}`);
+	}
+	const n = result.results.length;
+	lines.push("", `${n} scenario${n === 1 ? "" : "s"} · ${result.passed} passed · ${result.failed} failed`, result.ok ? "SECURITY TEST: PASS" : "SECURITY TEST: FAIL");
+	return lines;
+}
+
+//#endregion
+export { parseSuite, renderSuite, runSuite };

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "prae-cli",
-  "version": "0.5.2",
-  "description": "Operator policy for agent harnesses: validate packs, explain a decision, verify a decision log, and provision a governed DSH profile.",
+  "version": "0.6.0",
+  "description": "Operator policy for agent harnesses: validate packs, explain a decision, test a pack against security invariants, verify a decision log, and provision a governed DSH profile.",
   "keywords": [
     "dsh-plugin",
     "governance",
@@ -41,9 +41,9 @@
     "typescript": "^5.9.2",
     "vitest": "^3.2.4",
     "@prae/controls": "0.1.0",
-    "@prae/extract": "0.1.0",
     "@prae/inspect": "0.1.0",
     "@prae/instructions": "0.1.0",
+    "@prae/extract": "0.1.0",
     "@prae/kernel": "0.1.0",
     "@prae/ledger": "0.1.0",
     "@prae/tokens": "0.1.0"