prae-cli 0.5.2 → 0.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +42 -0
- package/lib/bin.js +4 -4
- package/lib/{cli-DfXHsQO7.js → cli-BUgU-LpN.js} +123 -24
- package/lib/docs/secret-catcher.md +16 -0
- package/lib/{evaluate-DMyL3qM_.js → evaluate-CkYySl7T.js} +16 -4
- package/lib/index.js +4 -4
- package/lib/{instructions-D4VOn2lk.js → instructions-oGCdVxHM.js} +1 -1
- package/lib/{mcp-CEN48TpN.js → mcp-Bq0DNPPR.js} +1 -1
- package/lib/playground.js +99 -5
- package/lib/{receipt-check-Dql0uqxw.js → receipt-check-DrDE3Wwi.js} +1 -1
- package/lib/secret-catcher.prae.json +18 -3
- package/lib/{src-Kubj3MTX.js → src-C9L6NBvu.js} +1 -1
- package/lib/test-run-04ubUcSb.js +137 -0
- package/lib/{views-DalPkBZZ.js → views-CAD9738P.js} +157 -10
- package/package.json +2 -2
- /package/lib/{src-BfYsUDW9.js → src-CQ0UmFUK.js} +0 -0
- /package/lib/{src-CXU0_W1P.js → src-Ciggx9ja.js} +0 -0
- /package/lib/{src-C4fnUm9V.js → src-CraUX64d.js} +0 -0
package/README.md
CHANGED
|
@@ -40,6 +40,40 @@ formats plus entropy, placeholders dropped. Findings name the type, rule,
|
|
|
40
40
|
line and confidence — structurally never the secret, so the output is safe to
|
|
41
41
|
paste anywhere. With `--pack`, also prints the verdict that pack would reach.
|
|
42
42
|
|
|
43
|
+
### `prae test <suite.json> [--pack <pack.json>] [--json]`
|
|
44
|
+
|
|
45
|
+
Run security invariants against a pack and exit non-zero if any stops holding.
|
|
46
|
+
A suite says what an agent attempts and what the pack must decide about it:
|
|
47
|
+
|
|
48
|
+
```json
|
|
49
|
+
{
|
|
50
|
+
"pack": "./acme.prae.json",
|
|
51
|
+
"scenarios": [
|
|
52
|
+
{
|
|
53
|
+
"name": "Reads outside the workspace are refused",
|
|
54
|
+
"given": "the agent is scoped to ./src and ./docs",
|
|
55
|
+
"when": { "tool": "read", "args": { "file_path": "/etc/shadow" } },
|
|
56
|
+
"then": { "decision": "deny", "rule": "ACME-001" }
|
|
57
|
+
}
|
|
58
|
+
]
|
|
59
|
+
}
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
`then` also takes `secretTypes`, asserting which credential classes the
|
|
63
|
+
inspector found before the verdict — so a rule about secrets is tested on the
|
|
64
|
+
detection as well as the decision. `when.repeat: 8` asserts the eighth call of
|
|
65
|
+
that capability, which is how a session budget is written down and how the gate
|
|
66
|
+
counts it.
|
|
67
|
+
|
|
68
|
+
A failure prints the invariant in full — given, when, then, and what it got
|
|
69
|
+
instead — because the useful thing about a red security test is the sentence
|
|
70
|
+
describing what was supposed to hold. `--pack` overrides the suite's own, which
|
|
71
|
+
is how one suite is run against a candidate pack before it is adopted.
|
|
72
|
+
|
|
73
|
+
This evaluates the same pack through the same evaluator and the same content
|
|
74
|
+
inspector the gate uses at runtime. It is not the harness: the adapter that
|
|
75
|
+
turns a live tool call into facts is covered by `prae-gate`'s own suite.
|
|
76
|
+
|
|
43
77
|
### `prae mcp --root <dir> [--pack <pack.json>] [--ledger <file>]`
|
|
44
78
|
|
|
45
79
|
Serve gated filesystem access over MCP stdio. Any MCP-capable IDE or agent
|
|
@@ -56,6 +90,14 @@ next to the result rather than in a footnote: a removed, edited, or reordered
|
|
|
56
90
|
row breaks the chain; a rewrite of the whole file by someone holding this code
|
|
57
91
|
does not.
|
|
58
92
|
|
|
93
|
+
### `prae report <ledger.jsonl> [--out <file.html>] [--pack <pack.json>]`
|
|
94
|
+
|
|
95
|
+
Render the decision record as a printable evidence report. With `--pack`,
|
|
96
|
+
each clause also names the framework risk the rule cites — OWASP Top 10 for
|
|
97
|
+
LLM Applications 2026 today — so a reviewer reads risk → control → decision →
|
|
98
|
+
seal in one row. Without the pack the clause id stands alone rather than a
|
|
99
|
+
name guessed from it.
|
|
100
|
+
|
|
59
101
|
### `prae console <ledger.jsonl> --out <dir> [--pack <file>]`
|
|
60
102
|
|
|
61
103
|
Write the control plane as linked pages: Overview, Policy packs, Playground,
|
package/lib/bin.js
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
import "./evaluate-
|
|
3
|
-
import
|
|
4
|
-
import "./
|
|
5
|
-
import "./
|
|
2
|
+
import "./evaluate-CkYySl7T.js";
|
|
3
|
+
import "./views-CAD9738P.js";
|
|
4
|
+
import { t as main } from "./cli-BUgU-LpN.js";
|
|
5
|
+
import "./src-Ciggx9ja.js";
|
|
6
6
|
|
|
7
7
|
//#region src/bin.ts
|
|
8
8
|
main(process.argv.slice(2)).catch((error) => {
|
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
import { t as evaluate } from "./evaluate-
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
1
|
+
import { t as evaluate } from "./evaluate-CkYySl7T.js";
|
|
2
|
+
import { a as renderPacks, c as renderShell, d as cite, i as renderOverview, l as VERDICT_LABEL$1, n as renderApprovals, o as esc, r as renderCompliance, s as renderNav, t as navFor, u as FRAMEWORKS } from "./views-CAD9738P.js";
|
|
3
|
+
import { m as seal, t as Ledger } from "./src-Ciggx9ja.js";
|
|
4
4
|
import { existsSync, readFileSync } from "node:fs";
|
|
5
5
|
import { mkdir, readFile, writeFile } from "node:fs/promises";
|
|
6
|
-
import { extname, join, normalize, resolve, sep } from "node:path";
|
|
6
|
+
import { dirname, extname, join, normalize, resolve, sep } from "node:path";
|
|
7
7
|
import { homedir } from "node:os";
|
|
8
8
|
import { composePatchFile, preflightOps, revisionOf, takeSnapshot } from "dsh-overlay-check";
|
|
9
9
|
import { createServer } from "node:http";
|
|
@@ -302,7 +302,8 @@ function readRule(check$1, path, value) {
|
|
|
302
302
|
"id",
|
|
303
303
|
"title",
|
|
304
304
|
"when",
|
|
305
|
-
"effect"
|
|
305
|
+
"effect",
|
|
306
|
+
"frameworks"
|
|
306
307
|
]);
|
|
307
308
|
const id = check$1.text(`${path}.id`, value.id);
|
|
308
309
|
const effect = readEffect(check$1, `${path}.effect`, value.effect);
|
|
@@ -318,8 +319,40 @@ function readRule(check$1, path, value) {
|
|
|
318
319
|
const title = check$1.text(`${path}.title`, value.title);
|
|
319
320
|
if (title !== void 0) rule.title = title;
|
|
320
321
|
}
|
|
322
|
+
if (value.frameworks !== void 0) {
|
|
323
|
+
const frameworks = readFrameworks(check$1, `${path}.frameworks`, value.frameworks);
|
|
324
|
+
if (frameworks !== void 0) rule.frameworks = frameworks;
|
|
325
|
+
}
|
|
321
326
|
return rule;
|
|
322
327
|
}
|
|
328
|
+
/**
|
|
329
|
+
* A citation names a framework edition and controls the registry knows. A
|
|
330
|
+
* typo here would be a rule that claims a mapping no view can render, so it
|
|
331
|
+
* is a pack error rather than an ignored key — the same line the rest of the
|
|
332
|
+
* parser takes.
|
|
333
|
+
*/
|
|
334
|
+
function readFrameworks(check$1, path, value) {
|
|
335
|
+
if (!isObject(value)) {
|
|
336
|
+
check$1.fail(path, "expected an object keyed by framework id");
|
|
337
|
+
return;
|
|
338
|
+
}
|
|
339
|
+
const out = {};
|
|
340
|
+
for (const [framework, controls] of Object.entries(value)) {
|
|
341
|
+
const known = FRAMEWORKS[framework];
|
|
342
|
+
if (known === void 0) {
|
|
343
|
+
check$1.fail(`${path}.${framework}`, `unknown framework; expected one of ${Object.keys(FRAMEWORKS).map((k) => `\`${k}\``).join(", ")}`);
|
|
344
|
+
continue;
|
|
345
|
+
}
|
|
346
|
+
if (!Array.isArray(controls) || !controls.every((c) => typeof c === "string")) {
|
|
347
|
+
check$1.fail(`${path}.${framework}`, "expected an array of control ids");
|
|
348
|
+
continue;
|
|
349
|
+
}
|
|
350
|
+
const ids = controls;
|
|
351
|
+
for (const control of ids) if (known.controls[control] === void 0) check$1.fail(`${path}.${framework}`, `unknown control \`${control}\` in ${known.name}`);
|
|
352
|
+
if (ids.length > 0) out[framework] = [...new Set(ids)];
|
|
353
|
+
}
|
|
354
|
+
return out;
|
|
355
|
+
}
|
|
323
356
|
/** Parse and validate a pack. Never throws; every problem lands in `errors`. */
|
|
324
357
|
function parsePack(source) {
|
|
325
358
|
let raw;
|
|
@@ -404,14 +437,16 @@ function ago(iso, now) {
|
|
|
404
437
|
const s = seconds % 60;
|
|
405
438
|
return `${String(h).padStart(2, "0")}:${String(m).padStart(2, "0")}:${String(s).padStart(2, "0")}`;
|
|
406
439
|
}
|
|
407
|
-
function row(entry) {
|
|
440
|
+
function row(entry, pack) {
|
|
408
441
|
const call = entry.subject === void 0 ? entry.tool : `${entry.tool} ${entry.subject}`;
|
|
409
442
|
const actor = entry.actor ?? entry.agent ?? "—";
|
|
443
|
+
const rule = pack?.rules.find((r) => r.id === entry.rule);
|
|
444
|
+
const cited = rule === void 0 ? "" : cite(rule).map((c) => `<br><span style="color:var(--secondary)">${esc(c.control)} ${esc(c.controlName)}</span>`).join("");
|
|
410
445
|
return ` <div class="grid row">
|
|
411
446
|
<div class="seq">${entry.seq}</div>
|
|
412
447
|
<div class="call">${esc(call)}</div>
|
|
413
448
|
<div class="actor">${esc(actor)}</div>
|
|
414
|
-
<div class="clause">${esc(entry.rule ?? "—")}</div>
|
|
449
|
+
<div class="clause">${esc(entry.rule ?? "—")}${cited}</div>
|
|
415
450
|
<div class="verdict v-${esc(entry.verdict)}">${esc(VERDICT_LABEL$1[entry.verdict] ?? entry.verdict)}</div>
|
|
416
451
|
<div class="seal">${esc(seal(entry))}</div>
|
|
417
452
|
</div>`;
|
|
@@ -452,7 +487,7 @@ function renderLedgerView(entries, verification, options) {
|
|
|
452
487
|
const table = ` <div class="grid thead">
|
|
453
488
|
<div>SEQ</div><div>CALL</div><div>ACTOR</div><div>CLAUSE</div><div>VERDICT</div><div>SEAL</div>
|
|
454
489
|
</div>
|
|
455
|
-
${entries.length === 0 ? " <div class=\"empty\">No decisions recorded yet.</div>" : [...entries].reverse().map(row).join("\n")}`;
|
|
490
|
+
${entries.length === 0 ? " <div class=\"empty\">No decisions recorded yet.</div>" : [...entries].reverse().map((entry) => row(entry, options.pack)).join("\n")}`;
|
|
456
491
|
const first = held.at(-1);
|
|
457
492
|
const awaiting = first === void 0 ? "" : ` <section class="await">
|
|
458
493
|
<div class="k">AWAITING HUMAN · 1 OF ${held.length}</div>
|
|
@@ -853,6 +888,12 @@ const USAGE = `prae — operator policy for agent harnesses
|
|
|
853
888
|
contents are returned, and every decision lands in the ledger.
|
|
854
889
|
(no --pack mounts the bundled secret-catcher pack)
|
|
855
890
|
|
|
891
|
+
prae test <suite.json> [--pack <pack.json>] [--json]
|
|
892
|
+
Run security scenarios against a pack and report PASS/FAIL. Each
|
|
893
|
+
scenario says what an agent attempts and what the pack must decide;
|
|
894
|
+
the exit code is non-zero if any invariant no longer holds, so a
|
|
895
|
+
control pack can be defended in CI. --pack overrides the suite's own.
|
|
896
|
+
|
|
856
897
|
prae verify <ledger.jsonl> [--key <public.pem>]
|
|
857
898
|
Re-derive every hash in a decision log and report any break. With
|
|
858
899
|
--key, also verify each row's signature against that pinned key.
|
|
@@ -868,8 +909,11 @@ const USAGE = `prae — operator policy for agent harnesses
|
|
|
868
909
|
Check one receipt with nothing but the file: recompute its hash, verify
|
|
869
910
|
its signature, and print the key fingerprint to compare against your own.
|
|
870
911
|
|
|
871
|
-
prae report <ledger.jsonl> [--out <file.html>]
|
|
872
|
-
Render the decision record as a printable evidence report.
|
|
912
|
+
prae report <ledger.jsonl> [--out <file.html>] [--pack <pack.json>]
|
|
913
|
+
Render the decision record as a printable evidence report. With
|
|
914
|
+
--pack, each clause also names the framework risk it addresses
|
|
915
|
+
(OWASP LLM Top 10 2026), so a reviewer can trace risk → control →
|
|
916
|
+
decision → seal.
|
|
873
917
|
|
|
874
918
|
prae console <ledger.jsonl> --out <dir> [--pack <file>] [--serve [port]]
|
|
875
919
|
[--instructions <dir>]
|
|
@@ -919,6 +963,7 @@ async function check(path) {
|
|
|
919
963
|
for (const rule of pack.rules) {
|
|
920
964
|
const label = rule.title ?? rule.effect.because;
|
|
921
965
|
console.log(` ${rule.id.padEnd(10)} ${rule.effect.kind.toUpperCase().padEnd(8)} ${label}`);
|
|
966
|
+
for (const c of cite(rule)) console.log(` ${"".padEnd(10)} ${"".padEnd(8)} ↳ ${c.frameworkShort} · ${c.control} ${c.controlName}`);
|
|
922
967
|
}
|
|
923
968
|
}
|
|
924
969
|
/**
|
|
@@ -970,13 +1015,13 @@ async function scanFile(path, packPath, json) {
|
|
|
970
1015
|
* not come up inspecting nothing.
|
|
971
1016
|
*/
|
|
972
1017
|
async function mcp(root, packPath, ledgerPath) {
|
|
973
|
-
const { serveMcp } = await import("./mcp-
|
|
1018
|
+
const { serveMcp } = await import("./mcp-Bq0DNPPR.js");
|
|
974
1019
|
const { resolve: resolve$1 } = await import("node:path");
|
|
975
1020
|
const bundled = [join(import.meta.dirname, "secret-catcher.prae.json"), join(import.meta.dirname, "..", "..", "..", "examples", "secret-catcher.prae.json")].find((candidate) => existsSync(candidate));
|
|
976
1021
|
const chosen = packPath ?? bundled;
|
|
977
1022
|
if (chosen === void 0) return fail("mcp needs --pack <file>; no bundled secret-catcher found");
|
|
978
1023
|
const pack = await loadPack(chosen);
|
|
979
|
-
const { Ledger: Ledger$1 } = await import("./src-
|
|
1024
|
+
const { Ledger: Ledger$1 } = await import("./src-C9L6NBvu.js");
|
|
980
1025
|
const ledger = ledgerPath === void 0 ? void 0 : Ledger$1.open(ledgerPath);
|
|
981
1026
|
console.error(`prae mcp: serving ${resolve$1(root)} under ${pack.pack} ${pack.version}` + (ledgerPath === void 0 ? " (no ledger)" : ` · ledger ${ledgerPath}`));
|
|
982
1027
|
await serveMcp({
|
|
@@ -1028,7 +1073,7 @@ async function explain(path, tool, pairs) {
|
|
|
1028
1073
|
* a key that arrived beside it is checked against nothing.
|
|
1029
1074
|
*/
|
|
1030
1075
|
async function keygen(dir) {
|
|
1031
|
-
const { generateKeypair, fingerprint } = await import("./src-
|
|
1076
|
+
const { generateKeypair, fingerprint } = await import("./src-C9L6NBvu.js");
|
|
1032
1077
|
const { writeFileSync: writeFileSync$1 } = await import("node:fs");
|
|
1033
1078
|
const keys = generateKeypair();
|
|
1034
1079
|
const priv = join(dir, "prae-signing.pem");
|
|
@@ -1044,7 +1089,7 @@ async function keygen(dir) {
|
|
|
1044
1089
|
console.log("private key with `signingKey` to sign every row.");
|
|
1045
1090
|
}
|
|
1046
1091
|
async function receipt(path, seq, keyPath, out) {
|
|
1047
|
-
const { Ledger: Ledger$1, makeReceipt, fingerprint } = await import("./src-
|
|
1092
|
+
const { Ledger: Ledger$1, makeReceipt, fingerprint } = await import("./src-C9L6NBvu.js");
|
|
1048
1093
|
const { readFileSync: readFileSync$1 } = await import("node:fs");
|
|
1049
1094
|
const { createPublicKey } = await import("node:crypto");
|
|
1050
1095
|
const privateKey = readFileSync$1(keyPath, "utf8");
|
|
@@ -1074,7 +1119,7 @@ async function receipt(path, seq, keyPath, out) {
|
|
|
1074
1119
|
* file and sets the exit code, so scripts can branch on it.
|
|
1075
1120
|
*/
|
|
1076
1121
|
async function verifyReceiptCmd(path) {
|
|
1077
|
-
const { checkReceipt } = await import("./receipt-check-
|
|
1122
|
+
const { checkReceipt } = await import("./receipt-check-DrDE3Wwi.js");
|
|
1078
1123
|
let parsed;
|
|
1079
1124
|
try {
|
|
1080
1125
|
parsed = JSON.parse(await readFile(path, "utf8"));
|
|
@@ -1086,6 +1131,52 @@ async function verifyReceiptCmd(path) {
|
|
|
1086
1131
|
else console.error(line);
|
|
1087
1132
|
if (!result.ok) process.exit(1);
|
|
1088
1133
|
}
|
|
1134
|
+
/**
|
|
1135
|
+
* `prae test`: assert a pack's invariants and set an exit code.
|
|
1136
|
+
*
|
|
1137
|
+
* The checking lives in `test-run.ts`. This does the three things a command
|
|
1138
|
+
* does — read the files, supply the inspector, and decide the exit code.
|
|
1139
|
+
*/
|
|
1140
|
+
async function test(suitePath, packOverride, asJson) {
|
|
1141
|
+
let raw;
|
|
1142
|
+
try {
|
|
1143
|
+
raw = await readFile(suitePath, "utf8");
|
|
1144
|
+
} catch {
|
|
1145
|
+
return fail(`cannot read ${suitePath}`);
|
|
1146
|
+
}
|
|
1147
|
+
let parsed;
|
|
1148
|
+
try {
|
|
1149
|
+
parsed = JSON.parse(raw);
|
|
1150
|
+
} catch (error) {
|
|
1151
|
+
return fail(`${suitePath} is not valid JSON: ${error instanceof Error ? error.message : error}`);
|
|
1152
|
+
}
|
|
1153
|
+
const { parseSuite, renderSuite, runSuite } = await import("./test-run-04ubUcSb.js");
|
|
1154
|
+
const suite = parseSuite(parsed);
|
|
1155
|
+
if (!suite.ok) {
|
|
1156
|
+
console.error(`prae: ${suitePath} is not a valid test suite`);
|
|
1157
|
+
for (const message of suite.errors) console.error(` ${message}`);
|
|
1158
|
+
process.exit(1);
|
|
1159
|
+
}
|
|
1160
|
+
const chosen = packOverride ?? (suite.suite.pack === void 0 ? void 0 : resolve(dirname(suitePath), suite.suite.pack));
|
|
1161
|
+
if (chosen === void 0) return fail("test needs a pack: set `pack` in the suite or pass --pack");
|
|
1162
|
+
const pack = await loadPack(chosen);
|
|
1163
|
+
const { capabilityForTool } = await import("capmark");
|
|
1164
|
+
const { scan } = await import("./src-Vnr0fcpK.js");
|
|
1165
|
+
const inspect = pack.rules.some((rule) => rule.when.secrets !== void 0) ? (path) => {
|
|
1166
|
+
try {
|
|
1167
|
+
return scan(readFileSync(path, "utf8"));
|
|
1168
|
+
} catch {
|
|
1169
|
+
return;
|
|
1170
|
+
}
|
|
1171
|
+
} : void 0;
|
|
1172
|
+
const result = runSuite(pack, suite.suite, {
|
|
1173
|
+
capabilityFor: (tool) => capabilityForTool(tool)?.id,
|
|
1174
|
+
...inspect === void 0 ? {} : { inspect }
|
|
1175
|
+
});
|
|
1176
|
+
if (asJson) console.log(JSON.stringify(result, null, 2));
|
|
1177
|
+
else for (const line of renderSuite(result)) console.log(line);
|
|
1178
|
+
if (!result.ok) process.exit(1);
|
|
1179
|
+
}
|
|
1089
1180
|
async function verify(path, keyPath) {
|
|
1090
1181
|
let ledger;
|
|
1091
1182
|
try {
|
|
@@ -1099,7 +1190,7 @@ async function verify(path, keyPath) {
|
|
|
1099
1190
|
const last = entries.at(-1);
|
|
1100
1191
|
console.log(`${result.entries} entries · chain intact${last === void 0 ? "" : ` · seal ${seal(last)}`}`);
|
|
1101
1192
|
if (keyPath !== void 0) {
|
|
1102
|
-
const { verifySignature, fingerprint } = await import("./src-
|
|
1193
|
+
const { verifySignature, fingerprint } = await import("./src-C9L6NBvu.js");
|
|
1103
1194
|
const { readFileSync: readFileSync$1 } = await import("node:fs");
|
|
1104
1195
|
const publicKey = readFileSync$1(keyPath, "utf8");
|
|
1105
1196
|
let signed = 0;
|
|
@@ -1138,18 +1229,20 @@ function flag(argv, name) {
|
|
|
1138
1229
|
* has to go and check separately. A report that looks authoritative while
|
|
1139
1230
|
* quietly sitting on a broken chain is worse than no report.
|
|
1140
1231
|
*/
|
|
1141
|
-
async function report(path, out) {
|
|
1232
|
+
async function report(path, out, packPath) {
|
|
1142
1233
|
let ledger;
|
|
1143
1234
|
try {
|
|
1144
1235
|
ledger = Ledger.open(path);
|
|
1145
1236
|
} catch (error) {
|
|
1146
1237
|
return fail(error instanceof Error ? error.message : String(error));
|
|
1147
1238
|
}
|
|
1239
|
+
const pack = packPath === void 0 ? void 0 : await loadPack(packPath);
|
|
1148
1240
|
const entries = ledger.read();
|
|
1149
1241
|
const verification = ledger.verify();
|
|
1150
1242
|
const html = renderLedgerView(entries, verification, {
|
|
1151
1243
|
ledgerPath: path,
|
|
1152
|
-
generatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
1244
|
+
generatedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
1245
|
+
...pack === void 0 ? {} : { pack }
|
|
1153
1246
|
});
|
|
1154
1247
|
if (out === void 0) {
|
|
1155
1248
|
process.stdout.write(html);
|
|
@@ -1175,7 +1268,7 @@ async function report(path, out) {
|
|
|
1175
1268
|
* that step is a proposal a human approves, and it starts from this list.
|
|
1176
1269
|
*/
|
|
1177
1270
|
async function instructions(from, asJson) {
|
|
1178
|
-
const { ingest } = await import("./src-
|
|
1271
|
+
const { ingest } = await import("./src-CraUX64d.js");
|
|
1179
1272
|
const result = ingest(from);
|
|
1180
1273
|
if (asJson) {
|
|
1181
1274
|
console.log(JSON.stringify(result, null, 2));
|
|
@@ -1211,7 +1304,7 @@ async function instructions(from, asJson) {
|
|
|
1211
1304
|
* on purpose rather than something that happened to their policy.
|
|
1212
1305
|
*/
|
|
1213
1306
|
async function propose(from, options) {
|
|
1214
|
-
const [{ ingest }, { extract }] = await Promise.all([import("./src-
|
|
1307
|
+
const [{ ingest }, { extract }] = await Promise.all([import("./src-CraUX64d.js"), import("./src-CQ0UmFUK.js")]);
|
|
1215
1308
|
const read = ingest(from);
|
|
1216
1309
|
if (read.files.length === 0) {
|
|
1217
1310
|
console.log(`No instruction files found at or above ${from}.`);
|
|
@@ -1271,9 +1364,9 @@ async function consoleOut(path, argv) {
|
|
|
1271
1364
|
const instructionsDir = flag(argv, "instructions");
|
|
1272
1365
|
if (instructionsDir !== void 0) {
|
|
1273
1366
|
const [{ ingest }, { extract }, { renderInstructions }] = await Promise.all([
|
|
1274
|
-
import("./src-
|
|
1275
|
-
import("./src-
|
|
1276
|
-
import("./instructions-
|
|
1367
|
+
import("./src-CraUX64d.js"),
|
|
1368
|
+
import("./src-CQ0UmFUK.js"),
|
|
1369
|
+
import("./instructions-oGCdVxHM.js")
|
|
1277
1370
|
]);
|
|
1278
1371
|
const read = ingest(instructionsDir);
|
|
1279
1372
|
instructionsView = renderInstructions(data, read, extract(read.statements));
|
|
@@ -1389,6 +1482,12 @@ async function main(argv) {
|
|
|
1389
1482
|
if (path === void 0 || seq === void 0 || key === void 0) fail("receipt needs <ledger.jsonl> <seq> --key <private.pem>");
|
|
1390
1483
|
return receipt(path, Number(seq), key, flag(rest, "out"));
|
|
1391
1484
|
}
|
|
1485
|
+
case "test": {
|
|
1486
|
+
const [suite] = rest;
|
|
1487
|
+
if (suite === void 0) fail("test needs a suite file");
|
|
1488
|
+
await test(suite, flag(argv, "pack"), argv.includes("--json"));
|
|
1489
|
+
return;
|
|
1490
|
+
}
|
|
1392
1491
|
case "verify": {
|
|
1393
1492
|
const path = rest[0];
|
|
1394
1493
|
if (path === void 0) fail("verify needs a ledger file");
|
|
@@ -1402,7 +1501,7 @@ async function main(argv) {
|
|
|
1402
1501
|
case "report": {
|
|
1403
1502
|
const path = rest[0];
|
|
1404
1503
|
if (path === void 0) fail("report needs a ledger file");
|
|
1405
|
-
return report(path, flag(rest, "out"));
|
|
1504
|
+
return report(path, flag(rest, "out"), flag(rest, "pack"));
|
|
1406
1505
|
}
|
|
1407
1506
|
case "console": {
|
|
1408
1507
|
const path = rest[0];
|
|
@@ -73,6 +73,22 @@ before the tool runs. The refusal is delivered to the agent as the tool result.
|
|
|
73
73
|
— one-minute profile, the exact command, the expected refusal, and the bypasses
|
|
74
74
|
to try with what each one does. Requires `prae-gate ≥ 0.5.1`.
|
|
75
75
|
|
|
76
|
+
## Prove the pack still holds, in your own pipeline
|
|
77
|
+
|
|
78
|
+
`prae test` runs a pack against written invariants and exits non-zero when one
|
|
79
|
+
stops holding — so a pack edit that quietly stops refusing something fails a
|
|
80
|
+
build instead of reaching an agent:
|
|
81
|
+
|
|
82
|
+
```bash
|
|
83
|
+
npx prae-cli test ./your-invariants.json --pack ./secret-catcher.prae.json
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
Each scenario names what an agent attempts and what the pack must decide about
|
|
87
|
+
it, including which credential classes the inspector had to find first — so the
|
|
88
|
+
detection is asserted alongside the verdict. The file format is in
|
|
89
|
+
`packages/cli/README.md`, and `demo/workspace/papaya.tests.json` in this
|
|
90
|
+
repository is a worked suite of fourteen. Needs `prae-cli >= 0.6.0`.
|
|
91
|
+
|
|
76
92
|
## What the ledger holds afterwards
|
|
77
93
|
|
|
78
94
|
One row per intercepted read — verdict, rule, subject path, and for content
|
|
@@ -80,13 +80,16 @@ function segments(command) {
|
|
|
80
80
|
const found = [];
|
|
81
81
|
for (const tokens of scan(command)) {
|
|
82
82
|
let at = 0;
|
|
83
|
+
const wrappers = [];
|
|
83
84
|
while (at < tokens.length) {
|
|
84
85
|
const token = tokens[at];
|
|
85
86
|
if (ASSIGNMENT.test(token)) {
|
|
86
87
|
at += 1;
|
|
87
88
|
continue;
|
|
88
89
|
}
|
|
89
|
-
|
|
90
|
+
const word = (token.split("/").pop() ?? token).toLowerCase();
|
|
91
|
+
if (WRAPPERS.has(word)) {
|
|
92
|
+
wrappers.push(word);
|
|
90
93
|
at += 1;
|
|
91
94
|
while (at < tokens.length) {
|
|
92
95
|
const option = tokens[at];
|
|
@@ -98,10 +101,19 @@ function segments(command) {
|
|
|
98
101
|
break;
|
|
99
102
|
}
|
|
100
103
|
const name = tokens[at];
|
|
101
|
-
if (name === void 0)
|
|
104
|
+
if (name === void 0) {
|
|
105
|
+
const last = wrappers.at(-1);
|
|
106
|
+
if (last !== void 0) found.push({
|
|
107
|
+
name: last,
|
|
108
|
+
tokens: [],
|
|
109
|
+
wrappers
|
|
110
|
+
});
|
|
111
|
+
continue;
|
|
112
|
+
}
|
|
102
113
|
found.push({
|
|
103
114
|
name: (name.split("/").pop() ?? name).toLowerCase(),
|
|
104
|
-
tokens: tokens.slice(at + 1)
|
|
115
|
+
tokens: tokens.slice(at + 1),
|
|
116
|
+
wrappers
|
|
105
117
|
});
|
|
106
118
|
}
|
|
107
119
|
return found;
|
|
@@ -122,7 +134,7 @@ function commandMatches(match, command) {
|
|
|
122
134
|
const args = list(match.token);
|
|
123
135
|
const pipes = list(match.pipedTo);
|
|
124
136
|
if (names.length > 0) {
|
|
125
|
-
const hit = parts.filter((segment) => names.some((pattern) => matches$1(pattern, segment.name)));
|
|
137
|
+
const hit = parts.filter((segment) => names.some((pattern) => matches$1(pattern, segment.name) || segment.wrappers.some((wrapper) => matches$1(pattern, wrapper))));
|
|
126
138
|
if (hit.length === 0) return false;
|
|
127
139
|
if (args.length > 0 && !hit.some((segment) => args.every((wanted) => segment.tokens.some((token) => matches$1(wanted, token))))) return false;
|
|
128
140
|
} else if (args.length > 0) {
|
package/lib/index.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import "./evaluate-
|
|
2
|
-
import
|
|
3
|
-
import "./
|
|
4
|
-
import "./
|
|
1
|
+
import "./evaluate-CkYySl7T.js";
|
|
2
|
+
import "./views-CAD9738P.js";
|
|
3
|
+
import { a as planWrite, i as dshHome, n as applyWrite, o as praeRow, r as defaultProfile, s as profileAt, t as main } from "./cli-BUgU-LpN.js";
|
|
4
|
+
import "./src-Ciggx9ja.js";
|
|
5
5
|
|
|
6
6
|
export { applyWrite, defaultProfile, dshHome, main, planWrite, praeRow, profileAt };
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { t as evaluate } from "./evaluate-
|
|
1
|
+
import { t as evaluate } from "./evaluate-CkYySl7T.js";
|
|
2
2
|
import { a as scan, n as MAX_SCAN_BYTES } from "./src-Br1tGPtS.js";
|
|
3
3
|
import { readFile, readdir, stat } from "node:fs/promises";
|
|
4
4
|
import { resolve, sep } from "node:path";
|
package/lib/playground.js
CHANGED
|
@@ -83,13 +83,16 @@
|
|
|
83
83
|
const found = [];
|
|
84
84
|
for (const tokens of scan(command)) {
|
|
85
85
|
let at = 0;
|
|
86
|
+
const wrappers = [];
|
|
86
87
|
while (at < tokens.length) {
|
|
87
88
|
const token = tokens[at];
|
|
88
89
|
if (ASSIGNMENT.test(token)) {
|
|
89
90
|
at += 1;
|
|
90
91
|
continue;
|
|
91
92
|
}
|
|
92
|
-
|
|
93
|
+
const word = (token.split("/").pop() ?? token).toLowerCase();
|
|
94
|
+
if (WRAPPERS.has(word)) {
|
|
95
|
+
wrappers.push(word);
|
|
93
96
|
at += 1;
|
|
94
97
|
while (at < tokens.length) {
|
|
95
98
|
const option = tokens[at];
|
|
@@ -101,10 +104,19 @@
|
|
|
101
104
|
break;
|
|
102
105
|
}
|
|
103
106
|
const name = tokens[at];
|
|
104
|
-
if (name === void 0)
|
|
107
|
+
if (name === void 0) {
|
|
108
|
+
const last = wrappers.at(-1);
|
|
109
|
+
if (last !== void 0) found.push({
|
|
110
|
+
name: last,
|
|
111
|
+
tokens: [],
|
|
112
|
+
wrappers
|
|
113
|
+
});
|
|
114
|
+
continue;
|
|
115
|
+
}
|
|
105
116
|
found.push({
|
|
106
117
|
name: (name.split("/").pop() ?? name).toLowerCase(),
|
|
107
|
-
tokens: tokens.slice(at + 1)
|
|
118
|
+
tokens: tokens.slice(at + 1),
|
|
119
|
+
wrappers
|
|
108
120
|
});
|
|
109
121
|
}
|
|
110
122
|
return found;
|
|
@@ -125,7 +137,7 @@
|
|
|
125
137
|
const args = list(match.token);
|
|
126
138
|
const pipes = list(match.pipedTo);
|
|
127
139
|
if (names.length > 0) {
|
|
128
|
-
const hit = parts.filter((segment) => names.some((pattern) => matches$1(pattern, segment.name)));
|
|
140
|
+
const hit = parts.filter((segment) => names.some((pattern) => matches$1(pattern, segment.name) || segment.wrappers.some((wrapper) => matches$1(pattern, wrapper))));
|
|
129
141
|
if (hit.length === 0) return false;
|
|
130
142
|
if (args.length > 0 && !hit.some((segment) => args.every((wanted) => segment.tokens.some((token) => matches$1(wanted, token))))) return false;
|
|
131
143
|
} else if (args.length > 0) {
|
|
@@ -454,6 +466,55 @@
|
|
|
454
466
|
};
|
|
455
467
|
}
|
|
456
468
|
|
|
469
|
+
//#endregion
|
|
470
|
+
//#region ../kernel/src/frameworks.ts
|
|
471
|
+
const FRAMEWORKS = { "owasp-llm-top-10-2026": {
|
|
472
|
+
name: "OWASP Top 10 for LLM Applications — 2026",
|
|
473
|
+
short: "OWASP LLM 2026",
|
|
474
|
+
controls: {
|
|
475
|
+
LLM01: {
|
|
476
|
+
name: "Prompt Injection",
|
|
477
|
+
summary: "Instructions in a prompt or in retrieved content alter what the model does."
|
|
478
|
+
},
|
|
479
|
+
LLM02: {
|
|
480
|
+
name: "Sensitive Information Disclosure",
|
|
481
|
+
summary: "Credentials, personal data or proprietary content reach the model or its output."
|
|
482
|
+
},
|
|
483
|
+
LLM03: {
|
|
484
|
+
name: "Excessive Agency",
|
|
485
|
+
summary: "The model can take actions, or holds permissions, beyond what its task needs."
|
|
486
|
+
},
|
|
487
|
+
LLM04: {
|
|
488
|
+
name: "Supply Chain",
|
|
489
|
+
summary: "A compromised model, dataset, plugin or dependency enters the application."
|
|
490
|
+
},
|
|
491
|
+
LLM05: {
|
|
492
|
+
name: "Data and Model Poisoning",
|
|
493
|
+
summary: "Training, fine-tuning or embedding data is manipulated to change behaviour."
|
|
494
|
+
},
|
|
495
|
+
LLM06: {
|
|
496
|
+
name: "Unbounded Consumption",
|
|
497
|
+
summary: "The application permits resource use without a limit, exhausting budget or service."
|
|
498
|
+
},
|
|
499
|
+
LLM07: {
|
|
500
|
+
name: "Misinformation",
|
|
501
|
+
summary: "The model produces false or misleading content that is acted on as if true."
|
|
502
|
+
},
|
|
503
|
+
LLM08: {
|
|
504
|
+
name: "Hidden Context Exposure",
|
|
505
|
+
summary: "System prompts, hidden instructions or internal context are revealed."
|
|
506
|
+
},
|
|
507
|
+
LLM09: {
|
|
508
|
+
name: "Vector and Embedding Weaknesses",
|
|
509
|
+
summary: "Retrieval stores are manipulated or leak across tenants and permissions."
|
|
510
|
+
},
|
|
511
|
+
LLM10: {
|
|
512
|
+
name: "Improper Output Handling",
|
|
513
|
+
summary: "Model output is passed to downstream systems without validation."
|
|
514
|
+
}
|
|
515
|
+
}
|
|
516
|
+
} };
|
|
517
|
+
|
|
457
518
|
//#endregion
|
|
458
519
|
//#region ../kernel/src/pack.ts
|
|
459
520
|
function isObject(value) {
|
|
@@ -749,7 +810,8 @@
|
|
|
749
810
|
"id",
|
|
750
811
|
"title",
|
|
751
812
|
"when",
|
|
752
|
-
"effect"
|
|
813
|
+
"effect",
|
|
814
|
+
"frameworks"
|
|
753
815
|
]);
|
|
754
816
|
const id = check.text(`${path}.id`, value.id);
|
|
755
817
|
const effect = readEffect(check, `${path}.effect`, value.effect);
|
|
@@ -765,8 +827,40 @@
|
|
|
765
827
|
const title = check.text(`${path}.title`, value.title);
|
|
766
828
|
if (title !== void 0) rule.title = title;
|
|
767
829
|
}
|
|
830
|
+
if (value.frameworks !== void 0) {
|
|
831
|
+
const frameworks = readFrameworks(check, `${path}.frameworks`, value.frameworks);
|
|
832
|
+
if (frameworks !== void 0) rule.frameworks = frameworks;
|
|
833
|
+
}
|
|
768
834
|
return rule;
|
|
769
835
|
}
|
|
836
|
+
/**
|
|
837
|
+
* A citation names a framework edition and controls the registry knows. A
|
|
838
|
+
* typo here would be a rule that claims a mapping no view can render, so it
|
|
839
|
+
* is a pack error rather than an ignored key — the same line the rest of the
|
|
840
|
+
* parser takes.
|
|
841
|
+
*/
|
|
842
|
+
function readFrameworks(check, path, value) {
|
|
843
|
+
if (!isObject(value)) {
|
|
844
|
+
check.fail(path, "expected an object keyed by framework id");
|
|
845
|
+
return;
|
|
846
|
+
}
|
|
847
|
+
const out = {};
|
|
848
|
+
for (const [framework, controls] of Object.entries(value)) {
|
|
849
|
+
const known = FRAMEWORKS[framework];
|
|
850
|
+
if (known === void 0) {
|
|
851
|
+
check.fail(`${path}.${framework}`, `unknown framework; expected one of ${Object.keys(FRAMEWORKS).map((k) => `\`${k}\``).join(", ")}`);
|
|
852
|
+
continue;
|
|
853
|
+
}
|
|
854
|
+
if (!Array.isArray(controls) || !controls.every((c) => typeof c === "string")) {
|
|
855
|
+
check.fail(`${path}.${framework}`, "expected an array of control ids");
|
|
856
|
+
continue;
|
|
857
|
+
}
|
|
858
|
+
const ids = controls;
|
|
859
|
+
for (const control of ids) if (known.controls[control] === void 0) check.fail(`${path}.${framework}`, `unknown control \`${control}\` in ${known.name}`);
|
|
860
|
+
if (ids.length > 0) out[framework] = [...new Set(ids)];
|
|
861
|
+
}
|
|
862
|
+
return out;
|
|
863
|
+
}
|
|
770
864
|
/** Parse and validate a pack. Never throws; every problem lands in `errors`. */
|
|
771
865
|
function parsePack(source) {
|
|
772
866
|
let raw;
|
|
@@ -17,11 +17,16 @@
|
|
|
17
17
|
"./**/*.sample"
|
|
18
18
|
]
|
|
19
19
|
},
|
|
20
|
-
"secrets": {
|
|
20
|
+
"secrets": {
|
|
21
|
+
"minConfidence": 0.8
|
|
22
|
+
}
|
|
21
23
|
},
|
|
22
24
|
"effect": {
|
|
23
25
|
"kind": "deny",
|
|
24
26
|
"because": "the requested file contains a protected credential; the contents were not returned"
|
|
27
|
+
},
|
|
28
|
+
"frameworks": {
|
|
29
|
+
"owasp-llm-top-10-2026": ["LLM02"]
|
|
25
30
|
}
|
|
26
31
|
},
|
|
27
32
|
{
|
|
@@ -29,17 +34,27 @@
|
|
|
29
34
|
"title": "Private keys are refused everywhere",
|
|
30
35
|
"when": {
|
|
31
36
|
"capability": "fs:read",
|
|
32
|
-
"secrets": {
|
|
37
|
+
"secrets": {
|
|
38
|
+
"types": ["private-key"]
|
|
39
|
+
}
|
|
33
40
|
},
|
|
34
41
|
"effect": {
|
|
35
42
|
"kind": "deny",
|
|
36
43
|
"because": "the requested file contains a private key; the contents were not returned"
|
|
44
|
+
},
|
|
45
|
+
"frameworks": {
|
|
46
|
+
"owasp-llm-top-10-2026": ["LLM02"]
|
|
37
47
|
}
|
|
38
48
|
},
|
|
39
49
|
{
|
|
40
50
|
"id": "SEC-003",
|
|
41
51
|
"title": "Anything credential-shaped is at least recorded",
|
|
42
|
-
"when": {
|
|
52
|
+
"when": {
|
|
53
|
+
"capability": "fs:read",
|
|
54
|
+
"secrets": {
|
|
55
|
+
"minConfidence": 0.5
|
|
56
|
+
}
|
|
57
|
+
},
|
|
43
58
|
"effect": {
|
|
44
59
|
"kind": "warn",
|
|
45
60
|
"because": "a possible credential was noticed in this file; where no stronger rule fired, the read proceeded and this row is the record"
|
|
@@ -1,3 +1,3 @@
|
|
|
1
|
-
import { a as generateKeypair, c as verifyReceipt, d as canonical, f as chain, i as fingerprint, l as verifySignature, m as seal, n as readEntries, o as makeReceipt, p as hashEntry, r as verifyEntries, s as signEntry, t as Ledger, u as GENESIS } from "./src-
|
|
1
|
+
import { a as generateKeypair, c as verifyReceipt, d as canonical, f as chain, i as fingerprint, l as verifySignature, m as seal, n as readEntries, o as makeReceipt, p as hashEntry, r as verifyEntries, s as signEntry, t as Ledger, u as GENESIS } from "./src-Ciggx9ja.js";
|
|
2
2
|
|
|
3
3
|
export { GENESIS, Ledger, canonical, chain, fingerprint, generateKeypair, hashEntry, makeReceipt, readEntries, seal, signEntry, verifyEntries, verifyReceipt, verifySignature };
|
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
import { t as evaluate } from "./evaluate-CkYySl7T.js";
|
|
2
|
+
|
|
3
|
+
//#region src/test-run.ts
|
|
4
|
+
/** The path argument a scenario's call carries, if any. */
|
|
5
|
+
function pathOf(args) {
|
|
6
|
+
for (const key of [
|
|
7
|
+
"path",
|
|
8
|
+
"file_path",
|
|
9
|
+
"filePath",
|
|
10
|
+
"target"
|
|
11
|
+
]) {
|
|
12
|
+
const value = args[key];
|
|
13
|
+
if (typeof value === "string" && value !== "") return value;
|
|
14
|
+
}
|
|
15
|
+
}
|
|
16
|
+
/**
|
|
17
|
+
* Reject a malformed suite by saying what is wrong with it.
|
|
18
|
+
*
|
|
19
|
+
* A test file that silently runs nothing is the worst possible outcome here —
|
|
20
|
+
* it reports success — so every scenario must name a tool and an expected
|
|
21
|
+
* decision, and anything else is an error rather than a default.
|
|
22
|
+
*/
|
|
23
|
+
function parseSuite(parsed) {
|
|
24
|
+
const errors = [];
|
|
25
|
+
if (typeof parsed !== "object" || parsed === null) return {
|
|
26
|
+
ok: false,
|
|
27
|
+
errors: ["expected a JSON object"]
|
|
28
|
+
};
|
|
29
|
+
const root = parsed;
|
|
30
|
+
const scenarios = root.scenarios;
|
|
31
|
+
if (!Array.isArray(scenarios) || scenarios.length === 0) return {
|
|
32
|
+
ok: false,
|
|
33
|
+
errors: ["`scenarios` must be a non-empty array"]
|
|
34
|
+
};
|
|
35
|
+
const kinds = new Set([
|
|
36
|
+
"allow",
|
|
37
|
+
"deny",
|
|
38
|
+
"ask",
|
|
39
|
+
"correct"
|
|
40
|
+
]);
|
|
41
|
+
scenarios.forEach((raw, index) => {
|
|
42
|
+
const at = `scenarios[${index}]`;
|
|
43
|
+
if (typeof raw !== "object" || raw === null) {
|
|
44
|
+
errors.push(`${at}: expected an object`);
|
|
45
|
+
return;
|
|
46
|
+
}
|
|
47
|
+
const s = raw;
|
|
48
|
+
if (typeof s.name !== "string" || s.name === "") errors.push(`${at}.name: required`);
|
|
49
|
+
const when = s.when;
|
|
50
|
+
if (typeof when !== "object" || when === null) errors.push(`${at}.when: required`);
|
|
51
|
+
else {
|
|
52
|
+
if (typeof when.tool !== "string" || when.tool === "") errors.push(`${at}.when.tool: required`);
|
|
53
|
+
if (when.repeat !== void 0 && (typeof when.repeat !== "number" || when.repeat < 1)) errors.push(`${at}.when.repeat: must be a positive number`);
|
|
54
|
+
}
|
|
55
|
+
const then = s.then;
|
|
56
|
+
if (typeof then !== "object" || then === null) errors.push(`${at}.then: required`);
|
|
57
|
+
else if (typeof then.decision !== "string" || !kinds.has(then.decision)) errors.push(`${at}.then.decision: one of allow, deny, ask, correct`);
|
|
58
|
+
});
|
|
59
|
+
if (errors.length > 0) return {
|
|
60
|
+
ok: false,
|
|
61
|
+
errors
|
|
62
|
+
};
|
|
63
|
+
return {
|
|
64
|
+
ok: true,
|
|
65
|
+
suite: {
|
|
66
|
+
...typeof root.pack === "string" ? { pack: root.pack } : {},
|
|
67
|
+
scenarios
|
|
68
|
+
}
|
|
69
|
+
};
|
|
70
|
+
}
|
|
71
|
+
function runSuite(pack, suite, options) {
|
|
72
|
+
const results = suite.scenarios.map((scenario) => {
|
|
73
|
+
const args = scenario.when.args ?? {};
|
|
74
|
+
const capability = options.capabilityFor(scenario.when.tool);
|
|
75
|
+
const target = pathOf(args);
|
|
76
|
+
const findings = target === void 0 || options.inspect === void 0 ? void 0 : options.inspect(target);
|
|
77
|
+
const decision = evaluate(pack, {
|
|
78
|
+
tool: scenario.when.tool,
|
|
79
|
+
...capability === void 0 ? {} : { capability },
|
|
80
|
+
args,
|
|
81
|
+
...findings !== void 0 && findings.length > 0 ? { secrets: findings } : {},
|
|
82
|
+
...capability === void 0 || scenario.when.repeat === void 0 ? {} : { counts: { [capability]: scenario.when.repeat } }
|
|
83
|
+
});
|
|
84
|
+
const secretTypes = [...new Set((findings ?? []).map((f) => f.type))];
|
|
85
|
+
const failures = [];
|
|
86
|
+
if (decision.kind !== scenario.then.decision) failures.push(`decision was ${decision.kind}, expected ${scenario.then.decision}`);
|
|
87
|
+
if (scenario.then.rule !== void 0 && decision.rule !== scenario.then.rule) failures.push(`rule was ${decision.rule ?? "none"}, expected ${scenario.then.rule}`);
|
|
88
|
+
if (scenario.then.secretTypes !== void 0) {
|
|
89
|
+
const missing = scenario.then.secretTypes.filter((type) => !secretTypes.includes(type));
|
|
90
|
+
if (missing.length > 0) failures.push(`no finding of type ${missing.join(", ")}`);
|
|
91
|
+
}
|
|
92
|
+
return {
|
|
93
|
+
name: scenario.name,
|
|
94
|
+
...scenario.given === void 0 ? {} : { given: scenario.given },
|
|
95
|
+
when: scenario.when,
|
|
96
|
+
expected: scenario.then,
|
|
97
|
+
actual: {
|
|
98
|
+
decision: decision.kind,
|
|
99
|
+
...decision.rule === void 0 ? {} : { rule: decision.rule },
|
|
100
|
+
secretTypes
|
|
101
|
+
},
|
|
102
|
+
pass: failures.length === 0,
|
|
103
|
+
failures
|
|
104
|
+
};
|
|
105
|
+
});
|
|
106
|
+
const failed = results.filter((r) => !r.pass).length;
|
|
107
|
+
return {
|
|
108
|
+
pack: `${pack.pack} ${pack.version}`,
|
|
109
|
+
passed: results.length - failed,
|
|
110
|
+
failed,
|
|
111
|
+
results,
|
|
112
|
+
ok: failed === 0
|
|
113
|
+
};
|
|
114
|
+
}
|
|
115
|
+
/** The report, as lines. Phrased for a reader; the command only prints it. */
|
|
116
|
+
function renderSuite(result) {
|
|
117
|
+
const lines = [`prae test — ${result.pack}`, ""];
|
|
118
|
+
const width = Math.max(...result.results.map((r) => r.name.length), 20);
|
|
119
|
+
for (const r of result.results) {
|
|
120
|
+
const mark = r.pass ? "✓" : "✕";
|
|
121
|
+
const shown = r.pass ? `${r.actual.decision.toUpperCase()}${r.actual.rule === void 0 ? "" : ` ${r.actual.rule}`}` : r.failures.join("; ");
|
|
122
|
+
lines.push(` ${mark} ${r.name.padEnd(width)} ${shown}`);
|
|
123
|
+
}
|
|
124
|
+
for (const r of result.results.filter((one) => !one.pass)) {
|
|
125
|
+
lines.push("", ` ✕ ${r.name}`);
|
|
126
|
+
if (r.given !== void 0) lines.push(` GIVEN ${r.given}`);
|
|
127
|
+
lines.push(` WHEN ${r.when.tool}${r.when.repeat === void 0 ? "" : ` ×${r.when.repeat}`} ${JSON.stringify(r.when.args ?? {})}`);
|
|
128
|
+
lines.push(` THEN ${r.expected.decision}${r.expected.rule === void 0 ? "" : ` · ${r.expected.rule}`}`);
|
|
129
|
+
lines.push(` GOT ${r.failures.join("; ")}`);
|
|
130
|
+
}
|
|
131
|
+
const n = result.results.length;
|
|
132
|
+
lines.push("", `${n} scenario${n === 1 ? "" : "s"} · ${result.passed} passed · ${result.failed} failed`, result.ok ? "SECURITY TEST: PASS" : "SECURITY TEST: FAIL");
|
|
133
|
+
return lines;
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
//#endregion
|
|
137
|
+
export { parseSuite, renderSuite, runSuite };
|
|
@@ -1,3 +1,72 @@
|
|
|
1
|
+
//#region ../kernel/src/frameworks.ts
|
|
2
|
+
const FRAMEWORKS = { "owasp-llm-top-10-2026": {
|
|
3
|
+
name: "OWASP Top 10 for LLM Applications — 2026",
|
|
4
|
+
short: "OWASP LLM 2026",
|
|
5
|
+
controls: {
|
|
6
|
+
LLM01: {
|
|
7
|
+
name: "Prompt Injection",
|
|
8
|
+
summary: "Instructions in a prompt or in retrieved content alter what the model does."
|
|
9
|
+
},
|
|
10
|
+
LLM02: {
|
|
11
|
+
name: "Sensitive Information Disclosure",
|
|
12
|
+
summary: "Credentials, personal data or proprietary content reach the model or its output."
|
|
13
|
+
},
|
|
14
|
+
LLM03: {
|
|
15
|
+
name: "Excessive Agency",
|
|
16
|
+
summary: "The model can take actions, or holds permissions, beyond what its task needs."
|
|
17
|
+
},
|
|
18
|
+
LLM04: {
|
|
19
|
+
name: "Supply Chain",
|
|
20
|
+
summary: "A compromised model, dataset, plugin or dependency enters the application."
|
|
21
|
+
},
|
|
22
|
+
LLM05: {
|
|
23
|
+
name: "Data and Model Poisoning",
|
|
24
|
+
summary: "Training, fine-tuning or embedding data is manipulated to change behaviour."
|
|
25
|
+
},
|
|
26
|
+
LLM06: {
|
|
27
|
+
name: "Unbounded Consumption",
|
|
28
|
+
summary: "The application permits resource use without a limit, exhausting budget or service."
|
|
29
|
+
},
|
|
30
|
+
LLM07: {
|
|
31
|
+
name: "Misinformation",
|
|
32
|
+
summary: "The model produces false or misleading content that is acted on as if true."
|
|
33
|
+
},
|
|
34
|
+
LLM08: {
|
|
35
|
+
name: "Hidden Context Exposure",
|
|
36
|
+
summary: "System prompts, hidden instructions or internal context are revealed."
|
|
37
|
+
},
|
|
38
|
+
LLM09: {
|
|
39
|
+
name: "Vector and Embedding Weaknesses",
|
|
40
|
+
summary: "Retrieval stores are manipulated or leak across tenants and permissions."
|
|
41
|
+
},
|
|
42
|
+
LLM10: {
|
|
43
|
+
name: "Improper Output Handling",
|
|
44
|
+
summary: "Model output is passed to downstream systems without validation."
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
} };
|
|
48
|
+
/**
|
|
49
|
+
* Every framework control a rule cites, resolved to names.
|
|
50
|
+
*
|
|
51
|
+
* A pack that parsed cites only ids the registry knows, so the lookups here
|
|
52
|
+
* cannot miss; the fallbacks exist for a `Rule` built by hand in code.
|
|
53
|
+
*/
|
|
54
|
+
function cite(rule) {
|
|
55
|
+
const out = [];
|
|
56
|
+
for (const [framework, controls] of Object.entries(rule.frameworks ?? {})) {
|
|
57
|
+
const known = FRAMEWORKS[framework];
|
|
58
|
+
for (const control of controls) out.push({
|
|
59
|
+
framework,
|
|
60
|
+
frameworkName: known?.name ?? framework,
|
|
61
|
+
frameworkShort: known?.short ?? framework,
|
|
62
|
+
control,
|
|
63
|
+
controlName: known?.controls[control]?.name ?? control
|
|
64
|
+
});
|
|
65
|
+
}
|
|
66
|
+
return out;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
//#endregion
|
|
1
70
|
//#region src/console/logo.ts
|
|
2
71
|
/**
|
|
3
72
|
* The mark, inlined.
|
|
@@ -629,7 +698,7 @@ function renderNav(items, active) {
|
|
|
629
698
|
* the obligation, and a tool that told them the answer would be selling the
|
|
630
699
|
* one thing it cannot supply.
|
|
631
700
|
*/
|
|
632
|
-
const
|
|
701
|
+
const OBLIGATIONS = [
|
|
633
702
|
{
|
|
634
703
|
id: "EU AI Act art. 12",
|
|
635
704
|
asks: "Automatic recording of events over the lifetime of the system, to a degree that enables tracing.",
|
|
@@ -671,6 +740,71 @@ const CONTROLS = [
|
|
|
671
740
|
})
|
|
672
741
|
}
|
|
673
742
|
];
|
|
743
|
+
const OWASP = "owasp-llm-top-10-2026";
|
|
744
|
+
/** Rules in the pack that cite one control of one framework. */
|
|
745
|
+
function citing(pack, framework, control) {
|
|
746
|
+
return (pack?.rules ?? []).filter((rule) => (rule.frameworks?.[framework] ?? []).includes(control));
|
|
747
|
+
}
|
|
748
|
+
/**
|
|
749
|
+
* Why a runtime execution gate is not the control for a risk no rule cites.
|
|
750
|
+
*
|
|
751
|
+
* Said per risk rather than as one generic line, because "out of scope" with
|
|
752
|
+
* no reason reads as an excuse. Each of these is a boundary a reader can
|
|
753
|
+
* check: the risk lives somewhere a tool-call gate does not stand.
|
|
754
|
+
*/
|
|
755
|
+
const NOT_THIS_BOUNDARY = {
|
|
756
|
+
LLM04: "Where a model or dependency came from is settled before an agent runs; the gate sees the call, not the provenance.",
|
|
757
|
+
LLM05: "Training and fine-tuning data sit outside the execution boundary this gate stands on.",
|
|
758
|
+
LLM07: "Whether an answer is true is not a property of a tool call.",
|
|
759
|
+
LLM08: "The assembled system prompt is sealed into the ledger as evidence of what the model was told; nothing here stops the model repeating it.",
|
|
760
|
+
LLM09: "Retrieval stores are upstream of the call the gate judges.",
|
|
761
|
+
LLM10: "What downstream code does with model output is that code's boundary, not this one's."
|
|
762
|
+
};
|
|
763
|
+
function owaspRow(control) {
|
|
764
|
+
const framework = FRAMEWORKS[OWASP];
|
|
765
|
+
const risk = framework?.controls[control];
|
|
766
|
+
const name = risk?.name ?? control;
|
|
767
|
+
return {
|
|
768
|
+
id: `${framework?.short ?? OWASP} · ${control}`,
|
|
769
|
+
asks: `${name}. ${risk?.summary ?? ""}`.trim(),
|
|
770
|
+
evidence: (data) => {
|
|
771
|
+
if (data.pack === void 0) return {
|
|
772
|
+
state: "absent",
|
|
773
|
+
note: "No pack supplied to this view."
|
|
774
|
+
};
|
|
775
|
+
if (control === "LLM01") {
|
|
776
|
+
const bounding = citing(data.pack, OWASP, "LLM03");
|
|
777
|
+
return {
|
|
778
|
+
state: "partial",
|
|
779
|
+
note: bounding.length === 0 ? "Not mitigated: this gate does not detect injected instructions, and no LLM03 control in this pack bounds what one can execute." : `Not mitigated: this gate does not detect injected instructions. Bounded: ${bounding.length} LLM03 control${bounding.length === 1 ? "" : "s"} (${bounding.map((r) => r.id).join(", ")}) decide what an injected instruction can execute.`
|
|
780
|
+
};
|
|
781
|
+
}
|
|
782
|
+
const rules = citing(data.pack, OWASP, control);
|
|
783
|
+
if (rules.length === 0) return {
|
|
784
|
+
state: "absent",
|
|
785
|
+
note: NOT_THIS_BOUNDARY[control] ?? "No rule in this pack cites this risk."
|
|
786
|
+
};
|
|
787
|
+
const ids = new Set(rules.map((rule) => rule.id));
|
|
788
|
+
const decisions = data.entries.filter((entry) => entry.rule !== void 0 && ids.has(entry.rule));
|
|
789
|
+
const listed = rules.map((rule) => rule.id).join(", ");
|
|
790
|
+
return decisions.length > 0 ? {
|
|
791
|
+
state: "covered",
|
|
792
|
+
note: `${rules.length} control${rules.length === 1 ? "" : "s"} mapped (${listed}) · ${decisions.length} sealed decision${decisions.length === 1 ? "" : "s"} under them in this record.`
|
|
793
|
+
} : {
|
|
794
|
+
state: "partial",
|
|
795
|
+
note: `${rules.length} control${rules.length === 1 ? "" : "s"} mapped (${listed}); none has decided a call in this record yet, so the mapping is declared, not exercised.`
|
|
796
|
+
};
|
|
797
|
+
}
|
|
798
|
+
};
|
|
799
|
+
}
|
|
800
|
+
const OWASP_ROWS = Object.keys(FRAMEWORKS[OWASP]?.controls ?? {}).map(owaspRow);
|
|
801
|
+
/**
|
|
802
|
+
* The obligations first, then the OWASP risks. Two kinds of row in one table
|
|
803
|
+
* on purpose: an auditor reads down one list and finds the audit obligations
|
|
804
|
+
* the ledger evidences and the application risks the pack's rules address,
|
|
805
|
+
* each stated in its framework's own words.
|
|
806
|
+
*/
|
|
807
|
+
const CONTROLS = [...OBLIGATIONS, ...OWASP_ROWS];
|
|
674
808
|
|
|
675
809
|
//#endregion
|
|
676
810
|
//#region src/console/views.ts
|
|
@@ -789,11 +923,17 @@ function renderPacks(data) {
|
|
|
789
923
|
});
|
|
790
924
|
const baseline = pack.baseline?.capabilities;
|
|
791
925
|
const caps = baseline === void 0 ? `<p style="padding:0 30px 22px;font-size:14px;color:var(--secondary);max-width:76ch">This pack sets no capability ceiling. It governs through its rules alone, and withholds nothing an author declares.</p>` : `<div class="caps">${baseline.map((c) => `<span class="cap">${esc(c)}</span>`).join("")}</div>`;
|
|
792
|
-
const
|
|
926
|
+
const cited = /* @__PURE__ */ new Map();
|
|
927
|
+
for (const rule of pack.rules) for (const c of cite(rule)) cited.set(c.framework, c.frameworkName);
|
|
928
|
+
const frameworks = cited.size === 0 ? "" : `<div class="caps">${[...cited.values()].map((name) => `<span class="cap">${esc(name)}</span>`).join("")}</div>`;
|
|
929
|
+
const rows = pack.rules.map((rule) => {
|
|
930
|
+
const tags = cite(rule).map((c) => `<br><span style="color:var(--secondary)">${esc(c.frameworkShort)} · ${esc(c.control)} ${esc(c.controlName)}</span>`).join("");
|
|
931
|
+
return ` <div class="grid rules row">
|
|
793
932
|
<div class="seq">${esc(rule.id)}</div>
|
|
794
|
-
<div class="call">${esc(rule.title ?? rule.effect.because)}<br><span style="color:var(--muted)">${esc(matcherOf(rule))}</span
|
|
933
|
+
<div class="call">${esc(rule.title ?? rule.effect.because)}<br><span style="color:var(--muted)">${esc(matcherOf(rule))}</span>${tags}</div>
|
|
795
934
|
<div class="effect e-${esc(rule.effect.kind)}">${esc(rule.effect.kind.toUpperCase())}</div>
|
|
796
|
-
</div
|
|
935
|
+
</div>`;
|
|
936
|
+
}).join("\n");
|
|
797
937
|
const body = ` <section class="sect">
|
|
798
938
|
<h2>${esc(pack.pack)} <span style="font-family:var(--proof);font-size:14px;color:var(--muted)">${esc(pack.version)}</span></h2>
|
|
799
939
|
<p>The operator baseline is intersected with whatever a plugin's own
|
|
@@ -801,6 +941,7 @@ function renderPacks(data) {
|
|
|
801
941
|
grant a capability an author never asked for.</p>
|
|
802
942
|
</section>
|
|
803
943
|
${caps}
|
|
944
|
+
${frameworks}
|
|
804
945
|
<div class="grid rules thead">
|
|
805
946
|
<div>CLAUSE</div><div>RULE</div><div>EFFECT</div>
|
|
806
947
|
</div>
|
|
@@ -840,11 +981,17 @@ ${CONTROLS.map((control) => {
|
|
|
840
981
|
</div>`;
|
|
841
982
|
}).join("\n")}
|
|
842
983
|
<section class="sect" style="padding-bottom:26px">
|
|
843
|
-
<h2>
|
|
844
|
-
<p>
|
|
845
|
-
harness records what was decided and never who decided
|
|
846
|
-
approval needs an identity anchor outside this machine,
|
|
847
|
-
control-plane concern this build does not claim.</p>
|
|
984
|
+
<h2>Reading the states</h2>
|
|
985
|
+
<p>For the audit obligations, several rows stop at <em>partial</em> for
|
|
986
|
+
one reason: the harness records what was decided and never who decided
|
|
987
|
+
it. Attributable approval needs an identity anchor outside this machine,
|
|
988
|
+
which is a control-plane concern this build does not claim.</p>
|
|
989
|
+
<p>For the OWASP risks, <em>evidence</em> means rules in this pack cite
|
|
990
|
+
the risk and have decided calls in this record; <em>partial</em> means
|
|
991
|
+
they cite it and have not yet, or — for prompt injection — that the gate
|
|
992
|
+
bounds the risk without claiming to detect it; <em>none</em> means no
|
|
993
|
+
rule cites it, with the reason a tool-call gate is not where that risk
|
|
994
|
+
is controlled.</p>
|
|
848
995
|
</section>`;
|
|
849
996
|
return renderShell({
|
|
850
997
|
title: "Compliance map",
|
|
@@ -926,4 +1073,4 @@ ${history}`;
|
|
|
926
1073
|
}
|
|
927
1074
|
|
|
928
1075
|
//#endregion
|
|
929
|
-
export { renderPacks as a, renderShell as c, renderOverview as i, VERDICT_LABEL as l, renderApprovals as n, esc as o, renderCompliance as r, renderNav as s, navFor as t };
|
|
1076
|
+
export { renderPacks as a, renderShell as c, cite as d, renderOverview as i, VERDICT_LABEL as l, renderApprovals as n, esc as o, renderCompliance as r, renderNav as s, navFor as t, FRAMEWORKS as u };
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "prae-cli",
|
|
3
|
-
"version": "0.
|
|
4
|
-
"description": "Operator policy for agent harnesses: validate packs, explain a decision, verify a decision log, and provision a governed DSH profile.",
|
|
3
|
+
"version": "0.6.1",
|
|
4
|
+
"description": "Operator policy for agent harnesses: validate packs, explain a decision, test a pack against security invariants, verify a decision log, and provision a governed DSH profile.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"dsh-plugin",
|
|
7
7
|
"governance",
|
|
File without changes
|
|
File without changes
|
|
File without changes
|