rigorrun 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +51 -3
- package/dist/rigorrun.mjs +1930 -265
- package/package.json +2 -2
- package/ui/assets/DemoPage-DvY0tdpJ.js +295 -0
- package/ui/assets/Evidence-B65d5xlS.js +1 -0
- package/ui/assets/index-CziJAnNn.css +2 -0
- package/ui/assets/index-DEjJuxpK.js +30 -0
- package/ui/index.html +25 -2
- package/ui/robots.txt +8 -0
- package/ui/assets/DemoPage-B1s79hcz.js +0 -295
- package/ui/assets/Proof-Doh3bIfF.js +0 -1
- package/ui/assets/index-CR8r1ZFu.css +0 -2
- package/ui/assets/index-DAiDcuMt.js +0 -14
package/dist/rigorrun.mjs
CHANGED
|
@@ -61,7 +61,7 @@ async function writeText(path, text2) {
|
|
|
61
61
|
}
|
|
62
62
|
|
|
63
63
|
// src/help.ts
|
|
64
|
-
var VERSION = "0.
|
|
64
|
+
var VERSION = "0.2.0";
|
|
65
65
|
var HELP = `RigorRun ${VERSION} - acceptance testing for tool-using AI agents.
|
|
66
66
|
|
|
67
67
|
Connect your system. Show RigorRun how one job is done. Connect your agent.
|
|
@@ -73,6 +73,15 @@ USAGE
|
|
|
73
73
|
where you connect a system, teach a job and watch a
|
|
74
74
|
run. Everything below is for scripts and CI.
|
|
75
75
|
|
|
76
|
+
VERIFY A SERVER
|
|
77
|
+
verify <server-ref> Run an MCP server's tools in a container RigorRun
|
|
78
|
+
controls, and report what each one actually does
|
|
79
|
+
against what the server says it does. Needs no
|
|
80
|
+
project, no agent and no browser.
|
|
81
|
+
|
|
82
|
+
rigorrun verify npm:<package>@<version>
|
|
83
|
+
rigorrun verify dir:<path>
|
|
84
|
+
|
|
76
85
|
PROJECTS
|
|
77
86
|
projects List the projects on this machine.
|
|
78
87
|
run --project <id> Run the project's suite against its agent.
|
|
@@ -140,10 +149,19 @@ RUN / GATE OPTIONS
|
|
|
140
149
|
COMPARE OPTIONS
|
|
141
150
|
--baseline <runId> Compare against this instead of the baseline.
|
|
142
151
|
|
|
152
|
+
VERIFY OPTIONS
|
|
153
|
+
--max-undetermined <n> Undetermined findings tolerated. Default 0.
|
|
154
|
+
--min-exercised <n> Tools that must have been exercised. Default 1.
|
|
155
|
+
--strict Treat minor contradictions as failures too.
|
|
156
|
+
-o, --out <file> Where to write the record.
|
|
157
|
+
--json Print the record and nothing else.
|
|
158
|
+
|
|
143
159
|
EXIT CODES
|
|
144
160
|
0 success, or the gate passed
|
|
145
|
-
1 the benchmark failed,
|
|
161
|
+
1 the benchmark failed, the gate was not met, or a declaration was
|
|
162
|
+
contradicted by what the server was observed to do
|
|
146
163
|
2 configuration or runtime error
|
|
164
|
+
3 verify only: it ran, but established too little to be worth much
|
|
147
165
|
|
|
148
166
|
EXAMPLES
|
|
149
167
|
rigorrun start here
|
|
@@ -152,13 +170,48 @@ EXAMPLES
|
|
|
152
170
|
rigorrun gate --project p_1a2b3c --min-success 0.95
|
|
153
171
|
rigorrun compare-runs --project p_1a2b3c run_9f8e7d
|
|
154
172
|
|
|
173
|
+
rigorrun verify npm:@modelcontextprotocol/server-memory@2026.8.31
|
|
174
|
+
|
|
155
175
|
rigorrun demo the bundled example
|
|
156
|
-
rigorrun
|
|
176
|
+
rigorrun run examples/refund-workflow/benchmark.json --agent naive
|
|
157
177
|
|
|
158
178
|
Your systems, your credentials and your recordings stay on this machine. There
|
|
159
179
|
is no account, and nothing is uploaded unless you ask for it.
|
|
160
180
|
`;
|
|
161
181
|
var COMMAND_HELP = {
|
|
182
|
+
verify: `rigorrun verify <server-ref> - find out what a server's tools do
|
|
183
|
+
|
|
184
|
+
Fetches the server, pins it to the exact bytes the registry published, runs it
|
|
185
|
+
in a container with no network and no access to this machine, calls each tool
|
|
186
|
+
with arguments derived from its own schema, and reads the container's
|
|
187
|
+
filesystem before and after to see what actually changed.
|
|
188
|
+
|
|
189
|
+
Then it compares that against what the server declared. A tool annotated
|
|
190
|
+
readOnlyHint: true that writes is reported CONTRADICTED, because that
|
|
191
|
+
annotation decides whether an agent may call it without asking.
|
|
192
|
+
|
|
193
|
+
Nothing it could not establish is reported as a fact. A tool it could not
|
|
194
|
+
exercise safely is listed, with the reason.
|
|
195
|
+
|
|
196
|
+
REFERENCES
|
|
197
|
+
npm:<package>@<version> A published server. Pinned by the registry's digest.
|
|
198
|
+
dir:<path> A server on this machine.
|
|
199
|
+
|
|
200
|
+
OPTIONS
|
|
201
|
+
--max-undetermined <n> Undetermined findings tolerated. Default 0.
|
|
202
|
+
--min-exercised <n> Tools that must have been exercised. Default 1.
|
|
203
|
+
--strict Treat minor contradictions as failures too.
|
|
204
|
+
-o, --out <file> Where to write the record.
|
|
205
|
+
--json Print the record to stdout and nothing else.
|
|
206
|
+
|
|
207
|
+
EXIT CODES
|
|
208
|
+
0 nothing contradicted
|
|
209
|
+
1 a declaration was contradicted
|
|
210
|
+
2 the verification could not be run at all
|
|
211
|
+
3 it ran, but established too little to be worth much
|
|
212
|
+
|
|
213
|
+
REQUIRES
|
|
214
|
+
A container runtime. Run \`rigorrun doctor\` to see whether you have one.`,
|
|
162
215
|
demo: `rigorrun demo - run the complete offline demo
|
|
163
216
|
|
|
164
217
|
Compiles the bundled recorded refund workflow into a contract, generates the
|
|
@@ -179,6 +232,9 @@ configuration error.
|
|
|
179
232
|
|
|
180
233
|
OPTIONS
|
|
181
234
|
--agent <id> Required. The agent to gate.
|
|
235
|
+
--allow-reference Permit --agent reference. It is handed the
|
|
236
|
+
answer, so the gate passes by construction
|
|
237
|
+
and measures the suite, not an agent.
|
|
182
238
|
--min-success <0..1> Default 0.95
|
|
183
239
|
--min-policy <0..1> Default 1
|
|
184
240
|
--max-policy-violations <n> Default 0
|
|
@@ -192,6 +248,12 @@ Starts a loopback-only HTTP listener that accepts a single sanitised workflow
|
|
|
192
248
|
trace from the RigorRun recorder extension and writes it to disk. Nothing is
|
|
193
249
|
sent anywhere; the listener stops as soon as a trace arrives.
|
|
194
250
|
|
|
251
|
+
The trace it writes is a record of what happened in a page. It is not a
|
|
252
|
+
contract, and "rigorrun compile" cannot read it \u2014 compiling needs the state
|
|
253
|
+
your system held before and after the job, and a browser recording of an
|
|
254
|
+
uninstrumented application does not carry that. To build a suite, connect the
|
|
255
|
+
system through the interface and do the job there.
|
|
256
|
+
|
|
195
257
|
OPTIONS
|
|
196
258
|
--port <n> Port to listen on. Default 8787.
|
|
197
259
|
-o, --out <path> Where to write the trace. Default .rigorrun/traces/<id>.json
|
|
@@ -272,6 +334,7 @@ var CANONICAL_TRACE_SCHEMA_VERSION = 1;
|
|
|
272
334
|
var ENVIRONMENT_CONTRACT_SCHEMA_VERSION = 1;
|
|
273
335
|
var BENCHMARK_SCHEMA_VERSION = 1;
|
|
274
336
|
var RUN_SCHEMA_VERSION = 1;
|
|
337
|
+
var RECORD_SCHEMA_VERSION = 1;
|
|
275
338
|
|
|
276
339
|
// ../core/src/hash.ts
|
|
277
340
|
function canonicalJson(value) {
|
|
@@ -286,9 +349,9 @@ function canonicalise(value) {
|
|
|
286
349
|
const source = value;
|
|
287
350
|
const out = {};
|
|
288
351
|
for (const key of Object.keys(source).sort()) {
|
|
289
|
-
const
|
|
290
|
-
if (
|
|
291
|
-
out[key] = canonicalise(
|
|
352
|
+
const entry2 = source[key];
|
|
353
|
+
if (entry2 === void 0) continue;
|
|
354
|
+
out[key] = canonicalise(entry2);
|
|
292
355
|
}
|
|
293
356
|
return out;
|
|
294
357
|
}
|
|
@@ -300,6 +363,14 @@ async function sha256(input) {
|
|
|
300
363
|
const digest = await globalThis.crypto.subtle.digest("SHA-256", bytes);
|
|
301
364
|
return toHex(digest);
|
|
302
365
|
}
|
|
366
|
+
async function sha512Base64(bytes) {
|
|
367
|
+
const buffer = new ArrayBuffer(bytes.byteLength);
|
|
368
|
+
new Uint8Array(buffer).set(bytes);
|
|
369
|
+
const digest = await globalThis.crypto.subtle.digest("SHA-512", buffer);
|
|
370
|
+
let binary = "";
|
|
371
|
+
for (const byte of new Uint8Array(digest)) binary += String.fromCharCode(byte);
|
|
372
|
+
return btoa(binary);
|
|
373
|
+
}
|
|
303
374
|
async function hashValue(value) {
|
|
304
375
|
return `sha256:${await sha256(canonicalJson(value))}`;
|
|
305
376
|
}
|
|
@@ -340,7 +411,14 @@ var ASSERTION_KINDS = [
|
|
|
340
411
|
var AssertionKindSchema = z.enum(ASSERTION_KINDS);
|
|
341
412
|
var EvaluatorSchema = z.enum(["deterministic", "model_judged", "human_review"]);
|
|
342
413
|
var AssertionSeveritySchema = z.enum(["success", "policy", "invariant"]);
|
|
343
|
-
var VERIFICATION_SOURCES = [
|
|
414
|
+
var VERIFICATION_SOURCES = [
|
|
415
|
+
"STATE",
|
|
416
|
+
"EVENT",
|
|
417
|
+
"OUTPUT",
|
|
418
|
+
"HUMAN",
|
|
419
|
+
"MODEL",
|
|
420
|
+
"DECLARED"
|
|
421
|
+
];
|
|
344
422
|
var VerificationSourceSchema = z.enum(VERIFICATION_SOURCES);
|
|
345
423
|
var FAILURE_SEVERITIES = ["INFO", "MINOR", "MAJOR", "CRITICAL"];
|
|
346
424
|
var FailureSeveritySchema = z.enum(FAILURE_SEVERITIES);
|
|
@@ -656,13 +734,13 @@ function surfaceText(trace) {
|
|
|
656
734
|
// ../core/src/traceNormalize.ts
|
|
657
735
|
var DEFAULT_ACTOR = { id: "operator", kind: "human", label: "Operator" };
|
|
658
736
|
function fromActionLog(entries, options) {
|
|
659
|
-
const steps = entries.map((
|
|
737
|
+
const steps = entries.map((entry2, index) => ({
|
|
660
738
|
id: `step_${String(index + 1).padStart(3, "0")}`,
|
|
661
739
|
ordinal: index,
|
|
662
|
-
at:
|
|
740
|
+
at: entry2.at,
|
|
663
741
|
kind: "action",
|
|
664
|
-
action: { name:
|
|
665
|
-
surfaceText:
|
|
742
|
+
action: { name: entry2.action, args: entry2.args ?? {}, ok: entry2.ok ?? true },
|
|
743
|
+
surfaceText: entry2.surfaceText ?? []
|
|
666
744
|
}));
|
|
667
745
|
const last = entries[entries.length - 1];
|
|
668
746
|
return finalise(
|
|
@@ -1214,7 +1292,7 @@ var AgentScoreSchema = z7.object({
|
|
|
1214
1292
|
failedThresholds: z7.array(z7.string()).default([])
|
|
1215
1293
|
});
|
|
1216
1294
|
var VERIFICATION_STRENGTHS = ["AUTHORITATIVE", "PARTIAL", "OBSERVATIONAL"];
|
|
1217
|
-
var ISOLATION_LEVELS = ["RESET", "NONE"];
|
|
1295
|
+
var ISOLATION_LEVELS = ["RESET", "PARTIAL", "DECLARED", "NONE"];
|
|
1218
1296
|
var RunLimitSchema = z7.object({
|
|
1219
1297
|
id: z7.string(),
|
|
1220
1298
|
limit: z7.string(),
|
|
@@ -1258,6 +1336,179 @@ var RunResultSchema = z7.object({
|
|
|
1258
1336
|
rigorrunVersion: z7.string().default("0.1.0")
|
|
1259
1337
|
});
|
|
1260
1338
|
|
|
1339
|
+
// ../core/src/record.ts
|
|
1340
|
+
import { z as z8 } from "zod";
|
|
1341
|
+
var VERIFICATION_RECORD_SCHEMA = "rigorrun.record/1";
|
|
1342
|
+
var TARGET_SCHEMES = ["npm", "dir"];
|
|
1343
|
+
var TargetSchemeSchema = z8.enum(TARGET_SCHEMES);
|
|
1344
|
+
var TargetSchema = z8.object({
|
|
1345
|
+
/** Exactly what the operator typed, kept verbatim. */
|
|
1346
|
+
ref: z8.string().min(1),
|
|
1347
|
+
scheme: TargetSchemeSchema,
|
|
1348
|
+
/** The version actually used. Never a tag, never a range. */
|
|
1349
|
+
resolvedVersion: z8.string(),
|
|
1350
|
+
/** `sha512-<base64>` from the registry, or `dirhash:sha256:<hex>`. */
|
|
1351
|
+
digest: z8.string().min(1),
|
|
1352
|
+
/**
|
|
1353
|
+
* The whole installed dependency closure, hashed.
|
|
1354
|
+
*
|
|
1355
|
+
* `digest` covers the package. This covers everything that got installed
|
|
1356
|
+
* with it, which is what changes when a transitive dependency moves and the
|
|
1357
|
+
* package version does not.
|
|
1358
|
+
*/
|
|
1359
|
+
treeDigest: z8.string().default(""),
|
|
1360
|
+
tarballUrl: z8.string().default(""),
|
|
1361
|
+
fetchedAt: z8.string()
|
|
1362
|
+
});
|
|
1363
|
+
var IsolationProofSchema = z8.object({
|
|
1364
|
+
resets: z8.number().int().nonnegative(),
|
|
1365
|
+
/** The state digest after each independent reset. Recomputable by a reader. */
|
|
1366
|
+
initialDigests: z8.array(z8.string()),
|
|
1367
|
+
/** Paths that came back with different contents each time. */
|
|
1368
|
+
unstablePaths: z8.array(z8.string()).default([])
|
|
1369
|
+
});
|
|
1370
|
+
var HarnessSchema = z8.object({
|
|
1371
|
+
version: z8.string(),
|
|
1372
|
+
/** e.g. `docker 28.5.2`. Which engine actually ran this. */
|
|
1373
|
+
runtime: z8.string(),
|
|
1374
|
+
/** Whether that engine's daemon runs as root, because it changes the boundary. */
|
|
1375
|
+
rootless: z8.boolean(),
|
|
1376
|
+
/** The base image, pinned by digest. A tag is not a baseline. */
|
|
1377
|
+
baseImage: z8.string(),
|
|
1378
|
+
imageId: z8.string().default(""),
|
|
1379
|
+
isolation: z8.enum(ISOLATION_LEVELS).default("NONE"),
|
|
1380
|
+
isolationProof: IsolationProofSchema,
|
|
1381
|
+
/** False for every run this harness performs. Opt-in egress is recorded here. */
|
|
1382
|
+
networkEgress: z8.boolean(),
|
|
1383
|
+
/** Whether the read-only posture held: the container layer stayed empty. */
|
|
1384
|
+
readonlyVerified: z8.boolean(),
|
|
1385
|
+
/** Whether the target's own install scripts ever ran, and where. */
|
|
1386
|
+
installScripts: z8.enum(["skipped", "ran-in-sandbox"]),
|
|
1387
|
+
limits: z8.object({
|
|
1388
|
+
pids: z8.number().int().positive(),
|
|
1389
|
+
memoryMb: z8.number().int().positive(),
|
|
1390
|
+
cpus: z8.number().positive()
|
|
1391
|
+
}),
|
|
1392
|
+
/** The literal argv, so a reader can check the posture rather than trust it. */
|
|
1393
|
+
runArgv: z8.array(z8.string()).default([]),
|
|
1394
|
+
seed: z8.string(),
|
|
1395
|
+
/** What this run does not prove. Required, and required non-empty. */
|
|
1396
|
+
caveats: z8.array(z8.string()).min(1)
|
|
1397
|
+
});
|
|
1398
|
+
var DeclaredSchema = z8.object({
|
|
1399
|
+
readOnlyHint: z8.boolean().optional(),
|
|
1400
|
+
destructiveHint: z8.boolean().optional(),
|
|
1401
|
+
idempotentHint: z8.boolean().optional(),
|
|
1402
|
+
openWorldHint: z8.boolean().optional()
|
|
1403
|
+
});
|
|
1404
|
+
var STATE_SURFACES = [
|
|
1405
|
+
"container_fs",
|
|
1406
|
+
"server_reads",
|
|
1407
|
+
"process_table",
|
|
1408
|
+
"container_diff"
|
|
1409
|
+
];
|
|
1410
|
+
var StateSurfaceSchema = z8.enum(STATE_SURFACES);
|
|
1411
|
+
var ObservedSchema = z8.object({
|
|
1412
|
+
/** Undetermined is a third answer, and is never collapsed into `false`. */
|
|
1413
|
+
mutatesState: z8.enum(["yes", "no", "undetermined"]),
|
|
1414
|
+
surfacesTouched: z8.array(StateSurfaceSchema).default([]),
|
|
1415
|
+
/** Paths created, modified or removed inside the container. */
|
|
1416
|
+
wrote: z8.array(z8.string()).default([]),
|
|
1417
|
+
sideEffects: z8.array(z8.string()).default([]),
|
|
1418
|
+
idempotent: z8.enum(["yes", "no", "not-probed"]),
|
|
1419
|
+
errorBehavior: z8.array(z8.string()).default([])
|
|
1420
|
+
});
|
|
1421
|
+
var CONFORMANCE_VERDICTS = ["CONFORMS", "CONTRADICTED", "UNDETERMINED"];
|
|
1422
|
+
var ConformanceVerdictSchema = z8.enum(CONFORMANCE_VERDICTS);
|
|
1423
|
+
var ConformanceSchema = z8.object({
|
|
1424
|
+
/** The claim being checked, as the server phrased it: `readOnlyHint: true`. */
|
|
1425
|
+
claim: z8.string(),
|
|
1426
|
+
verdict: ConformanceVerdictSchema,
|
|
1427
|
+
severity: z8.enum(FAILURE_SEVERITIES),
|
|
1428
|
+
/**
|
|
1429
|
+
* Whether contradicting this claim changes what an agent is allowed to do
|
|
1430
|
+
* without asking. `readOnlyHint` and `destructiveHint` drive automatic
|
|
1431
|
+
* permissioning in at least one major client, so a tool that declares itself
|
|
1432
|
+
* read-only and writes is a permission bypass rather than a documentation
|
|
1433
|
+
* defect, and the record says which it is.
|
|
1434
|
+
*/
|
|
1435
|
+
permissionRelevant: z8.boolean().default(false),
|
|
1436
|
+
/** One sentence a person can read without the rest of the record. */
|
|
1437
|
+
because: z8.string(),
|
|
1438
|
+
/** The surface that produced the observation behind this verdict. */
|
|
1439
|
+
surface: StateSurfaceSchema.optional(),
|
|
1440
|
+
/** The case that proves it. Every fact points at a run. */
|
|
1441
|
+
evidenceRef: z8.string().default("")
|
|
1442
|
+
});
|
|
1443
|
+
var ToolRecordSchema = z8.object({
|
|
1444
|
+
name: z8.string(),
|
|
1445
|
+
description: z8.string().default(""),
|
|
1446
|
+
declared: DeclaredSchema,
|
|
1447
|
+
observed: ObservedSchema,
|
|
1448
|
+
conformance: z8.array(ConformanceSchema).default([]),
|
|
1449
|
+
verification: z8.enum(VERIFICATION_STRENGTHS)
|
|
1450
|
+
});
|
|
1451
|
+
var UNTESTED_REASONS = [
|
|
1452
|
+
"NEEDS_FIXTURE",
|
|
1453
|
+
"NEEDS_CREDENTIAL",
|
|
1454
|
+
"UNSAFE_TO_EXERCISE",
|
|
1455
|
+
"UNDETERMINED"
|
|
1456
|
+
];
|
|
1457
|
+
var UntestedReasonSchema = z8.enum(UNTESTED_REASONS);
|
|
1458
|
+
var UntestedSchema = z8.object({
|
|
1459
|
+
tool: z8.string(),
|
|
1460
|
+
reason: UntestedReasonSchema,
|
|
1461
|
+
detail: z8.string()
|
|
1462
|
+
});
|
|
1463
|
+
var CaseRecordSchema = z8.object({
|
|
1464
|
+
id: z8.string(),
|
|
1465
|
+
tool: z8.string(),
|
|
1466
|
+
args: z8.record(z8.string(), z8.unknown()).default({}),
|
|
1467
|
+
/** How each argument was chosen, so a reader can reproduce the plan. */
|
|
1468
|
+
derivation: z8.array(z8.string()).default([]),
|
|
1469
|
+
ok: z8.boolean(),
|
|
1470
|
+
isError: z8.boolean().default(false),
|
|
1471
|
+
errorText: z8.string().default(""),
|
|
1472
|
+
beforeDigest: z8.string().default(""),
|
|
1473
|
+
afterDigest: z8.string().default(""),
|
|
1474
|
+
changedPaths: z8.array(z8.string()).default([]),
|
|
1475
|
+
durationMs: z8.number().nonnegative().default(0),
|
|
1476
|
+
stderrExcerpt: z8.string().default("")
|
|
1477
|
+
});
|
|
1478
|
+
var VerificationRecordSchema = z8.object({
|
|
1479
|
+
schema: z8.literal(VERIFICATION_RECORD_SCHEMA),
|
|
1480
|
+
schemaVersion: z8.literal(RECORD_SCHEMA_VERSION),
|
|
1481
|
+
recordId: z8.string(),
|
|
1482
|
+
target: TargetSchema,
|
|
1483
|
+
harness: HarnessSchema,
|
|
1484
|
+
server: z8.object({
|
|
1485
|
+
name: z8.string().default(""),
|
|
1486
|
+
version: z8.string().default(""),
|
|
1487
|
+
protocolVersion: z8.string().default("")
|
|
1488
|
+
}),
|
|
1489
|
+
tools: z8.array(ToolRecordSchema).default([]),
|
|
1490
|
+
/** Required. Present and empty means "everything was tested", and is a claim. */
|
|
1491
|
+
untested: z8.array(UntestedSchema),
|
|
1492
|
+
cases: z8.array(CaseRecordSchema).default([]),
|
|
1493
|
+
summary: z8.object({
|
|
1494
|
+
toolsDiscovered: z8.number().int().nonnegative(),
|
|
1495
|
+
toolsExercised: z8.number().int().nonnegative(),
|
|
1496
|
+
conforms: z8.number().int().nonnegative(),
|
|
1497
|
+
contradicted: z8.number().int().nonnegative(),
|
|
1498
|
+
undetermined: z8.number().int().nonnegative(),
|
|
1499
|
+
untested: z8.number().int().nonnegative()
|
|
1500
|
+
}),
|
|
1501
|
+
startedAt: z8.string(),
|
|
1502
|
+
finishedAt: z8.string(),
|
|
1503
|
+
/** Over the record with this field set to ''. Recomputable from the file. */
|
|
1504
|
+
recordHash: z8.string().default("")
|
|
1505
|
+
});
|
|
1506
|
+
var ROOTFUL_DAEMON_CAVEAT = "The container runtime daemon runs as root. RigorRun reduces blast radius but does not claim to prevent a container escape; an escape reaches the host.";
|
|
1507
|
+
var SERVER_READS_CAVEAT = "Where a state reading came only from the server\u2019s own tools, it is corroboration and not proof: the server that may be misdeclaring a tool is the one answering the read.";
|
|
1508
|
+
|
|
1509
|
+
// ../core/src/index.ts
|
|
1510
|
+
var RIGORRUN_VERSION = "0.2.0";
|
|
1511
|
+
|
|
1261
1512
|
// ../environment/src/schema.ts
|
|
1262
1513
|
function validateSchema(schema6) {
|
|
1263
1514
|
const problems = [];
|
|
@@ -1340,13 +1591,13 @@ function validateSchema(schema6) {
|
|
|
1340
1591
|
});
|
|
1341
1592
|
}
|
|
1342
1593
|
} else {
|
|
1343
|
-
const
|
|
1344
|
-
if (!
|
|
1594
|
+
const join15 = byName.get(relationship.via.entity);
|
|
1595
|
+
if (!join15) {
|
|
1345
1596
|
problems.push({ path, message: `unknown join entity ${relationship.via.entity}` });
|
|
1346
1597
|
} else {
|
|
1347
1598
|
for (const field of [relationship.via.fromField, relationship.via.toField]) {
|
|
1348
|
-
if (!
|
|
1349
|
-
problems.push({ path, message: `join field "${field}" is not a field of ${
|
|
1599
|
+
if (!join15.fields.some((f) => f.name === field)) {
|
|
1600
|
+
problems.push({ path, message: `join field "${field}" is not a field of ${join15.name}` });
|
|
1350
1601
|
}
|
|
1351
1602
|
}
|
|
1352
1603
|
}
|
|
@@ -1457,8 +1708,11 @@ function verificationStrength(caps) {
|
|
|
1457
1708
|
if (caps.stateRead === "designated-reads") return "PARTIAL";
|
|
1458
1709
|
return "OBSERVATIONAL";
|
|
1459
1710
|
}
|
|
1460
|
-
|
|
1461
|
-
|
|
1711
|
+
var RESET_WE_CONTROL = /* @__PURE__ */ new Set(["snapshot", "container"]);
|
|
1712
|
+
function isolationLevel(caps, observed) {
|
|
1713
|
+
if (caps.reset === "none") return "NONE";
|
|
1714
|
+
if (observed) return observed.stable ? "RESET" : observed.pathsStable ? "PARTIAL" : "NONE";
|
|
1715
|
+
return RESET_WE_CONTROL.has(caps.reset) ? "RESET" : "DECLARED";
|
|
1462
1716
|
}
|
|
1463
1717
|
function mayRepeatMutatingCases(caps) {
|
|
1464
1718
|
return caps.reset !== "none";
|
|
@@ -2074,7 +2328,7 @@ function countViaJoin(state, relationship, row, entity2) {
|
|
|
2074
2328
|
if (relationship.via.kind !== "join") return 0;
|
|
2075
2329
|
const { entity: joinEntity, fromField } = relationship.via;
|
|
2076
2330
|
const id2 = String(row[entity2.idField]);
|
|
2077
|
-
return rowsOf(state, joinEntity).filter((
|
|
2331
|
+
return rowsOf(state, joinEntity).filter((join15) => String(join15[fromField]) === id2).length;
|
|
2078
2332
|
}
|
|
2079
2333
|
function bucketise(entity2, rows6) {
|
|
2080
2334
|
const by = {};
|
|
@@ -2528,7 +2782,7 @@ function pathAgreement(context) {
|
|
|
2528
2782
|
function thresholdGuard(context) {
|
|
2529
2783
|
const rules = [];
|
|
2530
2784
|
const allQuantities = quantityPaths(context);
|
|
2531
|
-
const ownQuantities = allQuantities.filter((
|
|
2785
|
+
const ownQuantities = allQuantities.filter((entry2) => !entry2.path.includes("__"));
|
|
2532
2786
|
const quantities = ownQuantities.length > 0 ? ownQuantities : allQuantities;
|
|
2533
2787
|
if (quantities.length === 0 || context.statedNumbers.length === 0) return rules;
|
|
2534
2788
|
const relationGuards = guardCandidates(context);
|
|
@@ -3164,27 +3418,27 @@ function readStatedNumbers(trace, schema6) {
|
|
|
3164
3418
|
const qualifier = /(approv|authoris|authoriz|sign.?off|review|escalat|permit|manager|supervisor)/i;
|
|
3165
3419
|
const declared = schema6.entities.flatMap((entity2) => entity2.fields.flatMap((field) => field.enumValues ?? [])).filter((value) => value.length > 3);
|
|
3166
3420
|
const found = [];
|
|
3167
|
-
for (const
|
|
3168
|
-
const relevant = qualifier.test(
|
|
3421
|
+
for (const entry2 of sentences(trace)) {
|
|
3422
|
+
const relevant = qualifier.test(entry2.text) || declared.some((value) => entry2.text.toLowerCase().includes(value.toLowerCase()));
|
|
3169
3423
|
if (!relevant) continue;
|
|
3170
|
-
for (const match of
|
|
3424
|
+
for (const match of entry2.text.matchAll(pattern)) {
|
|
3171
3425
|
const raw = match[1];
|
|
3172
3426
|
if (!raw) continue;
|
|
3173
3427
|
const value = Number(raw.replace(/,/g, ""));
|
|
3174
3428
|
if (!Number.isFinite(value)) continue;
|
|
3175
|
-
found.push({ value, text:
|
|
3429
|
+
found.push({ value, text: entry2.text, stepId: entry2.stepId });
|
|
3176
3430
|
}
|
|
3177
3431
|
}
|
|
3178
3432
|
const seen = /* @__PURE__ */ new Set();
|
|
3179
|
-
return found.filter((
|
|
3180
|
-
if (seen.has(
|
|
3181
|
-
seen.add(
|
|
3433
|
+
return found.filter((entry2) => {
|
|
3434
|
+
if (seen.has(entry2.value)) return false;
|
|
3435
|
+
seen.add(entry2.value);
|
|
3182
3436
|
return true;
|
|
3183
3437
|
});
|
|
3184
3438
|
}
|
|
3185
3439
|
function sentences(trace) {
|
|
3186
3440
|
return surfaceText(trace).flatMap(
|
|
3187
|
-
(
|
|
3441
|
+
(entry2) => entry2.text.split(/(?<=[.!?])\s+/).map((text2) => text2.trim()).filter((text2) => text2.length > 0).map((text2) => ({ text: text2, stepId: entry2.stepId }))
|
|
3188
3442
|
);
|
|
3189
3443
|
}
|
|
3190
3444
|
function unitMatchesText(unit, text2) {
|
|
@@ -4342,7 +4596,7 @@ function resolvePath(root, path) {
|
|
|
4342
4596
|
continue;
|
|
4343
4597
|
}
|
|
4344
4598
|
if (!Array.isArray(current)) return NOT_FOUND;
|
|
4345
|
-
current = current.filter((
|
|
4599
|
+
current = current.filter((entry2) => matchesAll(entry2, segment.conditions));
|
|
4346
4600
|
afterFilter = true;
|
|
4347
4601
|
}
|
|
4348
4602
|
return { found: current !== void 0, value: current };
|
|
@@ -4393,12 +4647,12 @@ function parseLiteral(raw) {
|
|
|
4393
4647
|
if (raw !== "" && !Number.isNaN(Number(raw))) return Number(raw);
|
|
4394
4648
|
return trimmed;
|
|
4395
4649
|
}
|
|
4396
|
-
function matchesAll(
|
|
4397
|
-
if (
|
|
4398
|
-
return conditions.every((condition) => matches(
|
|
4650
|
+
function matchesAll(entry2, conditions) {
|
|
4651
|
+
if (entry2 === null || typeof entry2 !== "object") return false;
|
|
4652
|
+
return conditions.every((condition) => matches(entry2, condition));
|
|
4399
4653
|
}
|
|
4400
|
-
function matches(
|
|
4401
|
-
const actual = resolvePath(
|
|
4654
|
+
function matches(entry2, condition) {
|
|
4655
|
+
const actual = resolvePath(entry2, condition.field).value;
|
|
4402
4656
|
switch (condition.op) {
|
|
4403
4657
|
case "=":
|
|
4404
4658
|
return looseEquals(actual, condition.value);
|
|
@@ -4575,7 +4829,7 @@ function containment(kind, target, expected, observation) {
|
|
|
4575
4829
|
if (typeof value === "string" && typeof expected === "string") {
|
|
4576
4830
|
contains = value.includes(expected);
|
|
4577
4831
|
} else if (Array.isArray(value)) {
|
|
4578
|
-
contains = value.some((
|
|
4832
|
+
contains = value.some((entry2) => deepEqual2(entry2, expected));
|
|
4579
4833
|
} else if (value !== null && value !== void 0) {
|
|
4580
4834
|
contains = JSON.stringify(value).includes(String(expected));
|
|
4581
4835
|
}
|
|
@@ -4654,7 +4908,7 @@ function deepEqual2(a, b) {
|
|
|
4654
4908
|
if (typeof a !== "object") return false;
|
|
4655
4909
|
if (Array.isArray(a) !== Array.isArray(b)) return false;
|
|
4656
4910
|
if (Array.isArray(a) && Array.isArray(b)) {
|
|
4657
|
-
return a.length === b.length && a.every((
|
|
4911
|
+
return a.length === b.length && a.every((entry2, i) => deepEqual2(entry2, b[i]));
|
|
4658
4912
|
}
|
|
4659
4913
|
const aKeys = Object.keys(a).sort();
|
|
4660
4914
|
const bKeys = Object.keys(b).sort();
|
|
@@ -4910,7 +5164,7 @@ async function findUntestableRules(adapter, contract, mutations, rules) {
|
|
|
4910
5164
|
}
|
|
4911
5165
|
function markUntestable(contract, untestable) {
|
|
4912
5166
|
if (untestable.length === 0) return contract;
|
|
4913
|
-
const byId = new Map(untestable.map((
|
|
5167
|
+
const byId = new Map(untestable.map((entry2) => [entry2.ruleId, entry2.reason]));
|
|
4914
5168
|
return {
|
|
4915
5169
|
...contract,
|
|
4916
5170
|
rules: contract.rules.map((rule) => {
|
|
@@ -4975,9 +5229,9 @@ async function generateBenchmark(adapter, contract, fixtures, options = {}) {
|
|
|
4975
5229
|
}
|
|
4976
5230
|
const contractHash = await hashValue(contract);
|
|
4977
5231
|
const createdAt = options.createdAt ?? (/* @__PURE__ */ new Date()).toISOString();
|
|
4978
|
-
const cases = generated.map((
|
|
5232
|
+
const cases = generated.map((entry2) => entry2.testCase);
|
|
4979
5233
|
const notTestable = [
|
|
4980
|
-
...untestable.map((
|
|
5234
|
+
...untestable.map((entry2) => ({ rule: entry2.statement, reason: entry2.reason })),
|
|
4981
5235
|
...[...uninstallable].map(([primitive, reason]) => ({ rule: primitive, reason })),
|
|
4982
5236
|
...capabilityGaps(adapter)
|
|
4983
5237
|
];
|
|
@@ -7434,23 +7688,23 @@ function createLlmAgent(config) {
|
|
|
7434
7688
|
}
|
|
7435
7689
|
|
|
7436
7690
|
// ../agents/src/http.ts
|
|
7437
|
-
import { z as
|
|
7438
|
-
var ResponseSchema =
|
|
7439
|
-
|
|
7440
|
-
action:
|
|
7441
|
-
tool:
|
|
7442
|
-
args:
|
|
7691
|
+
import { z as z9 } from "zod";
|
|
7692
|
+
var ResponseSchema = z9.union([
|
|
7693
|
+
z9.object({
|
|
7694
|
+
action: z9.object({
|
|
7695
|
+
tool: z9.string().min(1),
|
|
7696
|
+
args: z9.record(z9.string(), z9.unknown()).default({})
|
|
7443
7697
|
}),
|
|
7444
|
-
note:
|
|
7698
|
+
note: z9.string().max(2e3).optional()
|
|
7445
7699
|
}),
|
|
7446
|
-
|
|
7447
|
-
done:
|
|
7448
|
-
report:
|
|
7449
|
-
usage:
|
|
7450
|
-
promptTokens:
|
|
7451
|
-
completionTokens:
|
|
7700
|
+
z9.object({
|
|
7701
|
+
done: z9.literal(true),
|
|
7702
|
+
report: z9.string().max(8e3),
|
|
7703
|
+
usage: z9.object({
|
|
7704
|
+
promptTokens: z9.number().int().nonnegative(),
|
|
7705
|
+
completionTokens: z9.number().int().nonnegative()
|
|
7452
7706
|
}).optional(),
|
|
7453
|
-
costUsd:
|
|
7707
|
+
costUsd: z9.number().nullable().optional()
|
|
7454
7708
|
})
|
|
7455
7709
|
]);
|
|
7456
7710
|
var MAX_RESPONSE_BYTES = 256 * 1024;
|
|
@@ -7696,13 +7950,13 @@ function resolveAgent(id2, options = {}) {
|
|
|
7696
7950
|
}
|
|
7697
7951
|
|
|
7698
7952
|
// ../scoring/src/stats.ts
|
|
7699
|
-
function wilsonInterval(successes, n,
|
|
7953
|
+
function wilsonInterval(successes, n, z12 = 1.96) {
|
|
7700
7954
|
if (n <= 0) return { point: 0, lower: 0, upper: 1, n: 0 };
|
|
7701
7955
|
const p = successes / n;
|
|
7702
|
-
const z22 =
|
|
7956
|
+
const z22 = z12 * z12;
|
|
7703
7957
|
const denominator = 1 + z22 / n;
|
|
7704
7958
|
const centre = (p + z22 / (2 * n)) / denominator;
|
|
7705
|
-
const margin =
|
|
7959
|
+
const margin = z12 / denominator * Math.sqrt(p * (1 - p) / n + z22 / (4 * n * n));
|
|
7706
7960
|
return {
|
|
7707
7961
|
point: round4(p),
|
|
7708
7962
|
lower: round4(Math.max(0, centre - margin)),
|
|
@@ -7760,10 +8014,10 @@ function scoreAgent(agent, results, thresholds) {
|
|
|
7760
8014
|
const errorCount = results.filter((r) => r.errored).length;
|
|
7761
8015
|
const byCase = /* @__PURE__ */ new Map();
|
|
7762
8016
|
for (const result of results) {
|
|
7763
|
-
const
|
|
7764
|
-
|
|
7765
|
-
if (result.taskSuccess)
|
|
7766
|
-
byCase.set(result.caseId,
|
|
8017
|
+
const entry2 = byCase.get(result.caseId) ?? { total: 0, successes: 0 };
|
|
8018
|
+
entry2.total += 1;
|
|
8019
|
+
if (result.taskSuccess) entry2.successes += 1;
|
|
8020
|
+
byCase.set(result.caseId, entry2);
|
|
7767
8021
|
}
|
|
7768
8022
|
const attempts = [...byCase.values()];
|
|
7769
8023
|
const passAt = {};
|
|
@@ -8313,8 +8567,8 @@ ${failures(data, mode)}
|
|
|
8313
8567
|
${contractSection(options.contract)}
|
|
8314
8568
|
${metadata(data, options, generatedAt, mode)}
|
|
8315
8569
|
<footer>
|
|
8316
|
-
RigorRun ${esc(data.rigorrunVersion)} \xB7 Acceptance testing for tool-using AI agents
|
|
8317
|
-
${esc(data.environment)} is a synthetic demo environment. Every record in it is fabricated
|
|
8570
|
+
RigorRun ${esc(data.rigorrunVersion)} \xB7 Acceptance testing for tool-using AI agents.${options.syntheticEnvironment ? `<br>
|
|
8571
|
+
${esc(data.environment)} is a synthetic demo environment. Every record in it is fabricated.` : ""}
|
|
8318
8572
|
</footer>
|
|
8319
8573
|
</div></body></html>`;
|
|
8320
8574
|
}
|
|
@@ -8616,7 +8870,12 @@ async function cmdDemo(flags) {
|
|
|
8616
8870
|
await writeJson(join(RUNS_DIR(), `${result.runId}.json`), result);
|
|
8617
8871
|
await writeText(
|
|
8618
8872
|
join(outDir, "report.html"),
|
|
8619
|
-
renderReportHtml(result, {
|
|
8873
|
+
renderReportHtml(result, {
|
|
8874
|
+
contract,
|
|
8875
|
+
benchmark,
|
|
8876
|
+
generatedAt: result.finishedAt,
|
|
8877
|
+
syntheticEnvironment: isBundledEnvironment(result.environment)
|
|
8878
|
+
})
|
|
8620
8879
|
);
|
|
8621
8880
|
line();
|
|
8622
8881
|
printComparison(result);
|
|
@@ -8757,10 +9016,10 @@ async function cmdGenerate(contractPath, flags) {
|
|
|
8757
9016
|
);
|
|
8758
9017
|
table(
|
|
8759
9018
|
["case", "category", "expected"],
|
|
8760
|
-
generation.cases.map((
|
|
8761
|
-
|
|
8762
|
-
|
|
8763
|
-
|
|
9019
|
+
generation.cases.map((entry2) => [
|
|
9020
|
+
entry2.testCase.name,
|
|
9021
|
+
entry2.testCase.category,
|
|
9022
|
+
entry2.expected.shouldPerform ? "do the work" : `refuse \u2014 ${entry2.expected.refusalReason}`
|
|
8764
9023
|
])
|
|
8765
9024
|
);
|
|
8766
9025
|
for (const conflict of generation.conflicts) {
|
|
@@ -8772,7 +9031,14 @@ async function cmdGenerate(contractPath, flags) {
|
|
|
8772
9031
|
}
|
|
8773
9032
|
async function cmdRun(benchmarkPath, flags) {
|
|
8774
9033
|
const benchmark = await loadBenchmark(benchmarkPath);
|
|
8775
|
-
|
|
9034
|
+
if (flags.agent.length === 0) {
|
|
9035
|
+
throw new CliError(
|
|
9036
|
+
`run needs at least one --agent.
|
|
9037
|
+
--agent naive --agent careful the two bundled examples
|
|
9038
|
+
--agent ${REFERENCE_AGENT_ID} checks the suite is satisfiable, not that an agent is good`
|
|
9039
|
+
);
|
|
9040
|
+
}
|
|
9041
|
+
const agents = agentsFor(benchmark, flags, []);
|
|
8776
9042
|
const result = await executeRun(benchmark, agents, flags);
|
|
8777
9043
|
await writeJson(join(".rigorrun", "runs", `${result.runId}.json`), result);
|
|
8778
9044
|
if (flags.json) {
|
|
@@ -8783,7 +9049,13 @@ async function cmdRun(benchmarkPath, flags) {
|
|
|
8783
9049
|
line(`${c.grey("run")} .rigorrun/runs/${result.runId}.json`);
|
|
8784
9050
|
}
|
|
8785
9051
|
if (flags.report) {
|
|
8786
|
-
const written = await writeText(
|
|
9052
|
+
const written = await writeText(
|
|
9053
|
+
flags.report,
|
|
9054
|
+
renderReportHtml(result, {
|
|
9055
|
+
benchmark,
|
|
9056
|
+
syntheticEnvironment: isBundledEnvironment(result.environment)
|
|
9057
|
+
})
|
|
9058
|
+
);
|
|
8787
9059
|
if (!flags.json) line(`${c.grey("report")} ${written}`);
|
|
8788
9060
|
}
|
|
8789
9061
|
return result.scores.every((s) => s.thresholdsPassed) ? 0 : 1;
|
|
@@ -8793,6 +9065,16 @@ async function cmdGate(benchmarkPath, flags) {
|
|
|
8793
9065
|
if (flags.agent.length !== 1) {
|
|
8794
9066
|
throw new CliError("gate needs exactly one --agent.");
|
|
8795
9067
|
}
|
|
9068
|
+
if (flags.agent[0] === REFERENCE_AGENT_ID && !flags.allowReference) {
|
|
9069
|
+
throw new CliError(
|
|
9070
|
+
`gate refuses --agent ${REFERENCE_AGENT_ID}: it is handed the answer, so the gate passes by
|
|
9071
|
+
construction and tells you nothing about an agent.
|
|
9072
|
+
|
|
9073
|
+
rigorrun run <benchmark> --agent ${REFERENCE_AGENT_ID} is the suite satisfiable?
|
|
9074
|
+
rigorrun gate <benchmark> --agent ${REFERENCE_AGENT_ID} --allow-reference
|
|
9075
|
+
same question, as a gate`
|
|
9076
|
+
);
|
|
9077
|
+
}
|
|
8796
9078
|
const agent = agentsFor(benchmark, flags, [])[0];
|
|
8797
9079
|
const gated = {
|
|
8798
9080
|
...benchmark,
|
|
@@ -8835,7 +9117,13 @@ async function cmdGate(benchmarkPath, flags) {
|
|
|
8835
9117
|
for (const failure of score.failedThresholds) line(` ${c.red("x")} ${failure}`);
|
|
8836
9118
|
}
|
|
8837
9119
|
if (flags.report) {
|
|
8838
|
-
await writeText(
|
|
9120
|
+
await writeText(
|
|
9121
|
+
flags.report,
|
|
9122
|
+
renderReportHtml(result, {
|
|
9123
|
+
benchmark: gated,
|
|
9124
|
+
syntheticEnvironment: isBundledEnvironment(result.environment)
|
|
9125
|
+
})
|
|
9126
|
+
);
|
|
8839
9127
|
}
|
|
8840
9128
|
return score.thresholdsPassed ? 0 : 1;
|
|
8841
9129
|
}
|
|
@@ -8844,7 +9132,8 @@ async function cmdReport(target, flags) {
|
|
|
8844
9132
|
const path = target.endsWith(".json") ? target : join(".rigorrun", "runs", `${target}.json`);
|
|
8845
9133
|
const run = await readJson(path);
|
|
8846
9134
|
const html = renderReportHtml(flags.published ? sanitizeRunResult(run) : run, {
|
|
8847
|
-
mode: flags.published ? "published" : "full"
|
|
9135
|
+
mode: flags.published ? "published" : "full",
|
|
9136
|
+
syntheticEnvironment: isBundledEnvironment(run.environment)
|
|
8848
9137
|
});
|
|
8849
9138
|
const out = flags.out ?? join(".rigorrun", `${run.runId}${flags.published ? ".published" : ""}.html`);
|
|
8850
9139
|
const written = await writeText(out, html);
|
|
@@ -8950,6 +9239,9 @@ function agentsFor(benchmark, flags, fallback) {
|
|
|
8950
9239
|
(id2) => id2 === REFERENCE_AGENT_ID ? createReferenceAgent(benchmark) : resolveAgentOrFail(id2)
|
|
8951
9240
|
);
|
|
8952
9241
|
}
|
|
9242
|
+
function isBundledEnvironment(environmentId) {
|
|
9243
|
+
return WORKFLOWS.some((workflow) => workflow.registration.id === environmentId);
|
|
9244
|
+
}
|
|
8953
9245
|
function resolveAgentOrFail(id2) {
|
|
8954
9246
|
try {
|
|
8955
9247
|
return resolveAgent(id2, { env: envFromProcess(process.env) });
|
|
@@ -9007,7 +9299,7 @@ async function cmdPrivacyInspect(action, target, json) {
|
|
|
9007
9299
|
line(c.bold("Captured"));
|
|
9008
9300
|
table(
|
|
9009
9301
|
["field", "times"],
|
|
9010
|
-
report3.capturedFields.map((
|
|
9302
|
+
report3.capturedFields.map((entry2) => [entry2.field, String(entry2.count)])
|
|
9011
9303
|
);
|
|
9012
9304
|
line();
|
|
9013
9305
|
line(`${c.bold("Also captured")} ${surfaceText2} text fragments the operator could see, ${uiFields} interface locators`);
|
|
@@ -9105,8 +9397,8 @@ function summarise2(events, project) {
|
|
|
9105
9397
|
for (const event of mine) {
|
|
9106
9398
|
if (event.attempt) attempts[event.attempt] = (attempts[event.attempt] ?? 0) + 1;
|
|
9107
9399
|
}
|
|
9108
|
-
const created = stages.find((
|
|
9109
|
-
const verdict2 = stages.find((
|
|
9400
|
+
const created = stages.find((entry2) => entry2.stage === "project_created");
|
|
9401
|
+
const verdict2 = stages.find((entry2) => entry2.stage === "first_real_verdict");
|
|
9110
9402
|
const furthest = [...stages].sort(
|
|
9111
9403
|
(a, b) => ACTIVATION_STAGES.indexOf(b.stage) - ACTIVATION_STAGES.indexOf(a.stage)
|
|
9112
9404
|
)[0];
|
|
@@ -9169,9 +9461,9 @@ async function sweepTemporaries(dir) {
|
|
|
9169
9461
|
return 0;
|
|
9170
9462
|
}
|
|
9171
9463
|
let swept = 0;
|
|
9172
|
-
for (const
|
|
9173
|
-
if (!/^\..*\.tmp-[0-9a-f]{12}$/.test(
|
|
9174
|
-
await rm(join3(dir,
|
|
9464
|
+
for (const entry2 of entries) {
|
|
9465
|
+
if (!/^\..*\.tmp-[0-9a-f]{12}$/.test(entry2)) continue;
|
|
9466
|
+
await rm(join3(dir, entry2), { force: true });
|
|
9175
9467
|
swept += 1;
|
|
9176
9468
|
}
|
|
9177
9469
|
return swept;
|
|
@@ -9223,7 +9515,7 @@ async function openWorkspace(root, version) {
|
|
|
9223
9515
|
return { meta: meta2, created: true, applied: [] };
|
|
9224
9516
|
}
|
|
9225
9517
|
if (existing.version > WORKSPACE_VERSION) throw new WorkspaceTooNewError(existing.version);
|
|
9226
|
-
const pending = MIGRATIONS.filter((
|
|
9518
|
+
const pending = MIGRATIONS.filter((entry2) => entry2.to > existing.version).sort(
|
|
9227
9519
|
(a, b) => a.to - b.to
|
|
9228
9520
|
);
|
|
9229
9521
|
let backup = existing.lastBackup;
|
|
@@ -9494,17 +9786,17 @@ function describe(value) {
|
|
|
9494
9786
|
}
|
|
9495
9787
|
|
|
9496
9788
|
// ../daemon/src/project.ts
|
|
9497
|
-
import { z as
|
|
9789
|
+
import { z as z10 } from "zod";
|
|
9498
9790
|
var PROJECT_SCHEMA_VERSION = 1;
|
|
9499
|
-
var secretNames =
|
|
9500
|
-
var McpConnectorSchema =
|
|
9501
|
-
kind:
|
|
9502
|
-
transport:
|
|
9791
|
+
var secretNames = z10.array(z10.string()).default([]);
|
|
9792
|
+
var McpConnectorSchema = z10.object({
|
|
9793
|
+
kind: z10.literal("mcp"),
|
|
9794
|
+
transport: z10.enum(["stdio", "http"]),
|
|
9503
9795
|
/** For stdio. The binary, never a shell string. */
|
|
9504
|
-
command:
|
|
9505
|
-
args:
|
|
9796
|
+
command: z10.string().default(""),
|
|
9797
|
+
args: z10.array(z10.string()).default([]),
|
|
9506
9798
|
/** For http. */
|
|
9507
|
-
url:
|
|
9799
|
+
url: z10.string().default(""),
|
|
9508
9800
|
/**
|
|
9509
9801
|
* How an HTTP server is authenticated to.
|
|
9510
9802
|
*
|
|
@@ -9514,19 +9806,19 @@ var McpConnectorSchema = z9.object({
|
|
|
9514
9806
|
* approves in a browser, and the tokens live with the other credentials.
|
|
9515
9807
|
* Ignored for stdio, which authenticates by environment.
|
|
9516
9808
|
*/
|
|
9517
|
-
auth:
|
|
9809
|
+
auth: z10.enum(["header", "oauth"]).default("header"),
|
|
9518
9810
|
secretNames
|
|
9519
9811
|
});
|
|
9520
|
-
var OpenApiConnectorSchema =
|
|
9521
|
-
kind:
|
|
9812
|
+
var OpenApiConnectorSchema = z10.object({
|
|
9813
|
+
kind: z10.literal("openapi"),
|
|
9522
9814
|
/** The document itself, kept whole so a run does not depend on a URL. */
|
|
9523
|
-
spec:
|
|
9815
|
+
spec: z10.string().max(8e6).default(""),
|
|
9524
9816
|
/** Where it was fetched from, for reconnecting and for saying so. */
|
|
9525
|
-
specUrl:
|
|
9817
|
+
specUrl: z10.string().default(""),
|
|
9526
9818
|
/** Where the API actually is. Whatever `servers` claims is a suggestion. */
|
|
9527
|
-
baseUrl:
|
|
9819
|
+
baseUrl: z10.string().default(""),
|
|
9528
9820
|
/** Header name to secret name, so a token is never in the project. */
|
|
9529
|
-
headers:
|
|
9821
|
+
headers: z10.record(z10.string(), z10.string()).default({}),
|
|
9530
9822
|
/**
|
|
9531
9823
|
* A client id and secret to exchange for a token, when the API wants one.
|
|
9532
9824
|
*
|
|
@@ -9538,20 +9830,20 @@ var OpenApiConnectorSchema = z9.object({
|
|
|
9538
9830
|
* an API whose only flow is a browser is one RigorRun should say it cannot
|
|
9539
9831
|
* sign in to rather than half-attempt.
|
|
9540
9832
|
*/
|
|
9541
|
-
oauth:
|
|
9542
|
-
tokenUrl:
|
|
9543
|
-
clientIdSecret:
|
|
9544
|
-
clientSecretSecret:
|
|
9545
|
-
scope:
|
|
9833
|
+
oauth: z10.object({
|
|
9834
|
+
tokenUrl: z10.string(),
|
|
9835
|
+
clientIdSecret: z10.string(),
|
|
9836
|
+
clientSecretSecret: z10.string(),
|
|
9837
|
+
scope: z10.string().default("")
|
|
9546
9838
|
}).nullable().default(null),
|
|
9547
9839
|
secretNames
|
|
9548
9840
|
});
|
|
9549
|
-
var BrowserConnectorSchema =
|
|
9550
|
-
kind:
|
|
9841
|
+
var BrowserConnectorSchema = z10.object({
|
|
9842
|
+
kind: z10.literal("browser"),
|
|
9551
9843
|
/** Where the job starts. Every navigation is checked against its origin. */
|
|
9552
|
-
startUrl:
|
|
9553
|
-
browser:
|
|
9554
|
-
headless:
|
|
9844
|
+
startUrl: z10.string(),
|
|
9845
|
+
browser: z10.enum(["chromium", "firefox", "webkit"]).default("chromium"),
|
|
9846
|
+
headless: z10.boolean().default(true),
|
|
9555
9847
|
/**
|
|
9556
9848
|
* Something that can be read for records, or nothing.
|
|
9557
9849
|
*
|
|
@@ -9561,10 +9853,10 @@ var BrowserConnectorSchema = z9.object({
|
|
|
9561
9853
|
* project says OBSERVATIONAL — RigorRun watched what the agent did and did
|
|
9562
9854
|
* not check what changed.
|
|
9563
9855
|
*/
|
|
9564
|
-
verifier:
|
|
9856
|
+
verifier: z10.discriminatedUnion("kind", [McpConnectorSchema, OpenApiConnectorSchema]).nullable().default(null),
|
|
9565
9857
|
secretNames
|
|
9566
9858
|
});
|
|
9567
|
-
var ConnectorSchema =
|
|
9859
|
+
var ConnectorSchema = z10.discriminatedUnion("kind", [
|
|
9568
9860
|
McpConnectorSchema,
|
|
9569
9861
|
OpenApiConnectorSchema,
|
|
9570
9862
|
BrowserConnectorSchema
|
|
@@ -9599,30 +9891,30 @@ function describeConnectorAction(connector) {
|
|
|
9599
9891
|
}
|
|
9600
9892
|
return connector.auth === "oauth" ? `open ${connector.url}, and sign in to it in a browser if it asks` : `open ${connector.url}`;
|
|
9601
9893
|
}
|
|
9602
|
-
var VerifierReadSchema =
|
|
9603
|
-
tool:
|
|
9604
|
-
args:
|
|
9894
|
+
var VerifierReadSchema = z10.object({
|
|
9895
|
+
tool: z10.string(),
|
|
9896
|
+
args: z10.record(z10.string(), z10.unknown()).default({})
|
|
9605
9897
|
});
|
|
9606
9898
|
var agentCommon = {
|
|
9607
|
-
id:
|
|
9608
|
-
name:
|
|
9899
|
+
id: z10.string(),
|
|
9900
|
+
name: z10.string(),
|
|
9609
9901
|
/** When the last successful handshake happened, and what it said. */
|
|
9610
|
-
lastProbeAt:
|
|
9611
|
-
lastProbeOk:
|
|
9612
|
-
lastProbeProblem:
|
|
9902
|
+
lastProbeAt: z10.string().nullable().default(null),
|
|
9903
|
+
lastProbeOk: z10.boolean().default(false),
|
|
9904
|
+
lastProbeProblem: z10.string().default("")
|
|
9613
9905
|
};
|
|
9614
|
-
var HttpAgentSchema =
|
|
9906
|
+
var HttpAgentSchema = z10.object({
|
|
9615
9907
|
...agentCommon,
|
|
9616
|
-
kind:
|
|
9617
|
-
endpoint:
|
|
9908
|
+
kind: z10.literal("http"),
|
|
9909
|
+
endpoint: z10.string()
|
|
9618
9910
|
});
|
|
9619
|
-
var ProcessAgentSchema =
|
|
9911
|
+
var ProcessAgentSchema = z10.object({
|
|
9620
9912
|
...agentCommon,
|
|
9621
|
-
kind:
|
|
9913
|
+
kind: z10.literal("process"),
|
|
9622
9914
|
/** The executable. Written here only by a person, at this machine. */
|
|
9623
|
-
command:
|
|
9624
|
-
args:
|
|
9625
|
-
cwd:
|
|
9915
|
+
command: z10.string(),
|
|
9916
|
+
args: z10.array(z10.string()).default([]),
|
|
9917
|
+
cwd: z10.string().default(""),
|
|
9626
9918
|
/**
|
|
9627
9919
|
* When somebody looked at that command and said yes.
|
|
9628
9920
|
*
|
|
@@ -9630,11 +9922,11 @@ var ProcessAgentSchema = z9.object({
|
|
|
9630
9922
|
* `connectorTrust`. A command that arrived in a file is not a command
|
|
9631
9923
|
* anybody on this machine has agreed to run.
|
|
9632
9924
|
*/
|
|
9633
|
-
confirmedByOperatorAt:
|
|
9925
|
+
confirmedByOperatorAt: z10.string().nullable().default(null)
|
|
9634
9926
|
});
|
|
9635
|
-
var ExternalAgentSchema =
|
|
9927
|
+
var ExternalAgentSchema = z10.object({
|
|
9636
9928
|
...agentCommon,
|
|
9637
|
-
kind:
|
|
9929
|
+
kind: z10.literal("external"),
|
|
9638
9930
|
/**
|
|
9639
9931
|
* The name of the credential holding this agent's key.
|
|
9640
9932
|
*
|
|
@@ -9643,9 +9935,9 @@ var ExternalAgentSchema = z9.object({
|
|
|
9643
9935
|
* file that cannot be copied safely would undo the arrangement the rest of
|
|
9644
9936
|
* this file is built on.
|
|
9645
9937
|
*/
|
|
9646
|
-
keyName:
|
|
9938
|
+
keyName: z10.string()
|
|
9647
9939
|
});
|
|
9648
|
-
var AgentConfigSchema =
|
|
9940
|
+
var AgentConfigSchema = z10.discriminatedUnion("kind", [
|
|
9649
9941
|
HttpAgentSchema,
|
|
9650
9942
|
ProcessAgentSchema,
|
|
9651
9943
|
ExternalAgentSchema
|
|
@@ -9655,48 +9947,48 @@ function describeAgent(agent) {
|
|
|
9655
9947
|
if (agent.kind === "external") return `driven by you (${agent.id})`;
|
|
9656
9948
|
return `${agent.command} ${agent.args.join(" ")}`.trim();
|
|
9657
9949
|
}
|
|
9658
|
-
var RunSummarySchema =
|
|
9659
|
-
runId:
|
|
9660
|
-
agentId:
|
|
9661
|
-
agentName:
|
|
9662
|
-
finishedAt:
|
|
9663
|
-
taskSuccessRate:
|
|
9664
|
-
policyComplianceRate:
|
|
9665
|
-
unsafeActions:
|
|
9666
|
-
criticalFailures:
|
|
9667
|
-
thresholdsPassed:
|
|
9668
|
-
verification:
|
|
9669
|
-
isolation:
|
|
9670
|
-
caseCount:
|
|
9950
|
+
var RunSummarySchema = z10.object({
|
|
9951
|
+
runId: z10.string(),
|
|
9952
|
+
agentId: z10.string(),
|
|
9953
|
+
agentName: z10.string(),
|
|
9954
|
+
finishedAt: z10.string(),
|
|
9955
|
+
taskSuccessRate: z10.number(),
|
|
9956
|
+
policyComplianceRate: z10.number(),
|
|
9957
|
+
unsafeActions: z10.number(),
|
|
9958
|
+
criticalFailures: z10.number(),
|
|
9959
|
+
thresholdsPassed: z10.boolean(),
|
|
9960
|
+
verification: z10.string(),
|
|
9961
|
+
isolation: z10.string(),
|
|
9962
|
+
caseCount: z10.number()
|
|
9671
9963
|
});
|
|
9672
|
-
var TimingsSchema =
|
|
9673
|
-
createdAt:
|
|
9674
|
-
environmentConnectedAt:
|
|
9675
|
-
workflowRecordedAt:
|
|
9676
|
-
benchmarkGeneratedAt:
|
|
9677
|
-
agentConnectedAt:
|
|
9678
|
-
firstVerdictAt:
|
|
9964
|
+
var TimingsSchema = z10.object({
|
|
9965
|
+
createdAt: z10.string(),
|
|
9966
|
+
environmentConnectedAt: z10.string().nullable().default(null),
|
|
9967
|
+
workflowRecordedAt: z10.string().nullable().default(null),
|
|
9968
|
+
benchmarkGeneratedAt: z10.string().nullable().default(null),
|
|
9969
|
+
agentConnectedAt: z10.string().nullable().default(null),
|
|
9970
|
+
firstVerdictAt: z10.string().nullable().default(null)
|
|
9679
9971
|
});
|
|
9680
|
-
var ProjectSchema =
|
|
9681
|
-
schemaVersion:
|
|
9682
|
-
id:
|
|
9683
|
-
name:
|
|
9972
|
+
var ProjectSchema = z10.object({
|
|
9973
|
+
schemaVersion: z10.literal(PROJECT_SCHEMA_VERSION),
|
|
9974
|
+
id: z10.string(),
|
|
9975
|
+
name: z10.string(),
|
|
9684
9976
|
/** What the person is trying to prove an agent can do. */
|
|
9685
|
-
goal:
|
|
9977
|
+
goal: z10.string().default(""),
|
|
9686
9978
|
connector: ConnectorSchema.nullable().default(null),
|
|
9687
9979
|
/** production / staging / local / ephemeral. Decides what may be written. */
|
|
9688
|
-
safety:
|
|
9980
|
+
safety: z10.enum(["production", "staging", "local", "ephemeral"]).default("staging"),
|
|
9689
9981
|
/** Tools a person has confirmed only read. Never the server's own opinion. */
|
|
9690
|
-
readOnlyTools:
|
|
9691
|
-
verifierReads:
|
|
9692
|
-
reset:
|
|
9982
|
+
readOnlyTools: z10.array(z10.string()).default([]),
|
|
9983
|
+
verifierReads: z10.array(VerifierReadSchema).default([]),
|
|
9984
|
+
reset: z10.object({ kind: z10.enum(["tool", "none"]), tool: z10.string().default("") }).default({ kind: "none", tool: "" }),
|
|
9693
9985
|
/** Answers to the schema questions, so a review survives a restart. */
|
|
9694
|
-
schemaAnswers:
|
|
9695
|
-
agents:
|
|
9696
|
-
runs:
|
|
9986
|
+
schemaAnswers: z10.array(z10.object({ questionId: z10.string(), value: z10.string() })).default([]),
|
|
9987
|
+
agents: z10.array(AgentConfigSchema).default([]),
|
|
9988
|
+
runs: z10.array(RunSummarySchema).default([]),
|
|
9697
9989
|
timings: TimingsSchema,
|
|
9698
9990
|
/** The run a later run is compared against, once somebody sets one. */
|
|
9699
|
-
baselineRunId:
|
|
9991
|
+
baselineRunId: z10.string().nullable().default(null),
|
|
9700
9992
|
/**
|
|
9701
9993
|
* Where this project's connector came from, and whether anybody has looked.
|
|
9702
9994
|
*
|
|
@@ -9710,9 +10002,9 @@ var ProjectSchema = z9.object({
|
|
|
9710
10002
|
* typed by whoever owns the machine it is on, which is exactly what `typed`
|
|
9711
10003
|
* means.
|
|
9712
10004
|
*/
|
|
9713
|
-
connectorTrust:
|
|
9714
|
-
origin:
|
|
9715
|
-
confirmedAt:
|
|
10005
|
+
connectorTrust: z10.object({
|
|
10006
|
+
origin: z10.enum(["typed", "imported"]).default("typed"),
|
|
10007
|
+
confirmedAt: z10.string().nullable().default(null)
|
|
9716
10008
|
}).default({ origin: "typed", confirmedAt: null })
|
|
9717
10009
|
});
|
|
9718
10010
|
function parseProject(input) {
|
|
@@ -9777,7 +10069,7 @@ function nextSteps(project) {
|
|
|
9777
10069
|
import { readFile as readFile4 } from "node:fs/promises";
|
|
9778
10070
|
import { join as join5 } from "node:path";
|
|
9779
10071
|
|
|
9780
|
-
// ../
|
|
10072
|
+
// ../exec/src/exec.ts
|
|
9781
10073
|
import { spawn } from "node:child_process";
|
|
9782
10074
|
var MAX_OUTPUT_BYTES = 1024 * 1024;
|
|
9783
10075
|
var GRACE_MS = 5e3;
|
|
@@ -10407,7 +10699,7 @@ function compareRuns(baseline, current) {
|
|
|
10407
10699
|
continue;
|
|
10408
10700
|
}
|
|
10409
10701
|
const { movement, detail } = movementOf(before, after);
|
|
10410
|
-
const
|
|
10702
|
+
const entry2 = {
|
|
10411
10703
|
caseId: after.caseId,
|
|
10412
10704
|
caseName: after.caseName,
|
|
10413
10705
|
movement,
|
|
@@ -10415,9 +10707,9 @@ function compareRuns(baseline, current) {
|
|
|
10415
10707
|
before: outcome(before),
|
|
10416
10708
|
after: outcome(after)
|
|
10417
10709
|
};
|
|
10418
|
-
if (movement === "regressed") regressed.push(
|
|
10419
|
-
else if (movement === "improved") improved.push(
|
|
10420
|
-
else unchanged.push(
|
|
10710
|
+
if (movement === "regressed") regressed.push(entry2);
|
|
10711
|
+
else if (movement === "improved") improved.push(entry2);
|
|
10712
|
+
else unchanged.push(entry2);
|
|
10421
10713
|
}
|
|
10422
10714
|
for (const before of baseline.caseResults) {
|
|
10423
10715
|
if (seen.has(before.caseId)) continue;
|
|
@@ -10467,7 +10759,7 @@ function isObject(value) {
|
|
|
10467
10759
|
function declaredTypes(node) {
|
|
10468
10760
|
const raw = node.type;
|
|
10469
10761
|
if (typeof raw === "string") return [raw];
|
|
10470
|
-
if (Array.isArray(raw)) return raw.filter((
|
|
10762
|
+
if (Array.isArray(raw)) return raw.filter((entry2) => typeof entry2 === "string");
|
|
10471
10763
|
return [];
|
|
10472
10764
|
}
|
|
10473
10765
|
function closedValues(node) {
|
|
@@ -10485,14 +10777,14 @@ function resolveType(node, depth, budget) {
|
|
|
10485
10777
|
if (budget.nodes <= 0) return void 0;
|
|
10486
10778
|
const values = closedValues(node);
|
|
10487
10779
|
if (values) return { type: "enum", enumValues: values };
|
|
10488
|
-
const branches = [node.anyOf, node.oneOf, node.allOf].find((
|
|
10780
|
+
const branches = [node.anyOf, node.oneOf, node.allOf].find((entry2) => Array.isArray(entry2));
|
|
10489
10781
|
if (Array.isArray(branches)) {
|
|
10490
|
-
const resolved = branches.filter(isObject).filter((branch) => declaredTypes(branch)[0] !== "null").map((branch) => resolveType(branch, depth + 1, budget)).filter((
|
|
10782
|
+
const resolved = branches.filter(isObject).filter((branch) => declaredTypes(branch)[0] !== "null").map((branch) => resolveType(branch, depth + 1, budget)).filter((entry2) => entry2 !== void 0);
|
|
10491
10783
|
if (resolved.length === 0) return void 0;
|
|
10492
10784
|
const first = resolved[0];
|
|
10493
|
-
return resolved.every((
|
|
10785
|
+
return resolved.every((entry2) => entry2.type === first.type) ? first : void 0;
|
|
10494
10786
|
}
|
|
10495
|
-
const types = declaredTypes(node).filter((
|
|
10787
|
+
const types = declaredTypes(node).filter((entry2) => entry2 !== "null");
|
|
10496
10788
|
if (types.length !== 1) return void 0;
|
|
10497
10789
|
switch (types[0]) {
|
|
10498
10790
|
case "string":
|
|
@@ -10520,7 +10812,7 @@ function paramsFromInputSchema(schema6) {
|
|
|
10520
10812
|
if (!properties) {
|
|
10521
10813
|
return { params, unsupported, truncated: false };
|
|
10522
10814
|
}
|
|
10523
|
-
const requiredList = Array.isArray(schema6["required"]) ? new Set(schema6["required"].filter((
|
|
10815
|
+
const requiredList = Array.isArray(schema6["required"]) ? new Set(schema6["required"].filter((entry2) => typeof entry2 === "string")) : /* @__PURE__ */ new Set();
|
|
10524
10816
|
const budget = { nodes: SCHEMA_LIMITS.maxNodes };
|
|
10525
10817
|
const names = Object.keys(properties);
|
|
10526
10818
|
if (names.length > SCHEMA_LIMITS.maxProperties) truncated = true;
|
|
@@ -10551,7 +10843,7 @@ function paramsFromInputSchema(schema6) {
|
|
|
10551
10843
|
return { params, unsupported, truncated };
|
|
10552
10844
|
}
|
|
10553
10845
|
function unsupportedReason(node) {
|
|
10554
|
-
const types = declaredTypes(node).filter((
|
|
10846
|
+
const types = declaredTypes(node).filter((entry2) => entry2 !== "null");
|
|
10555
10847
|
if (types.length === 0) return "no type is declared, so RigorRun cannot assert on it";
|
|
10556
10848
|
if (types.length > 1) return `it may be ${types.join(" or ")}, and RigorRun needs one type`;
|
|
10557
10849
|
const type = types[0];
|
|
@@ -10669,7 +10961,7 @@ function rowsFromPayload(payload, entity2) {
|
|
|
10669
10961
|
const walk = (value, depth) => {
|
|
10670
10962
|
if (depth > MAX_DEPTH || value === null || typeof value !== "object") return;
|
|
10671
10963
|
if (Array.isArray(value)) {
|
|
10672
|
-
for (const
|
|
10964
|
+
for (const entry2 of value) walk(entry2, depth + 1);
|
|
10673
10965
|
return;
|
|
10674
10966
|
}
|
|
10675
10967
|
if (looksLike(value, entity2)) {
|
|
@@ -10894,7 +11186,7 @@ import { Client } from "@modelcontextprotocol/sdk/client/index.js";
|
|
|
10894
11186
|
import { StdioClientTransport, getDefaultEnvironment } from "@modelcontextprotocol/sdk/client/stdio.js";
|
|
10895
11187
|
import { StreamableHTTPClientTransport } from "@modelcontextprotocol/sdk/client/streamableHttp.js";
|
|
10896
11188
|
import { UnauthorizedError } from "@modelcontextprotocol/sdk/client/auth.js";
|
|
10897
|
-
var CLIENT_INFO = { name: "rigorrun", version:
|
|
11189
|
+
var CLIENT_INFO = { name: "rigorrun", version: RIGORRUN_VERSION };
|
|
10898
11190
|
var DEFAULT_TIMEOUT_MS = 2e4;
|
|
10899
11191
|
function signInFor(config) {
|
|
10900
11192
|
return config.transport === "http" ? config.auth : void 0;
|
|
@@ -11044,6 +11336,7 @@ async function discoverTools(client, timeout) {
|
|
|
11044
11336
|
params: converted.params,
|
|
11045
11337
|
unsupported: converted.unsupported,
|
|
11046
11338
|
schemaTruncated: converted.truncated,
|
|
11339
|
+
...tool.inputSchema ? { inputSchema: tool.inputSchema } : {},
|
|
11047
11340
|
...tool.outputSchema ? { outputSchema: tool.outputSchema } : {},
|
|
11048
11341
|
hints,
|
|
11049
11342
|
risk: assessFromHints(hints)
|
|
@@ -11061,7 +11354,7 @@ import { randomBytes as randomBytes2 } from "node:crypto";
|
|
|
11061
11354
|
var DEFAULT_TIMEOUT_MS2 = 5 * 6e4;
|
|
11062
11355
|
var CLIENT_METADATA = {
|
|
11063
11356
|
client_name: "RigorRun",
|
|
11064
|
-
client_uri: "https://rigorrun.
|
|
11357
|
+
client_uri: "https://rigorrun.xyz",
|
|
11065
11358
|
redirect_uris: [],
|
|
11066
11359
|
grant_types: ["authorization_code", "refresh_token"],
|
|
11067
11360
|
response_types: ["code"],
|
|
@@ -11221,7 +11514,7 @@ var MAX_ENUM_VALUE_LENGTH = 64;
|
|
|
11221
11514
|
function isRecordLike(value) {
|
|
11222
11515
|
if (typeof value !== "object" || value === null || Array.isArray(value)) return false;
|
|
11223
11516
|
const scalars = Object.values(value).filter(
|
|
11224
|
-
(
|
|
11517
|
+
(entry2) => entry2 === null || typeof entry2 !== "object"
|
|
11225
11518
|
);
|
|
11226
11519
|
return scalars.length >= MIN_FIELDS_FOR_RECORD;
|
|
11227
11520
|
}
|
|
@@ -11253,16 +11546,16 @@ function collect(payload, containerName, depth, into) {
|
|
|
11253
11546
|
if (depth > MAX_WALK_DEPTH) return;
|
|
11254
11547
|
if (Array.isArray(payload)) {
|
|
11255
11548
|
const group = /* @__PURE__ */ new Map();
|
|
11256
|
-
for (const
|
|
11257
|
-
if (isRecordLike(
|
|
11258
|
-
const evidence = evidenceFor(
|
|
11259
|
-
evidence.rows.push(
|
|
11549
|
+
for (const entry2 of payload) {
|
|
11550
|
+
if (isRecordLike(entry2)) {
|
|
11551
|
+
const evidence = evidenceFor(entry2, containerName, into);
|
|
11552
|
+
evidence.rows.push(entry2);
|
|
11260
11553
|
const indices = group.get(evidence) ?? [];
|
|
11261
11554
|
indices.push(evidence.rows.length - 1);
|
|
11262
11555
|
group.set(evidence, indices);
|
|
11263
11556
|
}
|
|
11264
|
-
if (typeof
|
|
11265
|
-
collect(
|
|
11557
|
+
if (typeof entry2 === "object" && entry2 !== null) {
|
|
11558
|
+
collect(entry2, containerName, depth + 1, into);
|
|
11266
11559
|
}
|
|
11267
11560
|
}
|
|
11268
11561
|
for (const [evidence, indices] of group) evidence.collections.push(indices);
|
|
@@ -11730,7 +12023,7 @@ function resolveSchema(document, node, depth = 0, budget = { nodes: MAX_NODES })
|
|
|
11730
12023
|
const resolved = deref(document, node, 0);
|
|
11731
12024
|
if (resolved === null || typeof resolved !== "object") return resolved;
|
|
11732
12025
|
if (Array.isArray(resolved)) {
|
|
11733
|
-
return resolved.map((
|
|
12026
|
+
return resolved.map((entry2) => resolveSchema(document, entry2, depth + 1, budget));
|
|
11734
12027
|
}
|
|
11735
12028
|
const out = {};
|
|
11736
12029
|
for (const [key, value] of Object.entries(resolved)) {
|
|
@@ -11781,8 +12074,8 @@ function operationsFrom(document) {
|
|
|
11781
12074
|
...shared,
|
|
11782
12075
|
...Array.isArray(operation["parameters"]) ? operation["parameters"] : []
|
|
11783
12076
|
];
|
|
11784
|
-
for (const
|
|
11785
|
-
const parameter = resolveSchema(document,
|
|
12077
|
+
for (const entry2 of declared.slice(0, SCHEMA_LIMITS.maxProperties)) {
|
|
12078
|
+
const parameter = resolveSchema(document, entry2);
|
|
11786
12079
|
if (!isObject2(parameter) || typeof parameter["name"] !== "string") continue;
|
|
11787
12080
|
const where = parameter["in"];
|
|
11788
12081
|
if (where !== "path" && where !== "query" && where !== "header") {
|
|
@@ -11964,7 +12257,7 @@ var OpenApiConnection = class _OpenApiConnection {
|
|
|
11964
12257
|
}
|
|
11965
12258
|
async call(name, args) {
|
|
11966
12259
|
const startedAt = Date.now();
|
|
11967
|
-
const operation = this.operations.find((
|
|
12260
|
+
const operation = this.operations.find((entry2) => entry2.name === name);
|
|
11968
12261
|
if (!operation) {
|
|
11969
12262
|
return {
|
|
11970
12263
|
ok: false,
|
|
@@ -12461,7 +12754,7 @@ var Workspace = class {
|
|
|
12461
12754
|
// The project stores a header name against a *secret* name; the value
|
|
12462
12755
|
// is substituted here and nowhere earlier.
|
|
12463
12756
|
headers: Object.fromEntries(
|
|
12464
|
-
Object.entries(connector.headers).map(([header, secret]) => [header, secrets[secret]]).filter((
|
|
12757
|
+
Object.entries(connector.headers).map(([header, secret]) => [header, secrets[secret]]).filter((entry2) => entry2[1] !== void 0)
|
|
12465
12758
|
)
|
|
12466
12759
|
});
|
|
12467
12760
|
}
|
|
@@ -12618,7 +12911,7 @@ var Workspace = class {
|
|
|
12618
12911
|
async demonstrate(project, tool, args) {
|
|
12619
12912
|
const live = this.live.get(project.id);
|
|
12620
12913
|
if (!live?.demonstration) throw new Error("Nothing is being recorded right now.");
|
|
12621
|
-
const claimsReadOnly = live.connection.discovery.tools.find((
|
|
12914
|
+
const claimsReadOnly = live.connection.discovery.tools.find((entry2) => entry2.name === tool)?.hints.readOnly === true;
|
|
12622
12915
|
const before = claimsReadOnly ? await this.readPayloads(project) : void 0;
|
|
12623
12916
|
const result = await live.connection.call(tool, args);
|
|
12624
12917
|
if (result.structured !== void 0) {
|
|
@@ -12631,7 +12924,7 @@ var Workspace = class {
|
|
|
12631
12924
|
{ readOnly: true },
|
|
12632
12925
|
JSON.stringify(after) !== JSON.stringify(before)
|
|
12633
12926
|
);
|
|
12634
|
-
if (mismatch && !live.mismatches.some((
|
|
12927
|
+
if (mismatch && !live.mismatches.some((entry2) => entry2.tool === mismatch.tool)) {
|
|
12635
12928
|
live.mismatches.push(mismatch);
|
|
12636
12929
|
}
|
|
12637
12930
|
}
|
|
@@ -12706,7 +12999,7 @@ function assertConnectorTrusted(project) {
|
|
|
12706
12999
|
}
|
|
12707
13000
|
|
|
12708
13001
|
// ../daemon/src/httpAgent.ts
|
|
12709
|
-
import { z as
|
|
13002
|
+
import { z as z11 } from "zod";
|
|
12710
13003
|
|
|
12711
13004
|
// ../proxy/src/session.ts
|
|
12712
13005
|
import { Server } from "@modelcontextprotocol/sdk/server/index.js";
|
|
@@ -12714,7 +13007,7 @@ import {
|
|
|
12714
13007
|
CallToolRequestSchema,
|
|
12715
13008
|
ListToolsRequestSchema
|
|
12716
13009
|
} from "@modelcontextprotocol/sdk/types.js";
|
|
12717
|
-
var PROXY_INFO = { name: "rigorrun-proxy", version:
|
|
13010
|
+
var PROXY_INFO = { name: "rigorrun-proxy", version: RIGORRUN_VERSION };
|
|
12718
13011
|
function createProxySession(options) {
|
|
12719
13012
|
const calls = [];
|
|
12720
13013
|
const allowed = new Map(options.tools.map((tool) => [tool.name, tool]));
|
|
@@ -12869,19 +13162,19 @@ function hostIsLocal(host) {
|
|
|
12869
13162
|
|
|
12870
13163
|
// ../daemon/src/httpAgent.ts
|
|
12871
13164
|
var AGENT_PROTOCOL_V2 = "rigorrun/agent/2";
|
|
12872
|
-
var CompletionSchema =
|
|
12873
|
-
status:
|
|
13165
|
+
var CompletionSchema = z11.object({
|
|
13166
|
+
status: z11.enum(["completed", "failed"]),
|
|
12874
13167
|
/** The agent's account of what it did. Displayed, never scored. */
|
|
12875
|
-
output:
|
|
12876
|
-
usage:
|
|
12877
|
-
promptTokens:
|
|
12878
|
-
completionTokens:
|
|
13168
|
+
output: z11.string().max(8e3).default(""),
|
|
13169
|
+
usage: z11.object({
|
|
13170
|
+
promptTokens: z11.number().int().nonnegative(),
|
|
13171
|
+
completionTokens: z11.number().int().nonnegative()
|
|
12879
13172
|
}).optional(),
|
|
12880
|
-
costUsd:
|
|
13173
|
+
costUsd: z11.number().nullable().optional()
|
|
12881
13174
|
});
|
|
12882
|
-
var ProbeSchema =
|
|
12883
|
-
ok:
|
|
12884
|
-
agent:
|
|
13175
|
+
var ProbeSchema = z11.object({
|
|
13176
|
+
ok: z11.literal(true),
|
|
13177
|
+
agent: z11.object({ name: z11.string().default(""), version: z11.string().default("") }).optional()
|
|
12885
13178
|
});
|
|
12886
13179
|
var DEFAULT_TIMEOUT_MS5 = 12e4;
|
|
12887
13180
|
var MAX_RESPONSE_BYTES3 = 256 * 1024;
|
|
@@ -13731,21 +14024,21 @@ function readAttributes(raw) {
|
|
|
13731
14024
|
if (isObject3(raw)) return raw;
|
|
13732
14025
|
if (!Array.isArray(raw)) return {};
|
|
13733
14026
|
const out = {};
|
|
13734
|
-
for (const
|
|
13735
|
-
if (!isObject3(
|
|
13736
|
-
const value =
|
|
14027
|
+
for (const entry2 of raw) {
|
|
14028
|
+
if (!isObject3(entry2) || typeof entry2["key"] !== "string") continue;
|
|
14029
|
+
const value = entry2["value"];
|
|
13737
14030
|
if (!isObject3(value)) {
|
|
13738
|
-
out[
|
|
14031
|
+
out[entry2["key"]] = value;
|
|
13739
14032
|
continue;
|
|
13740
14033
|
}
|
|
13741
14034
|
for (const tag of ["stringValue", "intValue", "doubleValue", "boolValue"]) {
|
|
13742
14035
|
if (value[tag] !== void 0) {
|
|
13743
|
-
out[
|
|
14036
|
+
out[entry2["key"]] = tag === "intValue" ? Number(value[tag]) : value[tag];
|
|
13744
14037
|
break;
|
|
13745
14038
|
}
|
|
13746
14039
|
}
|
|
13747
|
-
if (out[
|
|
13748
|
-
out[
|
|
14040
|
+
if (out[entry2["key"]] === void 0 && isObject3(value["kvlistValue"])) {
|
|
14041
|
+
out[entry2["key"]] = readAttributes(value["kvlistValue"].values);
|
|
13749
14042
|
}
|
|
13750
14043
|
}
|
|
13751
14044
|
return out;
|
|
@@ -13758,7 +14051,7 @@ function collectSpans(document) {
|
|
|
13758
14051
|
}
|
|
13759
14052
|
};
|
|
13760
14053
|
if (Array.isArray(document)) {
|
|
13761
|
-
for (const
|
|
14054
|
+
for (const entry2 of document) push(entry2);
|
|
13762
14055
|
return spans;
|
|
13763
14056
|
}
|
|
13764
14057
|
if (!isObject3(document)) return spans;
|
|
@@ -14088,7 +14381,7 @@ var Service = class {
|
|
|
14088
14381
|
return `These reads did not answer: ${failed.join(", ")}. RigorRun will not be able to verify what they cover.`;
|
|
14089
14382
|
}
|
|
14090
14383
|
if (induceSchema(observations).schema.entities.length !== 0) return "";
|
|
14091
|
-
if (payloadsAreEmpty(observations.map((
|
|
14384
|
+
if (payloadsAreEmpty(observations.map((entry2) => entry2.payload))) {
|
|
14092
14385
|
return "These reads answered, and there is nothing in this system yet, so RigorRun cannot tell what its records look like. That is fine \u2014 it will work them out from what your demonstration produces. Nothing to do here.";
|
|
14093
14386
|
}
|
|
14094
14387
|
return "These reads answered, but none of them returned structured records \u2014 RigorRun got text back, and it reads structure rather than prose because guessing at the meaning of a sentence is how a verdict stops being trustworthy. Nominate a read that returns structured output if this system has one. If none do, RigorRun can watch your agent work here but cannot check the result.";
|
|
@@ -14134,7 +14427,7 @@ var Service = class {
|
|
|
14134
14427
|
const saved = await this.store.readArtefact(projectId, "demonstration");
|
|
14135
14428
|
return {
|
|
14136
14429
|
inProgress: saved !== void 0,
|
|
14137
|
-
steps: (saved?.entries ?? []).map((
|
|
14430
|
+
steps: (saved?.entries ?? []).map((entry2) => ({ tool: entry2.action, ok: entry2.ok !== false }))
|
|
14138
14431
|
};
|
|
14139
14432
|
}
|
|
14140
14433
|
/** Just the steps. Kept for callers that only want the log. */
|
|
@@ -14326,7 +14619,7 @@ var Service = class {
|
|
|
14326
14619
|
const same = describeAgent(agent);
|
|
14327
14620
|
const updated = {
|
|
14328
14621
|
...project,
|
|
14329
|
-
agents: [...project.agents.filter((
|
|
14622
|
+
agents: [...project.agents.filter((entry2) => describeAgent(entry2) !== same), agent],
|
|
14330
14623
|
timings: {
|
|
14331
14624
|
...project.timings,
|
|
14332
14625
|
agentConnectedAt: agent.lastProbeOk && !project.timings.agentConnectedAt ? this.now().toISOString() : project.timings.agentConnectedAt
|
|
@@ -14426,7 +14719,7 @@ var Service = class {
|
|
|
14426
14719
|
const project = await this.store.read(projectId);
|
|
14427
14720
|
const benchmark = await this.store.readArtefact(projectId, "benchmark");
|
|
14428
14721
|
if (!benchmark) throw new Error("There is no benchmark to run yet.");
|
|
14429
|
-
const config = project.agents.find((
|
|
14722
|
+
const config = project.agents.find((entry2) => entry2.id === agentId);
|
|
14430
14723
|
if (!config) throw new Error(`No agent "${agentId}" on this project.`);
|
|
14431
14724
|
if (!config.lastProbeOk) {
|
|
14432
14725
|
throw new Error(
|
|
@@ -14464,7 +14757,7 @@ var Service = class {
|
|
|
14464
14757
|
// Counted from the assertion results rather than a per-case tally,
|
|
14465
14758
|
// which does not exist: severity lives on each failed check.
|
|
14466
14759
|
criticalFailures: result.caseResults.reduce(
|
|
14467
|
-
(total,
|
|
14760
|
+
(total, entry2) => total + entry2.assertions.filter(
|
|
14468
14761
|
(assertion) => assertion.status === "FAIL" && assertion.failureSeverity === "CRITICAL"
|
|
14469
14762
|
).length,
|
|
14470
14763
|
0
|
|
@@ -14530,7 +14823,7 @@ var Service = class {
|
|
|
14530
14823
|
if (!key) return null;
|
|
14531
14824
|
const { projects } = await this.listAllProjects();
|
|
14532
14825
|
for (const project of projects) {
|
|
14533
|
-
const agent = project.agents.find((
|
|
14826
|
+
const agent = project.agents.find((entry2) => entry2.id === agentId);
|
|
14534
14827
|
if (!agent || agent.kind !== "external") continue;
|
|
14535
14828
|
const expected = await this.store.secret(agent.keyName);
|
|
14536
14829
|
if (expected === void 0 || !keyMatches(key, expected)) return null;
|
|
@@ -14550,13 +14843,13 @@ var Service = class {
|
|
|
14550
14843
|
async noteDriverCheckIn(projectId, agentId) {
|
|
14551
14844
|
this.driver.noteCheckIn(agentId);
|
|
14552
14845
|
const project = await this.store.read(projectId);
|
|
14553
|
-
const agent = project.agents.find((
|
|
14846
|
+
const agent = project.agents.find((entry2) => entry2.id === agentId);
|
|
14554
14847
|
if (!agent || agent.kind !== "external" || agent.lastProbeOk) return;
|
|
14555
14848
|
const now3 = this.now().toISOString();
|
|
14556
14849
|
await this.store.write({
|
|
14557
14850
|
...project,
|
|
14558
14851
|
agents: project.agents.map(
|
|
14559
|
-
(
|
|
14852
|
+
(entry2) => entry2.id === agentId ? { ...entry2, lastProbeAt: now3, lastProbeOk: true, lastProbeProblem: "" } : entry2
|
|
14560
14853
|
),
|
|
14561
14854
|
timings: {
|
|
14562
14855
|
...project.timings,
|
|
@@ -14656,13 +14949,13 @@ function payloadsAreEmpty(payloads) {
|
|
|
14656
14949
|
if (Array.isArray(value)) {
|
|
14657
14950
|
sawContainer = true;
|
|
14658
14951
|
if (value.length > 0) sawContent = true;
|
|
14659
|
-
for (const
|
|
14952
|
+
for (const entry2 of value.slice(0, 50)) walk(entry2, depth + 1);
|
|
14660
14953
|
return;
|
|
14661
14954
|
}
|
|
14662
14955
|
if (typeof value === "object") {
|
|
14663
14956
|
sawContainer = true;
|
|
14664
14957
|
const entries = Object.entries(value);
|
|
14665
|
-
for (const [,
|
|
14958
|
+
for (const [, entry2] of entries.slice(0, 50)) walk(entry2, depth + 1);
|
|
14666
14959
|
return;
|
|
14667
14960
|
}
|
|
14668
14961
|
if (typeof value === "string" && value.trim().length > 0) sawContent = true;
|
|
@@ -14780,11 +15073,11 @@ var Runner = class {
|
|
|
14780
15073
|
app.get("/", async (context) => {
|
|
14781
15074
|
const code = context.req.query("code");
|
|
14782
15075
|
if (code) {
|
|
14783
|
-
const
|
|
14784
|
-
if (!
|
|
15076
|
+
const token2 = this.pairing.redeem(code);
|
|
15077
|
+
if (!token2) return htmlResponse(EXPIRED_PAGE, 403);
|
|
14785
15078
|
context.header(
|
|
14786
15079
|
"set-cookie",
|
|
14787
|
-
`${SESSION_COOKIE}=${
|
|
15080
|
+
`${SESSION_COOKIE}=${token2}; Path=/; HttpOnly; SameSite=Strict; Max-Age=86400`
|
|
14788
15081
|
);
|
|
14789
15082
|
return context.redirect("/", 302);
|
|
14790
15083
|
}
|
|
@@ -14840,6 +15133,21 @@ var Runner = class {
|
|
|
14840
15133
|
);
|
|
14841
15134
|
});
|
|
14842
15135
|
app.use("/api/*", async (context, next) => {
|
|
15136
|
+
const method = context.req.method.toUpperCase();
|
|
15137
|
+
if (method !== "GET" && method !== "HEAD" && method !== "OPTIONS") {
|
|
15138
|
+
const site = context.req.header("sec-fetch-site");
|
|
15139
|
+
const origin = context.req.header("origin");
|
|
15140
|
+
const crossOrigin = site ? site !== "same-origin" && site !== "none" : origin !== void 0 && !originMatchesHost(origin, context.req.header("host"));
|
|
15141
|
+
if (crossOrigin) {
|
|
15142
|
+
return context.json(
|
|
15143
|
+
{
|
|
15144
|
+
error: "cross-origin request refused",
|
|
15145
|
+
detail: "A state-changing request must come from the runner\u2019s own page. This one looks cross-origin, so it was refused even though it carried a session."
|
|
15146
|
+
},
|
|
15147
|
+
403
|
|
15148
|
+
);
|
|
15149
|
+
}
|
|
15150
|
+
}
|
|
14843
15151
|
const bearer = context.req.header("authorization")?.replace(/^Bearer\s+/i, "");
|
|
14844
15152
|
const cookie = cookieValue(context.req.header("cookie"), SESSION_COOKIE);
|
|
14845
15153
|
if (!this.pairing.authorises(bearer ?? cookie)) {
|
|
@@ -14881,10 +15189,10 @@ var Runner = class {
|
|
|
14881
15189
|
project: summarise3(project),
|
|
14882
15190
|
contract: contract ?? null,
|
|
14883
15191
|
benchmark: benchmark ? {
|
|
14884
|
-
cases: benchmark.cases.map((
|
|
14885
|
-
id:
|
|
14886
|
-
name:
|
|
14887
|
-
category:
|
|
15192
|
+
cases: benchmark.cases.map((entry2) => ({
|
|
15193
|
+
id: entry2.id,
|
|
15194
|
+
name: entry2.name,
|
|
15195
|
+
category: entry2.category
|
|
14888
15196
|
})),
|
|
14889
15197
|
notTestable: benchmark.notTestable
|
|
14890
15198
|
} : null,
|
|
@@ -14989,10 +15297,10 @@ var Runner = class {
|
|
|
14989
15297
|
app.post("/api/projects/:id/benchmark", async (context) => {
|
|
14990
15298
|
const benchmark = await service.generate(context.req.param("id"));
|
|
14991
15299
|
return context.json({
|
|
14992
|
-
cases: benchmark.cases.map((
|
|
14993
|
-
id:
|
|
14994
|
-
name:
|
|
14995
|
-
category:
|
|
15300
|
+
cases: benchmark.cases.map((entry2) => ({
|
|
15301
|
+
id: entry2.id,
|
|
15302
|
+
name: entry2.name,
|
|
15303
|
+
category: entry2.category
|
|
14996
15304
|
})),
|
|
14997
15305
|
notTestable: benchmark.notTestable
|
|
14998
15306
|
});
|
|
@@ -15013,7 +15321,7 @@ var Runner = class {
|
|
|
15013
15321
|
typeof body.command === "string" && body.command.length > 0 ? {
|
|
15014
15322
|
name,
|
|
15015
15323
|
command: body.command,
|
|
15016
|
-
args: Array.isArray(body.args) ? body.args.filter((
|
|
15324
|
+
args: Array.isArray(body.args) ? body.args.filter((entry2) => typeof entry2 === "string") : []
|
|
15017
15325
|
} : { name, endpoint: String(body.endpoint ?? "") }
|
|
15018
15326
|
);
|
|
15019
15327
|
return context.json({ project: summarise3(added.project), agent: added.agent });
|
|
@@ -15129,6 +15437,14 @@ function hostIsLocal2(host) {
|
|
|
15129
15437
|
if (!host) return false;
|
|
15130
15438
|
return ALLOWED_HOSTS2.has(host.replace(/:\d+$/, "").toLowerCase());
|
|
15131
15439
|
}
|
|
15440
|
+
function originMatchesHost(origin, host) {
|
|
15441
|
+
if (!host) return false;
|
|
15442
|
+
try {
|
|
15443
|
+
return new URL(origin).host.toLowerCase() === host.toLowerCase();
|
|
15444
|
+
} catch {
|
|
15445
|
+
return false;
|
|
15446
|
+
}
|
|
15447
|
+
}
|
|
15132
15448
|
function publicTool(tool) {
|
|
15133
15449
|
return {
|
|
15134
15450
|
name: tool.name,
|
|
@@ -15161,14 +15477,14 @@ function publicRun(run) {
|
|
|
15161
15477
|
notTestable: run.notTestable,
|
|
15162
15478
|
verdict: run.verdict,
|
|
15163
15479
|
scores: run.scores,
|
|
15164
|
-
caseResults: run.caseResults.map((
|
|
15165
|
-
caseId:
|
|
15166
|
-
caseName:
|
|
15167
|
-
category:
|
|
15168
|
-
taskSuccess:
|
|
15169
|
-
policyCompliant:
|
|
15170
|
-
unsafeActions:
|
|
15171
|
-
durationMs:
|
|
15480
|
+
caseResults: run.caseResults.map((entry2) => ({
|
|
15481
|
+
caseId: entry2.caseId,
|
|
15482
|
+
caseName: entry2.caseName,
|
|
15483
|
+
category: entry2.category,
|
|
15484
|
+
taskSuccess: entry2.taskSuccess,
|
|
15485
|
+
policyCompliant: entry2.policyCompliant,
|
|
15486
|
+
unsafeActions: entry2.unsafeActions,
|
|
15487
|
+
durationMs: entry2.durationMs,
|
|
15172
15488
|
// Everything below is the evidence for the line above. It was all
|
|
15173
15489
|
// recorded already and none of it reached the screen, so a failure was a
|
|
15174
15490
|
// red mark with a category beside it and no way to find out what
|
|
@@ -15177,16 +15493,16 @@ function publicRun(run) {
|
|
|
15177
15493
|
// What the agent did, in order, with what it passed. Truncated per step
|
|
15178
15494
|
// rather than as a whole, so a long run stays readable and the shape of
|
|
15179
15495
|
// what happened survives.
|
|
15180
|
-
steps:
|
|
15496
|
+
steps: entry2.steps.slice(0, 60).map((step) => ({
|
|
15181
15497
|
tool: step.tool,
|
|
15182
15498
|
args: clip(step.args),
|
|
15183
15499
|
ok: step.ok,
|
|
15184
15500
|
error: step.error ?? ""
|
|
15185
15501
|
})),
|
|
15186
|
-
stepsOmitted: Math.max(0,
|
|
15502
|
+
stepsOmitted: Math.max(0, entry2.steps.length - 60),
|
|
15187
15503
|
/** What the system said afterwards, which is what the verdict rests on. */
|
|
15188
|
-
finalState: clip(
|
|
15189
|
-
checks:
|
|
15504
|
+
finalState: clip(entry2.finalStateSummary),
|
|
15505
|
+
checks: entry2.assertions.map((assertion) => ({
|
|
15190
15506
|
description: assertion.description,
|
|
15191
15507
|
status: assertion.status,
|
|
15192
15508
|
message: assertion.message,
|
|
@@ -15205,14 +15521,14 @@ function publicRun(run) {
|
|
|
15205
15521
|
* failure this product exists to catch, so its claim is evidence about
|
|
15206
15522
|
* the agent rather than evidence about the system.
|
|
15207
15523
|
*/
|
|
15208
|
-
agentReport:
|
|
15209
|
-
error:
|
|
15524
|
+
agentReport: entry2.agentReport.slice(0, 2e3),
|
|
15525
|
+
error: entry2.error ?? ""
|
|
15210
15526
|
}))
|
|
15211
15527
|
};
|
|
15212
15528
|
}
|
|
15213
15529
|
function clip(value) {
|
|
15214
15530
|
const out = {};
|
|
15215
|
-
for (const [key,
|
|
15531
|
+
for (const [key, entry2] of Object.entries(value).slice(0, 24)) out[key] = clipValue(entry2);
|
|
15216
15532
|
return out;
|
|
15217
15533
|
}
|
|
15218
15534
|
function clipValue(value) {
|
|
@@ -15522,7 +15838,12 @@ async function receiveTrace(port, outPath) {
|
|
|
15522
15838
|
line(`${c.green("received")} ${trace.events.length} events from ${trace.app.origin}`);
|
|
15523
15839
|
line(`${c.grey("written")} ${written}`);
|
|
15524
15840
|
line();
|
|
15525
|
-
line(c.grey(
|
|
15841
|
+
line(c.grey("This is a browser recording. It is not a contract, and `rigorrun compile`"));
|
|
15842
|
+
line(c.grey("cannot read it: compiling needs the state your system held before and"));
|
|
15843
|
+
line(c.grey("after the job, which a page cannot supply."));
|
|
15844
|
+
line();
|
|
15845
|
+
line(c.grey("To build a suite, connect the system through `rigorrun` and do the job"));
|
|
15846
|
+
line(c.grey("there instead. See docs/V1_GAP_AUDIT.md for what this path does not do."));
|
|
15526
15847
|
server.close(() => resolve5(0));
|
|
15527
15848
|
})();
|
|
15528
15849
|
});
|
|
@@ -15597,7 +15918,7 @@ async function cmdServe(options = {}) {
|
|
|
15597
15918
|
if (reaped.length > 0) {
|
|
15598
15919
|
line(
|
|
15599
15920
|
c.grey(
|
|
15600
|
-
` Ended ${reaped.length} server(s) a previous run left behind: ${reaped.map((
|
|
15921
|
+
` Ended ${reaped.length} server(s) a previous run left behind: ${reaped.map((entry2) => entry2.fingerprint.split(" ")[0]).join(", ")}`
|
|
15601
15922
|
)
|
|
15602
15923
|
);
|
|
15603
15924
|
}
|
|
@@ -15640,6 +15961,1235 @@ async function cmdServe(options = {}) {
|
|
|
15640
15961
|
return 0;
|
|
15641
15962
|
}
|
|
15642
15963
|
|
|
15964
|
+
// ../sandbox/src/reference.ts
|
|
15965
|
+
var NPM_SPEC = /^(@[a-z0-9-~][a-z0-9-._~]*\/)?([a-z0-9-~][a-z0-9-._~]*)(?:@(.+))?$/;
|
|
15966
|
+
var ReferenceError = class extends Error {
|
|
15967
|
+
};
|
|
15968
|
+
function parseReference(input) {
|
|
15969
|
+
const raw = input.trim();
|
|
15970
|
+
if (raw.length === 0) throw new ReferenceError("No server reference given.");
|
|
15971
|
+
const colon = raw.indexOf(":");
|
|
15972
|
+
const scheme = colon === -1 ? "" : raw.slice(0, colon);
|
|
15973
|
+
const rest = colon === -1 ? "" : raw.slice(colon + 1);
|
|
15974
|
+
if (scheme === "npm") {
|
|
15975
|
+
const match = NPM_SPEC.exec(rest);
|
|
15976
|
+
if (!match) {
|
|
15977
|
+
throw new ReferenceError(
|
|
15978
|
+
`"${rest}" is not an npm package name. Expected npm:<package>@<version>, for example npm:@modelcontextprotocol/server-memory@2026.8.31.`
|
|
15979
|
+
);
|
|
15980
|
+
}
|
|
15981
|
+
return {
|
|
15982
|
+
scheme: "npm",
|
|
15983
|
+
raw,
|
|
15984
|
+
name: `${match[1] ?? ""}${match[2] ?? ""}`,
|
|
15985
|
+
version: match[3] ?? ""
|
|
15986
|
+
};
|
|
15987
|
+
}
|
|
15988
|
+
if (scheme === "dir") {
|
|
15989
|
+
if (rest.length === 0) throw new ReferenceError("dir: needs a path.");
|
|
15990
|
+
return { scheme: "dir", raw, name: rest, version: "" };
|
|
15991
|
+
}
|
|
15992
|
+
throw new ReferenceError(
|
|
15993
|
+
`RigorRun does not know how to fetch "${raw}". Supported today: npm:<package>@<version> for a published server, and dir:<path> for one on this machine. A container image reference, a git ref and a remote endpoint are planned and deliberately not pretended to work yet.`
|
|
15994
|
+
);
|
|
15995
|
+
}
|
|
15996
|
+
|
|
15997
|
+
// ../sandbox/src/registry.ts
|
|
15998
|
+
var RegistryError = class extends Error {
|
|
15999
|
+
};
|
|
16000
|
+
var REGISTRY = "https://registry.npmjs.org";
|
|
16001
|
+
function encodeName(name) {
|
|
16002
|
+
return name.replace("/", "%2f");
|
|
16003
|
+
}
|
|
16004
|
+
async function getJson(url) {
|
|
16005
|
+
const response = await fetch(url, { headers: { accept: "application/json" } });
|
|
16006
|
+
if (!response.ok) {
|
|
16007
|
+
throw new RegistryError(
|
|
16008
|
+
`The registry answered ${response.status} for ${url}. If the package name is right, it may not be published.`
|
|
16009
|
+
);
|
|
16010
|
+
}
|
|
16011
|
+
return response.json();
|
|
16012
|
+
}
|
|
16013
|
+
async function resolvePackage(name, version) {
|
|
16014
|
+
let exact = version;
|
|
16015
|
+
if (exact === "" || exact === "latest") {
|
|
16016
|
+
const doc2 = await getJson(`${REGISTRY}/${encodeName(name)}`);
|
|
16017
|
+
const latest = doc2["dist-tags"]?.["latest"];
|
|
16018
|
+
if (typeof latest !== "string") {
|
|
16019
|
+
throw new RegistryError(`${name} publishes no "latest" version.`);
|
|
16020
|
+
}
|
|
16021
|
+
exact = latest;
|
|
16022
|
+
}
|
|
16023
|
+
const doc = await getJson(`${REGISTRY}/${encodeName(name)}/${exact}`);
|
|
16024
|
+
const resolved = typeof doc.version === "string" ? doc.version : exact;
|
|
16025
|
+
const integrity = doc.dist?.integrity;
|
|
16026
|
+
const tarball = doc.dist?.tarball;
|
|
16027
|
+
if (typeof integrity !== "string" || !integrity.startsWith("sha512-")) {
|
|
16028
|
+
throw new RegistryError(
|
|
16029
|
+
`${name}@${resolved} publishes no sha512 integrity. RigorRun will not verify a package it cannot pin, because the record would be about bytes it could not name.`
|
|
16030
|
+
);
|
|
16031
|
+
}
|
|
16032
|
+
if (typeof tarball !== "string") {
|
|
16033
|
+
throw new RegistryError(`${name}@${resolved} publishes no tarball URL.`);
|
|
16034
|
+
}
|
|
16035
|
+
return { name, version: resolved, integrity, tarballUrl: tarball };
|
|
16036
|
+
}
|
|
16037
|
+
async function fetchVerifiedTarball(pkg) {
|
|
16038
|
+
const response = await fetch(pkg.tarballUrl);
|
|
16039
|
+
if (!response.ok) {
|
|
16040
|
+
throw new RegistryError(
|
|
16041
|
+
`Downloading ${pkg.name}@${pkg.version} failed with ${response.status}.`
|
|
16042
|
+
);
|
|
16043
|
+
}
|
|
16044
|
+
const bytes = new Uint8Array(await response.arrayBuffer());
|
|
16045
|
+
const actual = `sha512-${await sha512Base64(bytes)}`;
|
|
16046
|
+
if (actual !== pkg.integrity) {
|
|
16047
|
+
throw new RegistryError(
|
|
16048
|
+
`The tarball for ${pkg.name}@${pkg.version} does not match the digest the registry published.
|
|
16049
|
+
expected ${pkg.integrity}
|
|
16050
|
+
received ${actual}
|
|
16051
|
+
RigorRun stops rather than run bytes it cannot account for.`
|
|
16052
|
+
);
|
|
16053
|
+
}
|
|
16054
|
+
return bytes;
|
|
16055
|
+
}
|
|
16056
|
+
|
|
16057
|
+
// ../sandbox/src/docker.ts
|
|
16058
|
+
var DockerError = class extends Error {
|
|
16059
|
+
};
|
|
16060
|
+
var SAFE_TOKEN = /^[a-zA-Z0-9][a-zA-Z0-9_.-]{0,127}$/;
|
|
16061
|
+
var SAFE_IMAGE = /^[a-z0-9][a-z0-9._/-]{0,127}(@sha256:[0-9a-f]{64}|:[a-zA-Z0-9_.-]{1,64})$/;
|
|
16062
|
+
function assertSafeToken(value, what) {
|
|
16063
|
+
if (!SAFE_TOKEN.test(value)) {
|
|
16064
|
+
throw new DockerError(`${what} "${value}" is not a name RigorRun will pass to a container.`);
|
|
16065
|
+
}
|
|
16066
|
+
return value;
|
|
16067
|
+
}
|
|
16068
|
+
function assertSafeImage(value) {
|
|
16069
|
+
if (!SAFE_IMAGE.test(value)) {
|
|
16070
|
+
throw new DockerError(
|
|
16071
|
+
`"${value}" is not an image reference RigorRun will run. An image must be pinned by digest or carry an ordinary tag.`
|
|
16072
|
+
);
|
|
16073
|
+
}
|
|
16074
|
+
return value;
|
|
16075
|
+
}
|
|
16076
|
+
var FORBIDDEN_ARGS = [
|
|
16077
|
+
"--privileged",
|
|
16078
|
+
"--cap-add",
|
|
16079
|
+
"--device",
|
|
16080
|
+
"--pid=host",
|
|
16081
|
+
"--net=host",
|
|
16082
|
+
"--network=host",
|
|
16083
|
+
"--userns=host",
|
|
16084
|
+
"-v",
|
|
16085
|
+
"--volume",
|
|
16086
|
+
"--mount"
|
|
16087
|
+
];
|
|
16088
|
+
function assertPostureIntact(args) {
|
|
16089
|
+
for (const arg of args) {
|
|
16090
|
+
const head = arg.split("=")[0] ?? arg;
|
|
16091
|
+
if (FORBIDDEN_ARGS.includes(head) || FORBIDDEN_ARGS.includes(arg)) {
|
|
16092
|
+
throw new DockerError(`Refusing to run a container with ${arg}.`);
|
|
16093
|
+
}
|
|
16094
|
+
if (arg.includes("docker.sock")) {
|
|
16095
|
+
throw new DockerError("Refusing to give a container access to the container runtime.");
|
|
16096
|
+
}
|
|
16097
|
+
}
|
|
16098
|
+
}
|
|
16099
|
+
async function docker(args, timeoutMs = 6e4) {
|
|
16100
|
+
assertPostureIntact(args);
|
|
16101
|
+
return runCommand({
|
|
16102
|
+
command: "docker",
|
|
16103
|
+
args,
|
|
16104
|
+
timeoutMs,
|
|
16105
|
+
provenance: "rigorrun-internal"
|
|
16106
|
+
});
|
|
16107
|
+
}
|
|
16108
|
+
async function inspectRuntime() {
|
|
16109
|
+
try {
|
|
16110
|
+
const version = await docker(["version", "--format", "{{.Server.Version}}"], 2e4);
|
|
16111
|
+
if (version.code !== 0) {
|
|
16112
|
+
return {
|
|
16113
|
+
available: false,
|
|
16114
|
+
version: "",
|
|
16115
|
+
rootless: false,
|
|
16116
|
+
detail: version.stderr.trim() || "The docker command exists but the daemon did not answer. Is it running?"
|
|
16117
|
+
};
|
|
16118
|
+
}
|
|
16119
|
+
const security = await docker(["info", "--format", "{{.SecurityOptions}}"], 2e4);
|
|
16120
|
+
const rootless = security.stdout.includes("rootless");
|
|
16121
|
+
return {
|
|
16122
|
+
available: true,
|
|
16123
|
+
version: version.stdout.trim(),
|
|
16124
|
+
rootless,
|
|
16125
|
+
detail: security.stdout.trim()
|
|
16126
|
+
};
|
|
16127
|
+
} catch (error) {
|
|
16128
|
+
return {
|
|
16129
|
+
available: false,
|
|
16130
|
+
version: "",
|
|
16131
|
+
rootless: false,
|
|
16132
|
+
detail: error instanceof Error ? error.message : String(error)
|
|
16133
|
+
};
|
|
16134
|
+
}
|
|
16135
|
+
}
|
|
16136
|
+
async function imageDigest(image) {
|
|
16137
|
+
const result = await docker(["image", "inspect", image, "--format", "{{index .RepoDigests 0}}"]);
|
|
16138
|
+
if (result.code !== 0) return "";
|
|
16139
|
+
return result.stdout.trim();
|
|
16140
|
+
}
|
|
16141
|
+
async function pullImage(image) {
|
|
16142
|
+
const result = await docker(["pull", assertSafeImage(image)], 6e5);
|
|
16143
|
+
if (result.code !== 0) {
|
|
16144
|
+
throw new DockerError(`Could not pull ${image}: ${result.stderr.trim()}`);
|
|
16145
|
+
}
|
|
16146
|
+
}
|
|
16147
|
+
async function buildImage(contextDir, tag) {
|
|
16148
|
+
assertSafeToken(tag.replace(/^rigorrun-target:/, ""), "image tag");
|
|
16149
|
+
const result = await docker(
|
|
16150
|
+
// No RUN instruction exists in the generated Dockerfile, so nothing the
|
|
16151
|
+
// target ships can execute during a build. `--network none` makes that
|
|
16152
|
+
// checkable from the argv rather than only from the Dockerfile.
|
|
16153
|
+
["build", "--network", "none", "--pull=false", "-t", tag, contextDir],
|
|
16154
|
+
6e5
|
|
16155
|
+
);
|
|
16156
|
+
if (result.code !== 0) {
|
|
16157
|
+
throw new DockerError(`Building the target image failed:
|
|
16158
|
+
${result.stderr.trim()}`);
|
|
16159
|
+
}
|
|
16160
|
+
}
|
|
16161
|
+
async function removeImage(tag) {
|
|
16162
|
+
await docker(["image", "rm", "-f", tag], 6e4);
|
|
16163
|
+
}
|
|
16164
|
+
async function forceRemove(name) {
|
|
16165
|
+
try {
|
|
16166
|
+
await docker(["rm", "-f", assertSafeToken(name, "container name")], 3e4);
|
|
16167
|
+
} catch {
|
|
16168
|
+
}
|
|
16169
|
+
}
|
|
16170
|
+
async function reapOrphans2() {
|
|
16171
|
+
const listed = await docker(["ps", "-aq", "--filter", "label=rigorrun.harness=1"], 3e4);
|
|
16172
|
+
if (listed.code !== 0) return [];
|
|
16173
|
+
const ids = listed.stdout.split("\n").map((s) => s.trim()).filter(Boolean);
|
|
16174
|
+
for (const id2 of ids) await forceRemove(id2);
|
|
16175
|
+
return ids;
|
|
16176
|
+
}
|
|
16177
|
+
async function execInContainer(name, argv, timeoutMs = 6e4) {
|
|
16178
|
+
return docker(["exec", assertSafeToken(name, "container name"), ...argv], timeoutMs);
|
|
16179
|
+
}
|
|
16180
|
+
async function containerLayerChanges(name) {
|
|
16181
|
+
const result = await docker(["diff", assertSafeToken(name, "container name")], 3e4);
|
|
16182
|
+
if (result.code !== 0) return [];
|
|
16183
|
+
return result.stdout.split("\n").map((s) => s.trim()).filter(Boolean);
|
|
16184
|
+
}
|
|
16185
|
+
|
|
16186
|
+
// ../sandbox/src/stage.ts
|
|
16187
|
+
import { mkdir as mkdir6, mkdtemp, readdir as readdir5, rm as rm4, writeFile as writeFile3, cp as cp3, readFile as readFile9, chmod as chmod2 } from "node:fs/promises";
|
|
16188
|
+
import { tmpdir } from "node:os";
|
|
16189
|
+
import { join as join13 } from "node:path";
|
|
16190
|
+
var StageError = class extends Error {
|
|
16191
|
+
};
|
|
16192
|
+
var BASE_IMAGE_TAG = "node:20-alpine";
|
|
16193
|
+
var ENTRYPOINT = `#!/bin/sh
|
|
16194
|
+
set -e
|
|
16195
|
+
mkdir -p /work/target
|
|
16196
|
+
cp -R /opt/rigorrun-target-src/. /work/target/
|
|
16197
|
+
cd /work/target
|
|
16198
|
+
exec node "$@"
|
|
16199
|
+
`;
|
|
16200
|
+
var STATEDUMP = `#!/bin/sh
|
|
16201
|
+
find /work /tmp -type f -exec sha256sum {} + 2>/dev/null | LC_ALL=C sort -k2
|
|
16202
|
+
`;
|
|
16203
|
+
var PSLIST = `#!/bin/sh
|
|
16204
|
+
for entry in /proc/[0-9]*; do
|
|
16205
|
+
tr '\\0' ' ' < "$entry/cmdline" 2>/dev/null
|
|
16206
|
+
echo
|
|
16207
|
+
done
|
|
16208
|
+
`;
|
|
16209
|
+
async function scratch() {
|
|
16210
|
+
return mkdtemp(join13(tmpdir(), "rigorrun-stage-"));
|
|
16211
|
+
}
|
|
16212
|
+
async function entryPointOf(treeDir, packageName) {
|
|
16213
|
+
const manifestPath = join13(treeDir, "node_modules", packageName, "package.json");
|
|
16214
|
+
let manifest;
|
|
16215
|
+
try {
|
|
16216
|
+
manifest = JSON.parse(await readFile9(manifestPath, "utf8"));
|
|
16217
|
+
} catch {
|
|
16218
|
+
throw new StageError(
|
|
16219
|
+
`${packageName} did not install a package.json, so RigorRun cannot tell what to run.`
|
|
16220
|
+
);
|
|
16221
|
+
}
|
|
16222
|
+
let relative2;
|
|
16223
|
+
if (typeof manifest.bin === "string") relative2 = manifest.bin;
|
|
16224
|
+
else if (manifest.bin && typeof manifest.bin === "object") {
|
|
16225
|
+
const entries = Object.values(manifest.bin);
|
|
16226
|
+
const first = entries.find((v) => typeof v === "string");
|
|
16227
|
+
relative2 = first;
|
|
16228
|
+
}
|
|
16229
|
+
if (!relative2 && typeof manifest.main === "string") relative2 = manifest.main;
|
|
16230
|
+
if (!relative2) {
|
|
16231
|
+
throw new StageError(
|
|
16232
|
+
`${packageName} declares neither a bin nor a main, so there is nothing to start.`
|
|
16233
|
+
);
|
|
16234
|
+
}
|
|
16235
|
+
return `/work/target/node_modules/${packageName}/${relative2.replace(/^\.\//, "")}`;
|
|
16236
|
+
}
|
|
16237
|
+
async function writeContext(dir, entryModule) {
|
|
16238
|
+
const entry2 = join13(dir, "rigorrun-entrypoint");
|
|
16239
|
+
const dump = join13(dir, "rigorrun-statedump");
|
|
16240
|
+
const ps = join13(dir, "rigorrun-pslist");
|
|
16241
|
+
await writeFile3(entry2, ENTRYPOINT, "utf8");
|
|
16242
|
+
await writeFile3(dump, STATEDUMP, "utf8");
|
|
16243
|
+
await writeFile3(ps, PSLIST, "utf8");
|
|
16244
|
+
await chmod2(entry2, 365);
|
|
16245
|
+
await chmod2(dump, 365);
|
|
16246
|
+
await chmod2(ps, 365);
|
|
16247
|
+
await writeFile3(
|
|
16248
|
+
join13(dir, "Dockerfile"),
|
|
16249
|
+
[
|
|
16250
|
+
`FROM ${BASE_IMAGE_TAG}`,
|
|
16251
|
+
// Owned by the user the container runs as, so copying it onto the
|
|
16252
|
+
// writable mount needs no privilege the container does not have.
|
|
16253
|
+
"COPY --chown=65532:65532 tree/ /opt/rigorrun-target-src/",
|
|
16254
|
+
"COPY rigorrun-entrypoint /rigorrun/entrypoint",
|
|
16255
|
+
"COPY rigorrun-statedump /rigorrun/statedump",
|
|
16256
|
+
"COPY rigorrun-pslist /rigorrun/pslist",
|
|
16257
|
+
"USER 65532:65532",
|
|
16258
|
+
'ENTRYPOINT ["/rigorrun/entrypoint"]',
|
|
16259
|
+
`CMD ["${entryModule}"]`,
|
|
16260
|
+
""
|
|
16261
|
+
].join("\n"),
|
|
16262
|
+
"utf8"
|
|
16263
|
+
);
|
|
16264
|
+
}
|
|
16265
|
+
async function digestOfDirectory(sourceDir) {
|
|
16266
|
+
const entries = [];
|
|
16267
|
+
const walk = async (dir, prefix) => {
|
|
16268
|
+
const listing = await readdir5(dir, { withFileTypes: true });
|
|
16269
|
+
for (const item of listing.sort((a, b) => a.name.localeCompare(b.name))) {
|
|
16270
|
+
if (item.name === "node_modules" || item.name === ".git") continue;
|
|
16271
|
+
const relative2 = prefix ? `${prefix}/${item.name}` : item.name;
|
|
16272
|
+
const full = join13(dir, item.name);
|
|
16273
|
+
if (item.isDirectory()) await walk(full, relative2);
|
|
16274
|
+
else if (item.isFile()) entries.push([relative2, await sha256(await readFile9(full, "utf8"))]);
|
|
16275
|
+
}
|
|
16276
|
+
};
|
|
16277
|
+
await walk(sourceDir, "");
|
|
16278
|
+
return hashValue(entries);
|
|
16279
|
+
}
|
|
16280
|
+
async function digestOfTree(treeDir) {
|
|
16281
|
+
try {
|
|
16282
|
+
const lock = JSON.parse(await readFile9(join13(treeDir, "package-lock.json"), "utf8"));
|
|
16283
|
+
const entries = Object.entries(lock.packages ?? {}).map(([path, meta]) => [path, meta.integrity ?? meta.version ?? ""]).sort(([a], [b]) => a.localeCompare(b));
|
|
16284
|
+
return hashValue(entries);
|
|
16285
|
+
} catch {
|
|
16286
|
+
return "";
|
|
16287
|
+
}
|
|
16288
|
+
}
|
|
16289
|
+
async function stageNpmPackage(packageName, version) {
|
|
16290
|
+
const dir = await scratch();
|
|
16291
|
+
const treeDir = join13(dir, "tree");
|
|
16292
|
+
await mkdir6(treeDir, { recursive: true });
|
|
16293
|
+
await writeFile3(
|
|
16294
|
+
join13(treeDir, "package.json"),
|
|
16295
|
+
JSON.stringify({ name: "rigorrun-target", private: true, version: "0.0.0" }, null, 2),
|
|
16296
|
+
"utf8"
|
|
16297
|
+
);
|
|
16298
|
+
const install = await runCommand({
|
|
16299
|
+
command: "npm",
|
|
16300
|
+
args: [
|
|
16301
|
+
"install",
|
|
16302
|
+
`${packageName}@${version}`,
|
|
16303
|
+
"--ignore-scripts",
|
|
16304
|
+
"--no-audit",
|
|
16305
|
+
"--no-fund",
|
|
16306
|
+
"--omit=dev",
|
|
16307
|
+
"--prefix",
|
|
16308
|
+
treeDir
|
|
16309
|
+
],
|
|
16310
|
+
timeoutMs: 6e5,
|
|
16311
|
+
provenance: "rigorrun-internal"
|
|
16312
|
+
});
|
|
16313
|
+
if (install.code !== 0) {
|
|
16314
|
+
await rm4(dir, { recursive: true, force: true });
|
|
16315
|
+
throw new StageError(
|
|
16316
|
+
`Installing ${packageName}@${version} failed.
|
|
16317
|
+
${install.stderr.trim() || install.stdout.trim()}`
|
|
16318
|
+
);
|
|
16319
|
+
}
|
|
16320
|
+
const entryModule = await entryPointOf(treeDir, packageName);
|
|
16321
|
+
await writeContext(dir, entryModule);
|
|
16322
|
+
return {
|
|
16323
|
+
dir,
|
|
16324
|
+
treeDigest: await digestOfTree(treeDir),
|
|
16325
|
+
contentDigest: "",
|
|
16326
|
+
entryModule,
|
|
16327
|
+
cleanup: () => rm4(dir, { recursive: true, force: true })
|
|
16328
|
+
};
|
|
16329
|
+
}
|
|
16330
|
+
async function stageDirectory(sourceDir) {
|
|
16331
|
+
const dir = await scratch();
|
|
16332
|
+
const treeDir = join13(dir, "tree");
|
|
16333
|
+
await mkdir6(treeDir, { recursive: true });
|
|
16334
|
+
const manifest = JSON.parse(
|
|
16335
|
+
await readFile9(join13(sourceDir, "package.json"), "utf8")
|
|
16336
|
+
);
|
|
16337
|
+
const name = manifest.name;
|
|
16338
|
+
if (!name) throw new StageError(`${sourceDir} has no package name.`);
|
|
16339
|
+
const deps = Object.entries(manifest.dependencies ?? {});
|
|
16340
|
+
await writeFile3(
|
|
16341
|
+
join13(treeDir, "package.json"),
|
|
16342
|
+
JSON.stringify(
|
|
16343
|
+
{
|
|
16344
|
+
name: "rigorrun-target",
|
|
16345
|
+
private: true,
|
|
16346
|
+
version: "0.0.0",
|
|
16347
|
+
...deps.length > 0 ? { dependencies: manifest.dependencies } : {}
|
|
16348
|
+
},
|
|
16349
|
+
null,
|
|
16350
|
+
2
|
|
16351
|
+
),
|
|
16352
|
+
"utf8"
|
|
16353
|
+
);
|
|
16354
|
+
if (deps.length > 0) {
|
|
16355
|
+
const install = await runCommand({
|
|
16356
|
+
command: "npm",
|
|
16357
|
+
args: ["install", "--ignore-scripts", "--no-audit", "--no-fund", "--omit=dev", "--prefix", treeDir],
|
|
16358
|
+
timeoutMs: 6e5,
|
|
16359
|
+
provenance: "rigorrun-internal"
|
|
16360
|
+
});
|
|
16361
|
+
if (install.code !== 0) {
|
|
16362
|
+
await rm4(dir, { recursive: true, force: true });
|
|
16363
|
+
throw new StageError(
|
|
16364
|
+
`Installing dependencies for ${sourceDir} failed.
|
|
16365
|
+
${install.stderr.trim()}`
|
|
16366
|
+
);
|
|
16367
|
+
}
|
|
16368
|
+
}
|
|
16369
|
+
const packageDir = join13(treeDir, "node_modules", name);
|
|
16370
|
+
await mkdir6(packageDir, { recursive: true });
|
|
16371
|
+
await cp3(sourceDir, packageDir, {
|
|
16372
|
+
recursive: true,
|
|
16373
|
+
filter: (src) => !src.includes("node_modules")
|
|
16374
|
+
});
|
|
16375
|
+
const entryModule = await entryPointOf(treeDir, name);
|
|
16376
|
+
await writeContext(dir, entryModule);
|
|
16377
|
+
return {
|
|
16378
|
+
dir,
|
|
16379
|
+
treeDigest: await digestOfTree(treeDir),
|
|
16380
|
+
contentDigest: await digestOfDirectory(sourceDir),
|
|
16381
|
+
entryModule,
|
|
16382
|
+
cleanup: () => rm4(dir, { recursive: true, force: true })
|
|
16383
|
+
};
|
|
16384
|
+
}
|
|
16385
|
+
|
|
16386
|
+
// ../sandbox/src/session.ts
|
|
16387
|
+
var DEFAULT_LIMITS = { pids: 256, memoryMb: 512, cpus: 1 };
|
|
16388
|
+
function runArgv(name, spec) {
|
|
16389
|
+
const limits = spec.limits ?? DEFAULT_LIMITS;
|
|
16390
|
+
assertSafeToken(name, "container name");
|
|
16391
|
+
assertSafeImage(spec.image);
|
|
16392
|
+
const argv = [
|
|
16393
|
+
"run",
|
|
16394
|
+
"--rm",
|
|
16395
|
+
"-i",
|
|
16396
|
+
"--name",
|
|
16397
|
+
name,
|
|
16398
|
+
"--label",
|
|
16399
|
+
"rigorrun.harness=1",
|
|
16400
|
+
"--network",
|
|
16401
|
+
"none",
|
|
16402
|
+
"--read-only",
|
|
16403
|
+
"--cap-drop",
|
|
16404
|
+
"ALL",
|
|
16405
|
+
"--security-opt",
|
|
16406
|
+
"no-new-privileges",
|
|
16407
|
+
"--user",
|
|
16408
|
+
"65532:65532",
|
|
16409
|
+
"--pids-limit",
|
|
16410
|
+
String(limits.pids),
|
|
16411
|
+
"--memory",
|
|
16412
|
+
`${limits.memoryMb}m`,
|
|
16413
|
+
// Equal to --memory, so the limit cannot be sidestepped by swapping.
|
|
16414
|
+
"--memory-swap",
|
|
16415
|
+
`${limits.memoryMb}m`,
|
|
16416
|
+
"--cpus",
|
|
16417
|
+
String(limits.cpus),
|
|
16418
|
+
// /work is not noexec: a native addon is dlopen'd and would fail. /tmp is.
|
|
16419
|
+
"--tmpfs",
|
|
16420
|
+
"/work:rw,nosuid,nodev,size=256m,mode=1777",
|
|
16421
|
+
"--tmpfs",
|
|
16422
|
+
"/tmp:rw,noexec,nosuid,nodev,size=64m,mode=1777",
|
|
16423
|
+
spec.image,
|
|
16424
|
+
...spec.command ?? []
|
|
16425
|
+
];
|
|
16426
|
+
assertPostureIntact(argv);
|
|
16427
|
+
return argv;
|
|
16428
|
+
}
|
|
16429
|
+
var ContainerSession = class {
|
|
16430
|
+
name;
|
|
16431
|
+
argv;
|
|
16432
|
+
removed = false;
|
|
16433
|
+
constructor(spec) {
|
|
16434
|
+
this.name = `rigorrun-verify-${prefixedId("c").replace(/[^a-z0-9]/gi, "").slice(0, 20)}`;
|
|
16435
|
+
this.argv = runArgv(this.name, spec);
|
|
16436
|
+
}
|
|
16437
|
+
/** Reads the writable mounts. The surface everything else is measured on. */
|
|
16438
|
+
async statedump() {
|
|
16439
|
+
const result = await execInContainer(this.name, ["/rigorrun/statedump"], 12e4);
|
|
16440
|
+
if (result.code !== 0) return "";
|
|
16441
|
+
return result.stdout;
|
|
16442
|
+
}
|
|
16443
|
+
async processList() {
|
|
16444
|
+
const result = await execInContainer(this.name, ["/rigorrun/pslist"], 6e4);
|
|
16445
|
+
return result.code === 0 ? result.stdout : "";
|
|
16446
|
+
}
|
|
16447
|
+
/**
|
|
16448
|
+
* Whether the read-only posture actually held.
|
|
16449
|
+
*
|
|
16450
|
+
* `docker diff` always reports the tmpfs mount points themselves — they are
|
|
16451
|
+
* directories created in the container layer at start — so those are not
|
|
16452
|
+
* evidence of anything. Anything else appearing here means a write reached
|
|
16453
|
+
* the image layer, which under `--read-only` should be impossible, and is
|
|
16454
|
+
* recorded as the posture having failed rather than quietly ignored.
|
|
16455
|
+
*/
|
|
16456
|
+
async layerChanges() {
|
|
16457
|
+
const changes = await containerLayerChanges(this.name);
|
|
16458
|
+
const mounts = /* @__PURE__ */ new Set(["A /work", "A /tmp", "C /work", "C /tmp"]);
|
|
16459
|
+
return changes.filter((entry2) => !mounts.has(entry2));
|
|
16460
|
+
}
|
|
16461
|
+
/** Idempotent, and safe to call from a signal handler. */
|
|
16462
|
+
async remove() {
|
|
16463
|
+
if (this.removed) return;
|
|
16464
|
+
this.removed = true;
|
|
16465
|
+
await forceRemove(this.name);
|
|
16466
|
+
}
|
|
16467
|
+
};
|
|
16468
|
+
|
|
16469
|
+
// ../sandbox/src/surfaces.ts
|
|
16470
|
+
function parseStatedump(output) {
|
|
16471
|
+
const entries = /* @__PURE__ */ new Map();
|
|
16472
|
+
for (const line2 of output.split("\n")) {
|
|
16473
|
+
const separator = line2.indexOf(" ");
|
|
16474
|
+
if (separator <= 0) continue;
|
|
16475
|
+
const digest = line2.slice(0, separator);
|
|
16476
|
+
const path = line2.slice(separator + 2);
|
|
16477
|
+
if (path.length === 0) continue;
|
|
16478
|
+
entries.set(path, digest);
|
|
16479
|
+
}
|
|
16480
|
+
return entries;
|
|
16481
|
+
}
|
|
16482
|
+
function diffReadings(before, after) {
|
|
16483
|
+
const created = [];
|
|
16484
|
+
const modified = [];
|
|
16485
|
+
const deleted = [];
|
|
16486
|
+
for (const [path, digest] of after.entries) {
|
|
16487
|
+
const previous = before.entries.get(path);
|
|
16488
|
+
if (previous === void 0) created.push(path);
|
|
16489
|
+
else if (previous !== digest) modified.push(path);
|
|
16490
|
+
}
|
|
16491
|
+
for (const path of before.entries.keys()) {
|
|
16492
|
+
if (!after.entries.has(path)) deleted.push(path);
|
|
16493
|
+
}
|
|
16494
|
+
return { created: created.sort(), modified: modified.sort(), deleted: deleted.sort() };
|
|
16495
|
+
}
|
|
16496
|
+
function isEmpty(delta2) {
|
|
16497
|
+
return delta2.created.length === 0 && delta2.modified.length === 0 && delta2.deleted.length === 0;
|
|
16498
|
+
}
|
|
16499
|
+
function destroyedAnything(delta2) {
|
|
16500
|
+
return delta2.deleted.length > 0 || delta2.modified.length > 0;
|
|
16501
|
+
}
|
|
16502
|
+
function touchedPaths(delta2) {
|
|
16503
|
+
return [...delta2.created, ...delta2.modified, ...delta2.deleted].sort();
|
|
16504
|
+
}
|
|
16505
|
+
|
|
16506
|
+
// ../sandbox/src/verify.ts
|
|
16507
|
+
import { rm as rm5 } from "node:fs/promises";
|
|
16508
|
+
|
|
16509
|
+
// ../conformance/src/plan.ts
|
|
16510
|
+
var FORBIDDEN = /[/\\*?~]|\.\./;
|
|
16511
|
+
var MAX_STRING = 64;
|
|
16512
|
+
function isObject4(value) {
|
|
16513
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
16514
|
+
}
|
|
16515
|
+
function declaredType(node) {
|
|
16516
|
+
const t = node["type"];
|
|
16517
|
+
if (typeof t === "string") return t;
|
|
16518
|
+
if (Array.isArray(t)) {
|
|
16519
|
+
const first = t.find((entry2) => typeof entry2 === "string" && entry2 !== "null");
|
|
16520
|
+
if (typeof first === "string") return first;
|
|
16521
|
+
}
|
|
16522
|
+
if (Array.isArray(node["enum"]) || node["const"] !== void 0) return "string";
|
|
16523
|
+
if (isObject4(node["properties"])) return "object";
|
|
16524
|
+
return void 0;
|
|
16525
|
+
}
|
|
16526
|
+
function token(seed, pointer) {
|
|
16527
|
+
let h = 2166136261;
|
|
16528
|
+
for (const ch of `${seed}${pointer}`) {
|
|
16529
|
+
h ^= ch.charCodeAt(0);
|
|
16530
|
+
h = Math.imul(h, 16777619) >>> 0;
|
|
16531
|
+
}
|
|
16532
|
+
return `rr-${h.toString(16).padStart(8, "0")}`;
|
|
16533
|
+
}
|
|
16534
|
+
function synth(node, pointer, depth, ctx) {
|
|
16535
|
+
if (ctx.blocked) return void 0;
|
|
16536
|
+
if (depth > SCHEMA_LIMITS.maxDepth) {
|
|
16537
|
+
ctx.blocked = `the schema nests deeper than ${SCHEMA_LIMITS.maxDepth} levels at ${pointer}`;
|
|
16538
|
+
return void 0;
|
|
16539
|
+
}
|
|
16540
|
+
if (++ctx.nodes > SCHEMA_LIMITS.maxNodes) {
|
|
16541
|
+
ctx.blocked = "the schema is larger than RigorRun will walk";
|
|
16542
|
+
return void 0;
|
|
16543
|
+
}
|
|
16544
|
+
if (!isObject4(node)) {
|
|
16545
|
+
ctx.blocked = `no schema for ${pointer}`;
|
|
16546
|
+
return void 0;
|
|
16547
|
+
}
|
|
16548
|
+
if (node["const"] !== void 0) {
|
|
16549
|
+
ctx.derivation.push(`${pointer}: the schema's single allowed value`);
|
|
16550
|
+
return node["const"];
|
|
16551
|
+
}
|
|
16552
|
+
if (Array.isArray(node["enum"])) {
|
|
16553
|
+
const values = [...node["enum"]];
|
|
16554
|
+
if (values.length === 0) {
|
|
16555
|
+
ctx.blocked = `${pointer} allows no values`;
|
|
16556
|
+
return void 0;
|
|
16557
|
+
}
|
|
16558
|
+
values.sort((a, b) => String(a).localeCompare(String(b)));
|
|
16559
|
+
ctx.derivation.push(`${pointer}: first of ${values.length} allowed values, sorted`);
|
|
16560
|
+
return values[0];
|
|
16561
|
+
}
|
|
16562
|
+
switch (declaredType(node)) {
|
|
16563
|
+
case "boolean":
|
|
16564
|
+
ctx.derivation.push(`${pointer}: false`);
|
|
16565
|
+
return false;
|
|
16566
|
+
case "integer":
|
|
16567
|
+
case "number": {
|
|
16568
|
+
const min = typeof node["minimum"] === "number" ? node["minimum"] : void 0;
|
|
16569
|
+
const max = typeof node["maximum"] === "number" ? node["maximum"] : void 0;
|
|
16570
|
+
let n = 1;
|
|
16571
|
+
if (min !== void 0 && n < min) n = min;
|
|
16572
|
+
if (max !== void 0 && n > max) n = max;
|
|
16573
|
+
ctx.derivation.push(`${pointer}: ${n}, inside any declared bounds`);
|
|
16574
|
+
return n;
|
|
16575
|
+
}
|
|
16576
|
+
case "string": {
|
|
16577
|
+
if (typeof node["pattern"] === "string") {
|
|
16578
|
+
ctx.blocked = `${pointer} constrains its value by pattern, which RigorRun will not guess`;
|
|
16579
|
+
return void 0;
|
|
16580
|
+
}
|
|
16581
|
+
if (node["format"] === "date-time") {
|
|
16582
|
+
ctx.derivation.push(`${pointer}: a fixed timestamp`);
|
|
16583
|
+
return "2026-01-01T00:00:00.000Z";
|
|
16584
|
+
}
|
|
16585
|
+
const value = token(ctx.seed, pointer);
|
|
16586
|
+
ctx.derivation.push(`${pointer}: an inert generated token`);
|
|
16587
|
+
return value;
|
|
16588
|
+
}
|
|
16589
|
+
case "array": {
|
|
16590
|
+
const items = node["items"];
|
|
16591
|
+
if (items === void 0) {
|
|
16592
|
+
ctx.blocked = `${pointer} is an array but does not say of what`;
|
|
16593
|
+
return void 0;
|
|
16594
|
+
}
|
|
16595
|
+
const element = synth(items, `${pointer}/0`, depth + 1, ctx);
|
|
16596
|
+
if (ctx.blocked) return void 0;
|
|
16597
|
+
ctx.derivation.push(`${pointer}: exactly one element`);
|
|
16598
|
+
return [element];
|
|
16599
|
+
}
|
|
16600
|
+
case "object": {
|
|
16601
|
+
const props = node["properties"];
|
|
16602
|
+
if (!isObject4(props)) {
|
|
16603
|
+
ctx.derivation.push(`${pointer}: an empty object`);
|
|
16604
|
+
return {};
|
|
16605
|
+
}
|
|
16606
|
+
const required = Array.isArray(node["required"]) ? node["required"].filter((r) => typeof r === "string") : [];
|
|
16607
|
+
const out = {};
|
|
16608
|
+
for (const key of required) {
|
|
16609
|
+
const child = props[key];
|
|
16610
|
+
if (child === void 0) {
|
|
16611
|
+
ctx.blocked = `${pointer} requires "${key}" but does not describe it`;
|
|
16612
|
+
return void 0;
|
|
16613
|
+
}
|
|
16614
|
+
out[key] = synth(child, `${pointer}/${key}`, depth + 1, ctx);
|
|
16615
|
+
if (ctx.blocked) return void 0;
|
|
16616
|
+
}
|
|
16617
|
+
return out;
|
|
16618
|
+
}
|
|
16619
|
+
default:
|
|
16620
|
+
ctx.blocked = `${pointer} declares no type RigorRun can produce a value for`;
|
|
16621
|
+
return void 0;
|
|
16622
|
+
}
|
|
16623
|
+
}
|
|
16624
|
+
function assertInert(value, pointer) {
|
|
16625
|
+
if (typeof value === "string") {
|
|
16626
|
+
if (value.length > MAX_STRING) return `${pointer} would need a string longer than ${MAX_STRING}`;
|
|
16627
|
+
if (FORBIDDEN.test(value)) return `${pointer} would need a value containing a path or wildcard`;
|
|
16628
|
+
}
|
|
16629
|
+
if (Array.isArray(value)) {
|
|
16630
|
+
for (const [i, entry2] of value.entries()) {
|
|
16631
|
+
const bad = assertInert(entry2, `${pointer}/${i}`);
|
|
16632
|
+
if (bad) return bad;
|
|
16633
|
+
}
|
|
16634
|
+
}
|
|
16635
|
+
if (isObject4(value)) {
|
|
16636
|
+
for (const [key, entry2] of Object.entries(value)) {
|
|
16637
|
+
const bad = assertInert(entry2, `${pointer}/${key}`);
|
|
16638
|
+
if (bad) return bad;
|
|
16639
|
+
}
|
|
16640
|
+
}
|
|
16641
|
+
return void 0;
|
|
16642
|
+
}
|
|
16643
|
+
function planTool(input) {
|
|
16644
|
+
if (input.hints.destructive === true) {
|
|
16645
|
+
return {
|
|
16646
|
+
class: "UNSAFE_TO_EXERCISE",
|
|
16647
|
+
reason: "the server declares this tool destructive, and RigorRun takes that at its word"
|
|
16648
|
+
};
|
|
16649
|
+
}
|
|
16650
|
+
if (input.needsCredential?.includes(input.name)) {
|
|
16651
|
+
return {
|
|
16652
|
+
class: "NEEDS_CREDENTIAL",
|
|
16653
|
+
reason: "the operator declared that this tool needs a credential"
|
|
16654
|
+
};
|
|
16655
|
+
}
|
|
16656
|
+
const ctx = { seed: input.seed, derivation: [], nodes: 0 };
|
|
16657
|
+
const args = synth(input.inputSchema, "", 0, ctx);
|
|
16658
|
+
if (ctx.blocked) return { class: "UNDETERMINED", reason: ctx.blocked };
|
|
16659
|
+
if (!isObject4(args)) {
|
|
16660
|
+
return { class: "UNDETERMINED", reason: "the tool does not take an object of arguments" };
|
|
16661
|
+
}
|
|
16662
|
+
const unsafe = assertInert(args, "");
|
|
16663
|
+
if (unsafe) return { class: "UNDETERMINED", reason: unsafe };
|
|
16664
|
+
return { class: "SAFE_AUTOMATIC", args, derivation: ctx.derivation };
|
|
16665
|
+
}
|
|
16666
|
+
|
|
16667
|
+
// ../conformance/src/judge.ts
|
|
16668
|
+
var AUTHORITATIVE_SURFACES = ["container_fs", "process_table"];
|
|
16669
|
+
function strongestSurface(surfaces) {
|
|
16670
|
+
return AUTHORITATIVE_SURFACES.find((s) => surfaces.includes(s)) ?? (surfaces.includes("server_reads") ? "server_reads" : void 0);
|
|
16671
|
+
}
|
|
16672
|
+
function canConvict(surfaces) {
|
|
16673
|
+
return AUTHORITATIVE_SURFACES.some((s) => surfaces.includes(s));
|
|
16674
|
+
}
|
|
16675
|
+
function entry(claim, verdict2, severity, permissionRelevant, because, observed) {
|
|
16676
|
+
const surface = strongestSurface(observed.surfaces);
|
|
16677
|
+
return {
|
|
16678
|
+
claim,
|
|
16679
|
+
verdict: verdict2,
|
|
16680
|
+
severity,
|
|
16681
|
+
permissionRelevant,
|
|
16682
|
+
because,
|
|
16683
|
+
...surface ? { surface } : {},
|
|
16684
|
+
evidenceRef: observed.evidenceRef
|
|
16685
|
+
};
|
|
16686
|
+
}
|
|
16687
|
+
function judge(hints, observed) {
|
|
16688
|
+
const out = [];
|
|
16689
|
+
if (hints.readOnly === true) {
|
|
16690
|
+
const claim = "readOnlyHint: true";
|
|
16691
|
+
if (observed.mutatesState === "yes" && canConvict(observed.surfaces)) {
|
|
16692
|
+
out.push(
|
|
16693
|
+
entry(
|
|
16694
|
+
claim,
|
|
16695
|
+
"CONTRADICTED",
|
|
16696
|
+
"CRITICAL",
|
|
16697
|
+
true,
|
|
16698
|
+
"the tool declares itself read-only and was observed changing state RigorRun read independently of the server",
|
|
16699
|
+
observed
|
|
16700
|
+
)
|
|
16701
|
+
);
|
|
16702
|
+
} else if (observed.mutatesState === "yes") {
|
|
16703
|
+
out.push(
|
|
16704
|
+
entry(
|
|
16705
|
+
claim,
|
|
16706
|
+
"UNDETERMINED",
|
|
16707
|
+
"INFO",
|
|
16708
|
+
false,
|
|
16709
|
+
"a change was seen only through the server\u2019s own tools, which is the same server that would have to be wrong for this to be a contradiction",
|
|
16710
|
+
observed
|
|
16711
|
+
)
|
|
16712
|
+
);
|
|
16713
|
+
} else if (observed.mutatesState === "no" && canConvict(observed.surfaces)) {
|
|
16714
|
+
out.push(entry(claim, "CONFORMS", "INFO", false, "no state change was observed", observed));
|
|
16715
|
+
} else {
|
|
16716
|
+
out.push(
|
|
16717
|
+
entry(
|
|
16718
|
+
claim,
|
|
16719
|
+
"UNDETERMINED",
|
|
16720
|
+
"INFO",
|
|
16721
|
+
false,
|
|
16722
|
+
"RigorRun could not read any state that would prove or disprove this",
|
|
16723
|
+
observed
|
|
16724
|
+
)
|
|
16725
|
+
);
|
|
16726
|
+
}
|
|
16727
|
+
}
|
|
16728
|
+
if (hints.readOnly === false) {
|
|
16729
|
+
const claim = "readOnlyHint: false";
|
|
16730
|
+
if (observed.mutatesState === "yes") {
|
|
16731
|
+
out.push(entry(claim, "CONFORMS", "INFO", false, "the tool changed state, as declared", observed));
|
|
16732
|
+
} else {
|
|
16733
|
+
out.push(
|
|
16734
|
+
entry(
|
|
16735
|
+
claim,
|
|
16736
|
+
"UNDETERMINED",
|
|
16737
|
+
"INFO",
|
|
16738
|
+
false,
|
|
16739
|
+
"the tool declares that it writes and nothing durable changed, which RigorRun cannot separate from a write held only in the process\u2019s memory",
|
|
16740
|
+
observed
|
|
16741
|
+
)
|
|
16742
|
+
);
|
|
16743
|
+
}
|
|
16744
|
+
}
|
|
16745
|
+
if (hints.destructive === false) {
|
|
16746
|
+
const claim = "destructiveHint: false";
|
|
16747
|
+
if (observed.destroyed === "yes" && canConvict(observed.surfaces)) {
|
|
16748
|
+
out.push(
|
|
16749
|
+
entry(
|
|
16750
|
+
claim,
|
|
16751
|
+
"CONTRADICTED",
|
|
16752
|
+
"CRITICAL",
|
|
16753
|
+
true,
|
|
16754
|
+
"the tool declares that it destroys nothing, and was observed deleting or overwriting existing state",
|
|
16755
|
+
observed
|
|
16756
|
+
)
|
|
16757
|
+
);
|
|
16758
|
+
} else if (observed.destroyed === "no" && canConvict(observed.surfaces)) {
|
|
16759
|
+
out.push(
|
|
16760
|
+
entry(claim, "CONFORMS", "INFO", false, "nothing was deleted or overwritten", observed)
|
|
16761
|
+
);
|
|
16762
|
+
} else {
|
|
16763
|
+
out.push(
|
|
16764
|
+
entry(
|
|
16765
|
+
claim,
|
|
16766
|
+
"UNDETERMINED",
|
|
16767
|
+
"INFO",
|
|
16768
|
+
false,
|
|
16769
|
+
"RigorRun could not establish whether anything was destroyed",
|
|
16770
|
+
observed
|
|
16771
|
+
)
|
|
16772
|
+
);
|
|
16773
|
+
}
|
|
16774
|
+
}
|
|
16775
|
+
if (hints.idempotent === true) {
|
|
16776
|
+
const claim = "idempotentHint: true";
|
|
16777
|
+
if (observed.idempotent === "no" && canConvict(observed.surfaces)) {
|
|
16778
|
+
out.push(
|
|
16779
|
+
entry(
|
|
16780
|
+
claim,
|
|
16781
|
+
"CONTRADICTED",
|
|
16782
|
+
"MAJOR",
|
|
16783
|
+
false,
|
|
16784
|
+
"a second identical call changed state again, so repeating it is not free",
|
|
16785
|
+
observed
|
|
16786
|
+
)
|
|
16787
|
+
);
|
|
16788
|
+
} else if (observed.idempotent === "yes") {
|
|
16789
|
+
out.push(
|
|
16790
|
+
entry(claim, "CONFORMS", "INFO", false, "a second identical call changed nothing", observed)
|
|
16791
|
+
);
|
|
16792
|
+
} else {
|
|
16793
|
+
out.push(
|
|
16794
|
+
entry(
|
|
16795
|
+
claim,
|
|
16796
|
+
"UNDETERMINED",
|
|
16797
|
+
"INFO",
|
|
16798
|
+
false,
|
|
16799
|
+
"the tool was not called twice, so idempotence was not tested",
|
|
16800
|
+
observed
|
|
16801
|
+
)
|
|
16802
|
+
);
|
|
16803
|
+
}
|
|
16804
|
+
}
|
|
16805
|
+
return out;
|
|
16806
|
+
}
|
|
16807
|
+
|
|
16808
|
+
// ../conformance/src/exit.ts
|
|
16809
|
+
var DEFAULT_THRESHOLDS = {
|
|
16810
|
+
maxUndetermined: 0,
|
|
16811
|
+
minExercised: 1,
|
|
16812
|
+
strict: false
|
|
16813
|
+
};
|
|
16814
|
+
var BLOCKING = /* @__PURE__ */ new Set(["MAJOR", "CRITICAL"]);
|
|
16815
|
+
function exitCodeFor(record, thresholds = DEFAULT_THRESHOLDS) {
|
|
16816
|
+
const contradicted = record.tools.flatMap(
|
|
16817
|
+
(tool) => tool.conformance.filter((c2) => c2.verdict === "CONTRADICTED")
|
|
16818
|
+
);
|
|
16819
|
+
const blocking = contradicted.filter(
|
|
16820
|
+
(c2) => BLOCKING.has(c2.severity) || thresholds.strict && c2.severity === "MINOR"
|
|
16821
|
+
);
|
|
16822
|
+
if (blocking.length > 0) return 1;
|
|
16823
|
+
if (record.summary.toolsExercised < thresholds.minExercised) return 3;
|
|
16824
|
+
if (record.summary.undetermined > thresholds.maxUndetermined) return 3;
|
|
16825
|
+
return 0;
|
|
16826
|
+
}
|
|
16827
|
+
function exitReason(code, record) {
|
|
16828
|
+
switch (code) {
|
|
16829
|
+
case 0: {
|
|
16830
|
+
const minor = record.tools.flatMap(
|
|
16831
|
+
(t) => t.conformance.filter((c2) => c2.verdict === "CONTRADICTED")
|
|
16832
|
+
).length;
|
|
16833
|
+
return minor === 0 ? `${record.summary.toolsExercised} tool(s) exercised, no declaration contradicted.` : `${record.summary.toolsExercised} tool(s) exercised. ${minor} declaration(s) contradicted, none of them blocking; --strict would fail on these.`;
|
|
16834
|
+
}
|
|
16835
|
+
case 1: {
|
|
16836
|
+
const n = record.tools.flatMap(
|
|
16837
|
+
(t) => t.conformance.filter((c2) => c2.verdict === "CONTRADICTED")
|
|
16838
|
+
).length;
|
|
16839
|
+
return `${n} declaration(s) contradicted by observed behaviour.`;
|
|
16840
|
+
}
|
|
16841
|
+
case 3:
|
|
16842
|
+
return `${record.summary.undetermined} finding(s) undetermined and ${record.summary.toolsExercised} tool(s) exercised. Verification completed but did not establish enough to be worth much.`;
|
|
16843
|
+
default:
|
|
16844
|
+
return "The verification could not be run.";
|
|
16845
|
+
}
|
|
16846
|
+
}
|
|
16847
|
+
|
|
16848
|
+
// ../sandbox/src/verify.ts
|
|
16849
|
+
var VerifyError = class extends Error {
|
|
16850
|
+
};
|
|
16851
|
+
var HARNESS_VERSION = RIGORRUN_VERSION;
|
|
16852
|
+
function reading(surface, dump) {
|
|
16853
|
+
return { surface, entries: parseStatedump(dump), readable: dump.length > 0 };
|
|
16854
|
+
}
|
|
16855
|
+
async function openInContainer(session, timeoutMs) {
|
|
16856
|
+
return McpConnection.open(
|
|
16857
|
+
{
|
|
16858
|
+
transport: "stdio",
|
|
16859
|
+
command: "docker",
|
|
16860
|
+
args: session.argv,
|
|
16861
|
+
// The docker *client* needs to know which daemon to talk to. The
|
|
16862
|
+
// container itself gets no environment at all — there is no -e flag in
|
|
16863
|
+
// the argv, and that is deliberate.
|
|
16864
|
+
env: dockerEndpointEnv()
|
|
16865
|
+
},
|
|
16866
|
+
{ timeoutMs }
|
|
16867
|
+
);
|
|
16868
|
+
}
|
|
16869
|
+
function dockerEndpointEnv() {
|
|
16870
|
+
const out = {};
|
|
16871
|
+
for (const name of ["DOCKER_HOST", "DOCKER_CONTEXT", "DOCKER_CERT_PATH", "DOCKER_TLS_VERIFY"]) {
|
|
16872
|
+
const value = process.env[name];
|
|
16873
|
+
if (value !== void 0) out[name] = value;
|
|
16874
|
+
}
|
|
16875
|
+
return out;
|
|
16876
|
+
}
|
|
16877
|
+
async function proveIsolation(image, limits, rounds, entry2) {
|
|
16878
|
+
const digests = [];
|
|
16879
|
+
const pathSets = [];
|
|
16880
|
+
for (let i = 0; i < rounds; i += 1) {
|
|
16881
|
+
const session = new ContainerSession({ image, limits, command: [entry2] });
|
|
16882
|
+
let connection;
|
|
16883
|
+
try {
|
|
16884
|
+
connection = await openInContainer(session, 6e4);
|
|
16885
|
+
const dump = await session.statedump();
|
|
16886
|
+
const parsed = parseStatedump(dump);
|
|
16887
|
+
digests.push(await hashValue([...parsed.entries()].sort()));
|
|
16888
|
+
pathSets.push([...parsed.keys()].sort());
|
|
16889
|
+
} finally {
|
|
16890
|
+
await connection?.close();
|
|
16891
|
+
await session.remove();
|
|
16892
|
+
}
|
|
16893
|
+
}
|
|
16894
|
+
const stable = digests.every((d) => d === digests[0]);
|
|
16895
|
+
const first = pathSets[0] ?? [];
|
|
16896
|
+
const pathsStable = pathSets.every(
|
|
16897
|
+
(set) => set.length === first.length && set.every((p, i) => p === first[i])
|
|
16898
|
+
);
|
|
16899
|
+
return { digests, stable, pathsStable, unstable: stable ? [] : first.slice(0, 0) };
|
|
16900
|
+
}
|
|
16901
|
+
async function verifyServer(reference, options = {}) {
|
|
16902
|
+
const startedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
16903
|
+
const progress = options.onProgress ?? (() => void 0);
|
|
16904
|
+
const limits = options.limits ?? DEFAULT_LIMITS;
|
|
16905
|
+
const seed = options.seed ?? "rigorrun-v2";
|
|
16906
|
+
const ref2 = parseReference(reference);
|
|
16907
|
+
const runtime = await inspectRuntime();
|
|
16908
|
+
if (!runtime.available) {
|
|
16909
|
+
throw new VerifyError(
|
|
16910
|
+
`RigorRun needs a container runtime to verify a server, and could not reach one.
|
|
16911
|
+
${runtime.detail}
|
|
16912
|
+
Install Docker or Podman and start it, then run \`rigorrun doctor\` to check. This is a configuration problem, not a finding about the server.`
|
|
16913
|
+
);
|
|
16914
|
+
}
|
|
16915
|
+
progress("resolve", ref2.raw);
|
|
16916
|
+
let staged;
|
|
16917
|
+
let digest;
|
|
16918
|
+
let resolvedVersion;
|
|
16919
|
+
let tarballUrl = "";
|
|
16920
|
+
if (ref2.scheme === "npm") {
|
|
16921
|
+
const pkg = await resolvePackage(ref2.name, ref2.version);
|
|
16922
|
+
resolvedVersion = pkg.version;
|
|
16923
|
+
digest = pkg.integrity;
|
|
16924
|
+
tarballUrl = pkg.tarballUrl;
|
|
16925
|
+
progress("fetch", `${pkg.name}@${pkg.version}`);
|
|
16926
|
+
await fetchVerifiedTarball(pkg);
|
|
16927
|
+
progress("stage", "installing with lifecycle scripts disabled");
|
|
16928
|
+
staged = await stageNpmPackage(pkg.name, pkg.version);
|
|
16929
|
+
} else {
|
|
16930
|
+
progress("stage", ref2.name);
|
|
16931
|
+
staged = await stageDirectory(ref2.name);
|
|
16932
|
+
resolvedVersion = "local";
|
|
16933
|
+
digest = `dirhash:${staged.contentDigest}`;
|
|
16934
|
+
}
|
|
16935
|
+
const imageTag = `rigorrun-target:${prefixedId("t").replace(/[^a-z0-9]/gi, "").slice(0, 12)}`;
|
|
16936
|
+
const cases = [];
|
|
16937
|
+
const toolRecords = [];
|
|
16938
|
+
const untested = [];
|
|
16939
|
+
let readonlyVerified;
|
|
16940
|
+
let serverInfo;
|
|
16941
|
+
try {
|
|
16942
|
+
progress("image", BASE_IMAGE_TAG);
|
|
16943
|
+
await pullImage(BASE_IMAGE_TAG).catch(() => void 0);
|
|
16944
|
+
const baseDigest = await imageDigest(BASE_IMAGE_TAG) || BASE_IMAGE_TAG;
|
|
16945
|
+
await buildImage(staged.dir, imageTag);
|
|
16946
|
+
progress("isolation", "measuring whether reset actually resets");
|
|
16947
|
+
const proof = await proveIsolation(imageTag, limits, options.resetProofs ?? 2, staged.entryModule);
|
|
16948
|
+
progress("launch", "starting the server");
|
|
16949
|
+
const session = new ContainerSession({ image: imageTag, limits, command: [staged.entryModule] });
|
|
16950
|
+
let connection;
|
|
16951
|
+
let tools = [];
|
|
16952
|
+
try {
|
|
16953
|
+
connection = await openInContainer(session, 12e4);
|
|
16954
|
+
tools = [...connection.discovery.tools];
|
|
16955
|
+
serverInfo = {
|
|
16956
|
+
name: connection.discovery.serverName,
|
|
16957
|
+
version: connection.discovery.serverVersion,
|
|
16958
|
+
protocolVersion: connection.discovery.protocolVersion
|
|
16959
|
+
};
|
|
16960
|
+
progress("discover", `${tools.length} tool(s)`);
|
|
16961
|
+
const layer = await session.layerChanges();
|
|
16962
|
+
readonlyVerified = layer.length === 0;
|
|
16963
|
+
} finally {
|
|
16964
|
+
await connection?.close();
|
|
16965
|
+
await session.remove();
|
|
16966
|
+
}
|
|
16967
|
+
const ordered = [...tools].sort((a, b) => {
|
|
16968
|
+
const rank2 = (t) => t.hints.readOnly === true ? 0 : 1;
|
|
16969
|
+
return rank2(a) - rank2(b) || a.name.localeCompare(b.name);
|
|
16970
|
+
});
|
|
16971
|
+
for (const tool of ordered) {
|
|
16972
|
+
const plan = planTool({
|
|
16973
|
+
name: tool.name,
|
|
16974
|
+
inputSchema: tool.inputSchema ?? {},
|
|
16975
|
+
hints: tool.hints,
|
|
16976
|
+
seed,
|
|
16977
|
+
...options.needsCredential ? { needsCredential: options.needsCredential } : {}
|
|
16978
|
+
});
|
|
16979
|
+
if (plan.class !== "SAFE_AUTOMATIC") {
|
|
16980
|
+
untested.push({ tool: tool.name, reason: plan.class, detail: plan.reason });
|
|
16981
|
+
toolRecords.push({
|
|
16982
|
+
name: tool.name,
|
|
16983
|
+
description: tool.description,
|
|
16984
|
+
declared: declaredOf(tool),
|
|
16985
|
+
observed: {
|
|
16986
|
+
mutatesState: "undetermined",
|
|
16987
|
+
surfacesTouched: [],
|
|
16988
|
+
wrote: [],
|
|
16989
|
+
sideEffects: [],
|
|
16990
|
+
idempotent: "not-probed",
|
|
16991
|
+
errorBehavior: []
|
|
16992
|
+
},
|
|
16993
|
+
// Deliberately empty. A tool RigorRun chose not to call has not been
|
|
16994
|
+
// found undetermined about anything -- it is in `untested`, with the
|
|
16995
|
+
// reason, and that is the honest place for it. Emitting a verdict per
|
|
16996
|
+
// annotation here would make declining to touch a destructive tool
|
|
16997
|
+
// look like three inconclusive findings.
|
|
16998
|
+
conformance: [],
|
|
16999
|
+
verification: "OBSERVATIONAL"
|
|
17000
|
+
});
|
|
17001
|
+
continue;
|
|
17002
|
+
}
|
|
17003
|
+
progress("exercise", tool.name);
|
|
17004
|
+
const observation = await exerciseTool(tool, plan.args, plan.derivation, {
|
|
17005
|
+
image: imageTag,
|
|
17006
|
+
limits,
|
|
17007
|
+
entry: staged.entryModule,
|
|
17008
|
+
cases
|
|
17009
|
+
});
|
|
17010
|
+
if (observation.refusedWithoutEffect) {
|
|
17011
|
+
untested.push({
|
|
17012
|
+
tool: tool.name,
|
|
17013
|
+
reason: "NEEDS_FIXTURE",
|
|
17014
|
+
detail: observation.errorBehavior[0] ?? "the tool refused generated arguments and changed nothing"
|
|
17015
|
+
});
|
|
17016
|
+
toolRecords.push({
|
|
17017
|
+
name: tool.name,
|
|
17018
|
+
description: tool.description,
|
|
17019
|
+
declared: declaredOf(tool),
|
|
17020
|
+
observed: {
|
|
17021
|
+
mutatesState: "undetermined",
|
|
17022
|
+
surfacesTouched: [],
|
|
17023
|
+
wrote: [],
|
|
17024
|
+
sideEffects: [],
|
|
17025
|
+
idempotent: "not-probed",
|
|
17026
|
+
errorBehavior: observation.errorBehavior
|
|
17027
|
+
},
|
|
17028
|
+
conformance: [],
|
|
17029
|
+
verification: "OBSERVATIONAL"
|
|
17030
|
+
});
|
|
17031
|
+
continue;
|
|
17032
|
+
}
|
|
17033
|
+
toolRecords.push({
|
|
17034
|
+
name: tool.name,
|
|
17035
|
+
description: tool.description,
|
|
17036
|
+
declared: declaredOf(tool),
|
|
17037
|
+
observed: {
|
|
17038
|
+
mutatesState: observation.behavior.mutatesState,
|
|
17039
|
+
surfacesTouched: observation.behavior.surfaces,
|
|
17040
|
+
wrote: observation.wrote,
|
|
17041
|
+
sideEffects: observation.sideEffects,
|
|
17042
|
+
idempotent: observation.behavior.idempotent,
|
|
17043
|
+
errorBehavior: observation.errorBehavior
|
|
17044
|
+
},
|
|
17045
|
+
conformance: judge(tool.hints, observation.behavior),
|
|
17046
|
+
// The strongest honest label: the filesystem enumeration is complete
|
|
17047
|
+
// over durable state but blind to memory, so it is PARTIAL.
|
|
17048
|
+
verification: observation.behavior.surfaces.includes("container_fs") ? "PARTIAL" : "OBSERVATIONAL"
|
|
17049
|
+
});
|
|
17050
|
+
}
|
|
17051
|
+
const conformance = toolRecords.flatMap((t) => t.conformance);
|
|
17052
|
+
const record = VerificationRecordSchema.parse({
|
|
17053
|
+
schema: VERIFICATION_RECORD_SCHEMA,
|
|
17054
|
+
schemaVersion: RECORD_SCHEMA_VERSION,
|
|
17055
|
+
recordId: prefixedId("rec"),
|
|
17056
|
+
target: {
|
|
17057
|
+
ref: ref2.raw,
|
|
17058
|
+
scheme: ref2.scheme,
|
|
17059
|
+
resolvedVersion,
|
|
17060
|
+
digest,
|
|
17061
|
+
treeDigest: staged.treeDigest,
|
|
17062
|
+
tarballUrl,
|
|
17063
|
+
fetchedAt: startedAt
|
|
17064
|
+
},
|
|
17065
|
+
harness: {
|
|
17066
|
+
version: HARNESS_VERSION,
|
|
17067
|
+
runtime: `docker ${runtime.version}`,
|
|
17068
|
+
rootless: runtime.rootless,
|
|
17069
|
+
baseImage: baseDigest,
|
|
17070
|
+
imageId: imageTag,
|
|
17071
|
+
isolation: proof.stable ? "RESET" : proof.pathsStable ? "PARTIAL" : "NONE",
|
|
17072
|
+
isolationProof: {
|
|
17073
|
+
resets: proof.digests.length,
|
|
17074
|
+
initialDigests: proof.digests,
|
|
17075
|
+
unstablePaths: proof.unstable
|
|
17076
|
+
},
|
|
17077
|
+
networkEgress: false,
|
|
17078
|
+
readonlyVerified,
|
|
17079
|
+
installScripts: "skipped",
|
|
17080
|
+
limits: { pids: limits.pids, memoryMb: limits.memoryMb, cpus: limits.cpus },
|
|
17081
|
+
runArgv: new ContainerSession({ image: imageTag, limits }).argv,
|
|
17082
|
+
seed,
|
|
17083
|
+
caveats: [
|
|
17084
|
+
...runtime.rootless ? [] : [ROOTFUL_DAEMON_CAVEAT],
|
|
17085
|
+
SERVER_READS_CAVEAT,
|
|
17086
|
+
"State is read from the container\u2019s writable mounts, which is complete over durable storage and blind to state a process holds only in memory."
|
|
17087
|
+
]
|
|
17088
|
+
},
|
|
17089
|
+
server: serverInfo,
|
|
17090
|
+
tools: toolRecords,
|
|
17091
|
+
untested,
|
|
17092
|
+
cases,
|
|
17093
|
+
summary: {
|
|
17094
|
+
toolsDiscovered: tools.length,
|
|
17095
|
+
toolsExercised: toolRecords.filter((t) => t.conformance.length > 0).length,
|
|
17096
|
+
conforms: conformance.filter((c2) => c2.verdict === "CONFORMS").length,
|
|
17097
|
+
contradicted: conformance.filter((c2) => c2.verdict === "CONTRADICTED").length,
|
|
17098
|
+
undetermined: conformance.filter((c2) => c2.verdict === "UNDETERMINED").length,
|
|
17099
|
+
untested: untested.length
|
|
17100
|
+
},
|
|
17101
|
+
startedAt,
|
|
17102
|
+
finishedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
17103
|
+
});
|
|
17104
|
+
return { ...record, recordHash: await hashValue({ ...record, recordHash: "" }) };
|
|
17105
|
+
} finally {
|
|
17106
|
+
await removeImage(imageTag).catch(() => void 0);
|
|
17107
|
+
await staged.cleanup().catch(() => void 0);
|
|
17108
|
+
await rm5(staged.dir, { recursive: true, force: true }).catch(() => void 0);
|
|
17109
|
+
}
|
|
17110
|
+
}
|
|
17111
|
+
function declaredOf(tool) {
|
|
17112
|
+
const { readOnly, destructive, idempotent, openWorld } = tool.hints;
|
|
17113
|
+
return {
|
|
17114
|
+
...readOnly !== void 0 ? { readOnlyHint: readOnly } : {},
|
|
17115
|
+
...destructive !== void 0 ? { destructiveHint: destructive } : {},
|
|
17116
|
+
...idempotent !== void 0 ? { idempotentHint: idempotent } : {},
|
|
17117
|
+
...openWorld !== void 0 ? { openWorldHint: openWorld } : {}
|
|
17118
|
+
};
|
|
17119
|
+
}
|
|
17120
|
+
async function exerciseTool(tool, args, derivation, ctx) {
|
|
17121
|
+
const session = new ContainerSession({
|
|
17122
|
+
image: ctx.image,
|
|
17123
|
+
limits: ctx.limits,
|
|
17124
|
+
command: [ctx.entry]
|
|
17125
|
+
});
|
|
17126
|
+
let connection;
|
|
17127
|
+
const errorBehavior = [];
|
|
17128
|
+
try {
|
|
17129
|
+
connection = await openInContainer(session, 12e4);
|
|
17130
|
+
const before = reading("container_fs", await session.statedump());
|
|
17131
|
+
const firstId = `case:${ctx.cases.length + 1}`;
|
|
17132
|
+
const first = await connection.call(tool.name, args, 6e4);
|
|
17133
|
+
const afterFirst = reading("container_fs", await session.statedump());
|
|
17134
|
+
const delta2 = diffReadings(before, afterFirst);
|
|
17135
|
+
if (!first.ok) errorBehavior.push(first.error?.message ?? "the call returned an error");
|
|
17136
|
+
ctx.cases.push({
|
|
17137
|
+
id: firstId,
|
|
17138
|
+
tool: tool.name,
|
|
17139
|
+
args,
|
|
17140
|
+
derivation,
|
|
17141
|
+
ok: first.ok,
|
|
17142
|
+
isError: !first.ok,
|
|
17143
|
+
errorText: first.error?.message ?? "",
|
|
17144
|
+
beforeDigest: await hashValue([...before.entries.entries()].sort()),
|
|
17145
|
+
afterDigest: await hashValue([...afterFirst.entries.entries()].sort()),
|
|
17146
|
+
changedPaths: touchedPaths(delta2),
|
|
17147
|
+
durationMs: first.durationMs,
|
|
17148
|
+
stderrExcerpt: ""
|
|
17149
|
+
});
|
|
17150
|
+
let idempotent = "not-probed";
|
|
17151
|
+
if (first.ok) {
|
|
17152
|
+
const second = await connection.call(tool.name, args, 6e4);
|
|
17153
|
+
const afterSecond = reading("container_fs", await session.statedump());
|
|
17154
|
+
const secondDelta = diffReadings(afterFirst, afterSecond);
|
|
17155
|
+
idempotent = isEmpty(secondDelta) ? "yes" : "no";
|
|
17156
|
+
ctx.cases.push({
|
|
17157
|
+
id: `case:${ctx.cases.length + 1}`,
|
|
17158
|
+
tool: tool.name,
|
|
17159
|
+
args,
|
|
17160
|
+
derivation: ["repeat of the previous case, to test idempotence"],
|
|
17161
|
+
ok: second.ok,
|
|
17162
|
+
isError: !second.ok,
|
|
17163
|
+
errorText: second.error?.message ?? "",
|
|
17164
|
+
beforeDigest: await hashValue([...afterFirst.entries.entries()].sort()),
|
|
17165
|
+
afterDigest: await hashValue([...afterSecond.entries.entries()].sort()),
|
|
17166
|
+
changedPaths: touchedPaths(secondDelta),
|
|
17167
|
+
durationMs: second.durationMs,
|
|
17168
|
+
stderrExcerpt: ""
|
|
17169
|
+
});
|
|
17170
|
+
}
|
|
17171
|
+
const readable = before.readable && afterFirst.readable;
|
|
17172
|
+
const surfaces = readable ? ["container_fs"] : [];
|
|
17173
|
+
return {
|
|
17174
|
+
behavior: {
|
|
17175
|
+
// Unreadable is undetermined. It is never quietly turned into "no".
|
|
17176
|
+
mutatesState: !readable ? "undetermined" : isEmpty(delta2) ? "no" : "yes",
|
|
17177
|
+
destroyed: !readable ? "undetermined" : destroyedAnything(delta2) ? "yes" : "no",
|
|
17178
|
+
idempotent,
|
|
17179
|
+
surfaces,
|
|
17180
|
+
evidenceRef: firstId
|
|
17181
|
+
},
|
|
17182
|
+
wrote: touchedPaths(delta2),
|
|
17183
|
+
sideEffects: [],
|
|
17184
|
+
errorBehavior,
|
|
17185
|
+
refusedWithoutEffect: !first.ok && isEmpty(delta2)
|
|
17186
|
+
};
|
|
17187
|
+
} finally {
|
|
17188
|
+
await connection?.close();
|
|
17189
|
+
await session.remove();
|
|
17190
|
+
}
|
|
17191
|
+
}
|
|
17192
|
+
|
|
15643
17193
|
// src/doctor.ts
|
|
15644
17194
|
async function cmdDoctor(flags) {
|
|
15645
17195
|
const checks = [];
|
|
@@ -15674,6 +17224,12 @@ async function cmdDoctor(flags) {
|
|
|
15674
17224
|
ok: "unknown",
|
|
15675
17225
|
detail: backend ? `${backend.kind} \u2014 ${backend.detail}` : "could not be determined"
|
|
15676
17226
|
});
|
|
17227
|
+
const runtime = await inspectRuntime();
|
|
17228
|
+
checks.push({
|
|
17229
|
+
what: "container runtime",
|
|
17230
|
+
ok: runtime.available ? true : "unknown",
|
|
17231
|
+
detail: runtime.available ? `docker ${runtime.version}${runtime.rootless ? " (rootless)" : " \u2014 daemon runs as root, so an escape reaches this host"}` : `none \u2014 \`rigorrun verify\` needs one. ${runtime.detail}`
|
|
17232
|
+
});
|
|
15677
17233
|
const proxy = new ProxyServer();
|
|
15678
17234
|
try {
|
|
15679
17235
|
const port = await proxy.start();
|
|
@@ -15764,7 +17320,7 @@ async function cmdDoctor(flags) {
|
|
|
15764
17320
|
if (flags.json) {
|
|
15765
17321
|
line(JSON.stringify({ machine: checks, projects: perProject }, null, 2));
|
|
15766
17322
|
}
|
|
15767
|
-
const failed = [...checks, ...perProject.flatMap((
|
|
17323
|
+
const failed = [...checks, ...perProject.flatMap((entry2) => entry2.checks)].some(
|
|
15768
17324
|
(check) => check.ok === false
|
|
15769
17325
|
);
|
|
15770
17326
|
return failed ? 2 : 0;
|
|
@@ -15781,7 +17337,7 @@ function printChecks(checks) {
|
|
|
15781
17337
|
}
|
|
15782
17338
|
|
|
15783
17339
|
// src/feedback.ts
|
|
15784
|
-
import { writeFile as
|
|
17340
|
+
import { writeFile as writeFile4 } from "node:fs/promises";
|
|
15785
17341
|
async function cmdFeedbackExport(flags) {
|
|
15786
17342
|
const store = new ProjectStore(storeRoot(flags.home));
|
|
15787
17343
|
const bundle = await buildFeedbackBundle({ store, version: VERSION });
|
|
@@ -15795,7 +17351,7 @@ async function cmdFeedbackExport(flags) {
|
|
|
15795
17351
|
const json = `${JSON.stringify(bundle, null, 2)}
|
|
15796
17352
|
`;
|
|
15797
17353
|
if (flags.out) {
|
|
15798
|
-
await
|
|
17354
|
+
await writeFile4(flags.out, json, { mode: 384 });
|
|
15799
17355
|
heading("Feedback bundle");
|
|
15800
17356
|
line(`Written to ${flags.out}`);
|
|
15801
17357
|
line();
|
|
@@ -15807,6 +17363,92 @@ async function cmdFeedbackExport(flags) {
|
|
|
15807
17363
|
return 0;
|
|
15808
17364
|
}
|
|
15809
17365
|
|
|
17366
|
+
// src/verify.ts
|
|
17367
|
+
import { writeFile as writeFile5, mkdir as mkdir7 } from "node:fs/promises";
|
|
17368
|
+
import { dirname as dirname4, join as join14 } from "node:path";
|
|
17369
|
+
var RUNTIME_FAILURES = [ReferenceError, RegistryError, DockerError, StageError, VerifyError];
|
|
17370
|
+
function isRuntimeFailure(error) {
|
|
17371
|
+
return RUNTIME_FAILURES.some((type) => error instanceof type);
|
|
17372
|
+
}
|
|
17373
|
+
async function cmdVerify(reference, flags) {
|
|
17374
|
+
if (!reference) {
|
|
17375
|
+
throw new CliError(
|
|
17376
|
+
"Give a server to verify, for example:\n rigorrun verify npm:@modelcontextprotocol/server-memory@2026.8.31"
|
|
17377
|
+
);
|
|
17378
|
+
}
|
|
17379
|
+
await reapOrphans2().catch(() => []);
|
|
17380
|
+
let record;
|
|
17381
|
+
try {
|
|
17382
|
+
record = await verifyServer(reference, {
|
|
17383
|
+
...flags.quiet || flags.json ? {} : { onProgress: (phase, detail) => line(` ${phase.padEnd(10)} ${detail}`) }
|
|
17384
|
+
});
|
|
17385
|
+
} catch (error) {
|
|
17386
|
+
if (isRuntimeFailure(error)) {
|
|
17387
|
+
throw new CliError(error.message, 2);
|
|
17388
|
+
}
|
|
17389
|
+
throw error;
|
|
17390
|
+
}
|
|
17391
|
+
const outPath = flags.out ?? join14(workspaceDir(), "records", `${record.recordId}.json`);
|
|
17392
|
+
await mkdir7(dirname4(outPath), { recursive: true });
|
|
17393
|
+
await writeFile5(outPath, `${canonicalJson(record)}
|
|
17394
|
+
`, "utf8");
|
|
17395
|
+
const code = exitCodeFor(record, {
|
|
17396
|
+
...DEFAULT_THRESHOLDS,
|
|
17397
|
+
...flags.maxUndetermined !== void 0 ? { maxUndetermined: flags.maxUndetermined } : {},
|
|
17398
|
+
...flags.minExercised !== void 0 ? { minExercised: flags.minExercised } : {},
|
|
17399
|
+
strict: flags.strict
|
|
17400
|
+
});
|
|
17401
|
+
if (flags.json) {
|
|
17402
|
+
process.stdout.write(`${JSON.stringify(record, null, 2)}
|
|
17403
|
+
`);
|
|
17404
|
+
return code;
|
|
17405
|
+
}
|
|
17406
|
+
render(record, outPath);
|
|
17407
|
+
line("");
|
|
17408
|
+
(code === 0 ? line : errorLine)(exitReason(code, record));
|
|
17409
|
+
return code;
|
|
17410
|
+
}
|
|
17411
|
+
function render(record, outPath) {
|
|
17412
|
+
line("");
|
|
17413
|
+
line(`${record.target.ref}`);
|
|
17414
|
+
line(` resolved ${record.target.resolvedVersion}`);
|
|
17415
|
+
line(` digest ${record.target.digest}`);
|
|
17416
|
+
line(` runtime ${record.harness.runtime}${record.harness.rootless ? " (rootless)" : ""}`);
|
|
17417
|
+
line(` isolation ${record.harness.isolation}`);
|
|
17418
|
+
line(` egress ${record.harness.networkEgress ? "permitted" : "none"}`);
|
|
17419
|
+
line("");
|
|
17420
|
+
const rows6 = record.tools.flatMap(
|
|
17421
|
+
(tool) => tool.conformance.length === 0 ? [[tool.name, "\u2014", "no annotation to check", tool.verification]] : tool.conformance.map((c2) => [tool.name, c2.claim, c2.verdict, tool.verification])
|
|
17422
|
+
);
|
|
17423
|
+
if (rows6.length > 0) {
|
|
17424
|
+
line("Declared, versus what it did");
|
|
17425
|
+
table(["tool", "declared", "result", "verification"], rows6);
|
|
17426
|
+
}
|
|
17427
|
+
const contradictions = record.tools.flatMap(
|
|
17428
|
+
(tool) => tool.conformance.filter((c2) => c2.verdict === "CONTRADICTED").map((c2) => ({ tool: tool.name, c: c2, wrote: tool.observed.wrote }))
|
|
17429
|
+
);
|
|
17430
|
+
for (const { tool, c: c2, wrote } of contradictions) {
|
|
17431
|
+
line("");
|
|
17432
|
+
errorLine(`${tool} \u2014 ${c2.claim} is CONTRADICTED${c2.permissionRelevant ? ", and that widens what an agent may do without asking" : ""}`);
|
|
17433
|
+
line(` ${c2.because}.`);
|
|
17434
|
+
if (wrote.length > 0) line(` wrote ${wrote.join(", ")}`);
|
|
17435
|
+
}
|
|
17436
|
+
line("");
|
|
17437
|
+
if (record.untested.length === 0) {
|
|
17438
|
+
line("Every discovered tool was exercised.");
|
|
17439
|
+
} else {
|
|
17440
|
+
line("Not exercised");
|
|
17441
|
+
table(
|
|
17442
|
+
["tool", "why"],
|
|
17443
|
+
record.untested.map((u) => [u.tool, `${u.reason} \u2014 ${u.detail}`])
|
|
17444
|
+
);
|
|
17445
|
+
}
|
|
17446
|
+
line("");
|
|
17447
|
+
for (const caveat of record.harness.caveats) line(` \xB7 ${caveat}`);
|
|
17448
|
+
line("");
|
|
17449
|
+
line(`Record written to ${outPath}`);
|
|
17450
|
+
}
|
|
17451
|
+
|
|
15810
17452
|
// src/project.ts
|
|
15811
17453
|
async function withService(home, run) {
|
|
15812
17454
|
const proxy = new ProxyServer();
|
|
@@ -15849,9 +17491,9 @@ async function cmdProjects(flags) {
|
|
|
15849
17491
|
if (broken.length > 0) {
|
|
15850
17492
|
line();
|
|
15851
17493
|
heading(`${broken.length} project(s) could not be read`);
|
|
15852
|
-
for (const
|
|
15853
|
-
line(` ${c.red(
|
|
15854
|
-
line(c.grey(` ${
|
|
17494
|
+
for (const entry2 of broken) {
|
|
17495
|
+
line(` ${c.red(entry2.id)} ${entry2.reason}`);
|
|
17496
|
+
line(c.grey(` ${entry2.detail}`));
|
|
15855
17497
|
}
|
|
15856
17498
|
return 2;
|
|
15857
17499
|
}
|
|
@@ -15863,7 +17505,7 @@ async function cmdProjectRun(projectId, flags) {
|
|
|
15863
17505
|
const project = await service.readProject(projectId).catch(() => {
|
|
15864
17506
|
throw new CliError(`No project "${projectId}" on this machine. Try \`rigorrun projects\`.`);
|
|
15865
17507
|
});
|
|
15866
|
-
const agent = flags.agent[0] ? project.agents.find((
|
|
17508
|
+
const agent = flags.agent[0] ? project.agents.find((entry2) => entry2.id === flags.agent[0] || entry2.name === flags.agent[0]) : project.agents[project.agents.length - 1];
|
|
15867
17509
|
if (!agent) {
|
|
15868
17510
|
throw new CliError(
|
|
15869
17511
|
project.agents.length === 0 ? `${project.name} has no agent connected. Connect one in the interface first.` : `No agent "${flags.agent[0]}" on ${project.name}.`
|
|
@@ -15875,8 +17517,8 @@ async function cmdProjectRun(projectId, flags) {
|
|
|
15875
17517
|
if (comparison2 && comparison2.currentRunId !== comparison2.baselineRunId) {
|
|
15876
17518
|
line();
|
|
15877
17519
|
line(`${c.bold("Against the baseline")} ${comparison2.headline}`);
|
|
15878
|
-
for (const
|
|
15879
|
-
line(` ${c.red("regressed")} ${
|
|
17520
|
+
for (const entry2 of comparison2.regressed.slice(0, 10)) {
|
|
17521
|
+
line(` ${c.red("regressed")} ${entry2.caseName} \u2014 ${entry2.detail}`);
|
|
15880
17522
|
}
|
|
15881
17523
|
}
|
|
15882
17524
|
const score = result.scores[0];
|
|
@@ -15889,7 +17531,7 @@ async function cmdProjectGate(projectId, flags) {
|
|
|
15889
17531
|
const project = await service.readProject(projectId).catch(() => {
|
|
15890
17532
|
throw new CliError(`No project "${projectId}" on this machine.`);
|
|
15891
17533
|
});
|
|
15892
|
-
const agent = flags.agent[0] ? project.agents.find((
|
|
17534
|
+
const agent = flags.agent[0] ? project.agents.find((entry2) => entry2.id === flags.agent[0] || entry2.name === flags.agent[0]) : project.agents[project.agents.length - 1];
|
|
15893
17535
|
if (!agent) throw new CliError(`${project.name} has no agent to gate.`);
|
|
15894
17536
|
const result = await service.runAgent(projectId, agent.id);
|
|
15895
17537
|
const score = result.scores[0];
|
|
@@ -15921,6 +17563,12 @@ async function cmdProjectGate(projectId, flags) {
|
|
|
15921
17563
|
);
|
|
15922
17564
|
line();
|
|
15923
17565
|
line(`${c.grey("verification")} ${result.verification} ${c.grey("isolation")} ${result.isolation}`);
|
|
17566
|
+
if (result.isolation === "DECLARED") {
|
|
17567
|
+
line(
|
|
17568
|
+
c.grey(" your system nominated a reset; RigorRun has not run it twice")
|
|
17569
|
+
);
|
|
17570
|
+
line(c.grey(" and compared, so isolation is believed rather than observed"));
|
|
17571
|
+
}
|
|
15924
17572
|
for (const limit of result.limits) line(`${c.grey("limit")} ${limit.limit}`);
|
|
15925
17573
|
line();
|
|
15926
17574
|
line(failures2.length === 0 ? c.green("PASS") : `${c.red("FAIL")} ${failures2.join("; ")}`);
|
|
@@ -15954,10 +17602,10 @@ async function cmdProjectCompare(projectId, runId, flags) {
|
|
|
15954
17602
|
return 2;
|
|
15955
17603
|
}
|
|
15956
17604
|
line();
|
|
15957
|
-
for (const
|
|
15958
|
-
for (const
|
|
15959
|
-
for (const
|
|
15960
|
-
for (const
|
|
17605
|
+
for (const entry2 of comparison2.regressed) line(` ${c.red("regressed")} ${entry2.caseName} \u2014 ${entry2.detail}`);
|
|
17606
|
+
for (const entry2 of comparison2.improved) line(` ${c.green("improved")} ${entry2.caseName} \u2014 ${entry2.detail}`);
|
|
17607
|
+
for (const entry2 of comparison2.added) line(` ${c.grey("added")} ${entry2.caseName}`);
|
|
17608
|
+
for (const entry2 of comparison2.removed) line(` ${c.grey("removed")} ${entry2.caseName}`);
|
|
15961
17609
|
line();
|
|
15962
17610
|
line(c.grey(`${comparison2.unchanged.length} case(s) unchanged.`));
|
|
15963
17611
|
return comparison2.regressed.length > 0 ? 1 : 0;
|
|
@@ -15968,7 +17616,7 @@ async function cmdSecret(action, name, flags) {
|
|
|
15968
17616
|
const names = (await store.secretNames()).sort();
|
|
15969
17617
|
const backend = await store.secretBackend();
|
|
15970
17618
|
heading("Secrets on this machine");
|
|
15971
|
-
for (const
|
|
17619
|
+
for (const entry2 of names) line(` ${entry2}`);
|
|
15972
17620
|
if (names.length === 0) line(c.grey(" none"));
|
|
15973
17621
|
line();
|
|
15974
17622
|
line(c.grey(`Kept in ${backend.detail}`));
|
|
@@ -16068,10 +17716,17 @@ async function dispatch(argv) {
|
|
|
16068
17716
|
force: { type: "boolean", default: false },
|
|
16069
17717
|
yes: { type: "boolean", default: false },
|
|
16070
17718
|
as: { type: "string" },
|
|
17719
|
+
// A gate on the reference implementation passes by construction. It
|
|
17720
|
+
// stays reachable for "is this suite satisfiable at all", by name.
|
|
17721
|
+
"allow-reference": { type: "boolean", default: false },
|
|
16071
17722
|
"min-success": { type: "string" },
|
|
16072
17723
|
"min-policy": { type: "string" },
|
|
16073
17724
|
"max-policy-violations": { type: "string" },
|
|
16074
17725
|
"max-unsafe": { type: "string" },
|
|
17726
|
+
// verify only.
|
|
17727
|
+
"max-undetermined": { type: "string" },
|
|
17728
|
+
"min-exercised": { type: "string" },
|
|
17729
|
+
strict: { type: "boolean", default: false },
|
|
16075
17730
|
help: { type: "boolean", short: "h", default: false },
|
|
16076
17731
|
version: { type: "boolean", short: "v", default: false }
|
|
16077
17732
|
}
|
|
@@ -16108,6 +17763,7 @@ async function dispatch(argv) {
|
|
|
16108
17763
|
project: values.project,
|
|
16109
17764
|
home: values.home,
|
|
16110
17765
|
baseline: values.baseline,
|
|
17766
|
+
allowReference: values["allow-reference"] ?? false,
|
|
16111
17767
|
minSuccess: rateFlag(values["min-success"], "min-success"),
|
|
16112
17768
|
minPolicy: rateFlag(values["min-policy"], "min-policy"),
|
|
16113
17769
|
maxPolicyViolations: numberFlag(values["max-policy-violations"], "max-policy-violations"),
|
|
@@ -16183,6 +17839,15 @@ async function dispatch(argv) {
|
|
|
16183
17839
|
return cmdInspectEnvironment(target, flags);
|
|
16184
17840
|
case "privacy":
|
|
16185
17841
|
return cmdPrivacyInspect(target, parsed.positionals[2], flags.json);
|
|
17842
|
+
case "verify":
|
|
17843
|
+
return cmdVerify(target, {
|
|
17844
|
+
json: flags.json,
|
|
17845
|
+
quiet: flags.quiet,
|
|
17846
|
+
out: flags.out,
|
|
17847
|
+
maxUndetermined: numberFlag(values["max-undetermined"], "max-undetermined"),
|
|
17848
|
+
minExercised: numberFlag(values["min-exercised"], "min-exercised"),
|
|
17849
|
+
strict: values.strict === true
|
|
17850
|
+
});
|
|
16186
17851
|
case "doctor":
|
|
16187
17852
|
return cmdDoctor(flags);
|
|
16188
17853
|
default:
|