prizmkit 1.1.136 → 1.1.137
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bundled/VERSION.json +3 -3
- package/bundled/dev-pipeline/scripts/generate-bootstrap-prompt.py +0 -1
- package/bundled/dev-pipeline/tests/test_generate_bootstrap_prompt.py +1 -1
- package/bundled/skills/_metadata.json +1 -1
- package/bundled/skills/prizmkit-test/SKILL.md +7 -8
- package/bundled/skills/prizmkit-test/assets/authoritative-records.schema.json +2 -6
- package/bundled/skills/prizmkit-test/assets/evidence-manifest.schema.json +1 -2
- package/bundled/skills/prizmkit-test/assets/evidence-package-template.json +9 -54
- package/bundled/skills/prizmkit-test/references/evidence-protocol.md +6 -6
- package/bundled/skills/prizmkit-test/references/evidence-request-protocol.md +2 -2
- package/bundled/skills/prizmkit-test/references/examples.md +1 -1
- package/bundled/skills/prizmkit-test/references/test-generation-steps.md +2 -2
- package/bundled/skills/prizmkit-test/references/trusted-evidence-execution.md +2 -2
- package/bundled/skills/prizmkit-test/scripts/build_test_evidence.py +56 -107
- package/bundled/skills/prizmkit-test/scripts/validate_test_evidence.py +1 -35
- package/package.json +1 -1
package/bundled/VERSION.json
CHANGED
|
@@ -993,7 +993,6 @@ def validate_feature_test_evidence(project_root, artifact_dir):
|
|
|
993
993
|
return None
|
|
994
994
|
for stage in stages:
|
|
995
995
|
if (not isinstance(stage, dict) or stage.get("status") not in {"complete", "not_applicable"} or
|
|
996
|
-
not _is_sha256(stage.get("input_sha256")) or
|
|
997
996
|
not isinstance(stage.get("outputs"), list)):
|
|
998
997
|
return None
|
|
999
998
|
if (not _is_sha256(manifest.get("working_diff_sha256")) or
|
|
@@ -246,7 +246,7 @@ def write_fresh_structured_evidence(project_root):
|
|
|
246
246
|
"EVIDENCE_VALIDATE",
|
|
247
247
|
]
|
|
248
248
|
for name in stage_names:
|
|
249
|
-
stages.append({"name": name, "status": "complete", "
|
|
249
|
+
stages.append({"name": name, "status": "complete", "outputs": []})
|
|
250
250
|
manifest = {
|
|
251
251
|
"protocol_version": "1.0",
|
|
252
252
|
"evidence_id": evidence_id,
|
|
@@ -13,7 +13,7 @@ This skill orchestrates each target project's native runners and conventions. It
|
|
|
13
13
|
|
|
14
14
|
Choose project-semantic parameters from the target project: framework, commands, cwd, timeout, retry/attempt policy, concurrency, Mock tools, test layers, module boundary, inventory patterns, mutation technology, and environment classification. Do not hard-code ecosystem-specific defaults when the project can decide them.
|
|
15
15
|
|
|
16
|
-
Fix only the protocol mechanisms needed for safe, replayable evidence: project/evidence/request locators, schema-shaped requests, path confinement, real process execution, complete raw capture, runner-generated
|
|
16
|
+
Fix only the protocol mechanisms needed for safe, replayable evidence: project/evidence/request locators, schema-shaped requests, path confinement, real process execution, complete raw capture, runner-generated receipts, append-only history, and resume invalidation. Read `${SKILL_DIR}/references/trusted-evidence-execution.md` before inventory, execution, differential proof, resume, or replay; it defines the trust boundary and builder commands. Read `${SKILL_DIR}/references/evidence-request-protocol.md` when authoring any request or authoritative JSON record; it defines request ownership, structured N/A, and the schema map.
|
|
17
17
|
|
|
18
18
|
## Canonical Evidence Builder Lifecycle
|
|
19
19
|
|
|
@@ -174,15 +174,14 @@ If a valid test reveals a business/contract defect, do not fix production behavi
|
|
|
174
174
|
|
|
175
175
|
### 7. EXECUTE_PROVE
|
|
176
176
|
|
|
177
|
-
Run required layers in plan order through `${SKILL_DIR}/scripts/build_test_evidence.py execute`. Use `test_paths` or `auto_bind=true` for new requests; the builder resolves and persists canonical test IDs before the native command runs. The model selects project-native argv, cwd, timeout, attempts, concurrency, layer, environment classification, and tool probes in schema-shaped requests. The runner performs the actual process execution and appends complete environment/tool versions, raw stdout/stderr hashes, request hash, runner hash, unique receipt
|
|
177
|
+
Run required layers in plan order through `${SKILL_DIR}/scripts/build_test_evidence.py execute`. Use `test_paths` or `auto_bind=true` for new requests; the builder resolves and persists canonical test IDs before the native command runs. The model selects project-native argv, cwd, timeout, attempts, concurrency, layer, environment classification, and tool probes in schema-shaped requests. The runner performs the actual process execution and appends complete environment/tool versions, raw stdout/stderr hashes, request hash, runner hash, and unique receipt to `executions.json`. Do not accept caller-authored execution JSON and do not retry until green.
|
|
178
178
|
|
|
179
179
|
Differentially prove each added/changed necessary behavior test:
|
|
180
180
|
|
|
181
|
-
- Prefer baseline failure plus current success
|
|
182
|
-
- If baseline is inapplicable, use a minimal controlled mutation tied to the same risk
|
|
183
|
-
-
|
|
184
|
-
-
|
|
185
|
-
- Completely remove mutations and verify the isolated current copy is unchanged. Never rebind completed proof to the mutable live project tree.
|
|
181
|
+
- Prefer baseline failure plus current success: for existing behavior, the test fails against the pre-change baseline (via `git worktree`) and passes against the current project.
|
|
182
|
+
- If baseline is inapplicable (new behavior), use a minimal controlled mutation tied to the same risk, applied in a `git worktree`.
|
|
183
|
+
- The current (success) side executes directly in the project root — tests and behavior under proof belong in the main repository, not an isolated copy.
|
|
184
|
+
- Bind proof to runner receipts, baseline commit, and worktree cleanup. Never rebind completed proof to the mutable live project tree.
|
|
186
185
|
- Classify each proof as `PROVEN`, structured `NOT_APPLICABLE`, or `UNPROVEN`.
|
|
187
186
|
|
|
188
187
|
Necessary `UNPROVEN` behavior, required flakiness, unavailable infrastructure after bounded recovery, failed cleanup, unreliable execution, or budget truncation yields `TEST_BLOCKED`.
|
|
@@ -220,7 +219,7 @@ python3 ${SKILL_DIR}/scripts/validate_test_evidence.py \
|
|
|
220
219
|
--project-root <target-project-root>
|
|
221
220
|
```
|
|
222
221
|
|
|
223
|
-
If this pre-check finds errors, fix them in the package before the second pass. The validator checks every authoritative record against its shipped schema, live target inventories, patch/diff binding,
|
|
222
|
+
If this pre-check finds errors, fix them in the package before the second pass. The validator checks every authoritative record against its shipped schema, live target inventories, patch/diff binding, changed-file/module-root/exclusion/Regression Ring cross-links, generated-test snapshots, planned-test execution links, runner-generated receipts/request/raw-output hashes, successful behavior-risk mappings, production/unknown external-target safety, and package integrity.
|
|
224
223
|
|
|
225
224
|
**Second pass (attestation, with `--attest`):**
|
|
226
225
|
|
|
@@ -357,7 +357,7 @@
|
|
|
357
357
|
"executionReceipt": {
|
|
358
358
|
"type": "object",
|
|
359
359
|
"additionalProperties": false,
|
|
360
|
-
"required": ["receipt_format", "runner_path", "runner_sha256", "execution_id", "attempt_index", "request_path", "request_sha256", "purpose", "command", "cwd", "environment", "tool_versions", "layer", "test_ids", "external_targets", "exit_code", "stdout_path", "stderr_path", "stdout_sha256", "stderr_sha256", "selected_execution", "reliable", "started_at", "finished_at"
|
|
360
|
+
"required": ["receipt_format", "runner_path", "runner_sha256", "execution_id", "attempt_index", "request_path", "request_sha256", "purpose", "command", "cwd", "environment", "tool_versions", "layer", "test_ids", "external_targets", "exit_code", "stdout_path", "stderr_path", "stdout_sha256", "stderr_sha256", "selected_execution", "reliable", "started_at", "finished_at"],
|
|
361
361
|
"properties": {
|
|
362
362
|
"receipt_format": { "const": "prizmkit-runner-generated-v1" },
|
|
363
363
|
"runner_path": { "$ref": "#/$defs/path" },
|
|
@@ -385,7 +385,6 @@
|
|
|
385
385
|
"reliable": { "type": "boolean" },
|
|
386
386
|
"started_at": { "type": "string", "minLength": 1 },
|
|
387
387
|
"finished_at": { "type": "string", "minLength": 1 },
|
|
388
|
-
"previous_receipt_sha256": { "type": ["string", "null"] },
|
|
389
388
|
"replay_of": { "type": ["string", "null"] },
|
|
390
389
|
"isolation": { "type": ["object", "null"] }
|
|
391
390
|
}
|
|
@@ -445,7 +444,7 @@
|
|
|
445
444
|
"items": {
|
|
446
445
|
"type": "object",
|
|
447
446
|
"additionalProperties": false,
|
|
448
|
-
"required": ["behavior_id", "classification", "method", "differential_request_path", "differential_request_sha256", "baseline_commit", "baseline_execution_id", "mutation_execution_id", "current_execution_id", "failure_reason_matched", "cleanup_succeeded", "
|
|
447
|
+
"required": ["behavior_id", "classification", "method", "differential_request_path", "differential_request_sha256", "baseline_commit", "baseline_execution_id", "mutation_execution_id", "current_execution_id", "failure_reason_matched", "cleanup_succeeded", "not_applicable"],
|
|
449
448
|
"properties": {
|
|
450
449
|
"behavior_id": { "type": "string", "minLength": 1 },
|
|
451
450
|
"classification": { "enum": ["PROVEN", "NOT_APPLICABLE", "UNPROVEN"] },
|
|
@@ -458,9 +457,6 @@
|
|
|
458
457
|
"current_execution_id": { "type": ["string", "null"] },
|
|
459
458
|
"failure_reason_matched": { "type": "boolean" },
|
|
460
459
|
"cleanup_succeeded": { "type": "boolean" },
|
|
461
|
-
"isolation_tree_sha256": { "type": ["string", "null"] },
|
|
462
|
-
"mutation_apply_sha256": { "type": ["string", "null"] },
|
|
463
|
-
"mutation_restore_sha256": { "type": ["string", "null"] },
|
|
464
460
|
"not_applicable": { "type": ["object", "null"] }
|
|
465
461
|
}
|
|
466
462
|
}
|
|
@@ -67,7 +67,7 @@
|
|
|
67
67
|
"stage": {
|
|
68
68
|
"type": "object",
|
|
69
69
|
"additionalProperties": false,
|
|
70
|
-
"required": ["name", "status", "
|
|
70
|
+
"required": ["name", "status", "outputs"],
|
|
71
71
|
"properties": {
|
|
72
72
|
"name": {
|
|
73
73
|
"enum": [
|
|
@@ -83,7 +83,6 @@
|
|
|
83
83
|
]
|
|
84
84
|
},
|
|
85
85
|
"status": { "enum": ["complete", "not_applicable"] },
|
|
86
|
-
"input_sha256": { "$ref": "#/$defs/sha256" },
|
|
87
86
|
"outputs": { "type": "array", "items": { "type": "string", "minLength": 1 } },
|
|
88
87
|
"not_applicable": {
|
|
89
88
|
"type": ["object", "null"],
|
|
@@ -38,60 +38,15 @@
|
|
|
38
38
|
"contract_snapshots": "contracts/"
|
|
39
39
|
},
|
|
40
40
|
"stage_statuses": [
|
|
41
|
-
{
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
},
|
|
47
|
-
{
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
"input_sha256": "<sha256>",
|
|
51
|
-
"outputs": []
|
|
52
|
-
},
|
|
53
|
-
{
|
|
54
|
-
"name": "CONTRACT_MODEL",
|
|
55
|
-
"status": "pending",
|
|
56
|
-
"input_sha256": "<sha256>",
|
|
57
|
-
"outputs": []
|
|
58
|
-
},
|
|
59
|
-
{
|
|
60
|
-
"name": "TEST_PLAN",
|
|
61
|
-
"status": "pending",
|
|
62
|
-
"input_sha256": "<sha256>",
|
|
63
|
-
"outputs": []
|
|
64
|
-
},
|
|
65
|
-
{
|
|
66
|
-
"name": "INFRA_READY",
|
|
67
|
-
"status": "pending",
|
|
68
|
-
"input_sha256": "<sha256>",
|
|
69
|
-
"outputs": []
|
|
70
|
-
},
|
|
71
|
-
{
|
|
72
|
-
"name": "TEST_BUILD",
|
|
73
|
-
"status": "pending",
|
|
74
|
-
"input_sha256": "<sha256>",
|
|
75
|
-
"outputs": []
|
|
76
|
-
},
|
|
77
|
-
{
|
|
78
|
-
"name": "EXECUTE_PROVE",
|
|
79
|
-
"status": "pending",
|
|
80
|
-
"input_sha256": "<sha256>",
|
|
81
|
-
"outputs": []
|
|
82
|
-
},
|
|
83
|
-
{
|
|
84
|
-
"name": "EVIDENCE_PACKAGE",
|
|
85
|
-
"status": "pending",
|
|
86
|
-
"input_sha256": "<sha256>",
|
|
87
|
-
"outputs": []
|
|
88
|
-
},
|
|
89
|
-
{
|
|
90
|
-
"name": "EVIDENCE_VALIDATE",
|
|
91
|
-
"status": "pending",
|
|
92
|
-
"input_sha256": "<sha256>",
|
|
93
|
-
"outputs": []
|
|
94
|
-
}
|
|
41
|
+
{ "name": "CHANGE_CLASSIFY", "status": "pending", "outputs": [] },
|
|
42
|
+
{ "name": "SCOPE_DISCOVER", "status": "pending", "outputs": [] },
|
|
43
|
+
{ "name": "CONTRACT_MODEL", "status": "pending", "outputs": [] },
|
|
44
|
+
{ "name": "TEST_PLAN", "status": "pending", "outputs": [] },
|
|
45
|
+
{ "name": "INFRA_READY", "status": "pending", "outputs": [] },
|
|
46
|
+
{ "name": "TEST_BUILD", "status": "pending", "outputs": [] },
|
|
47
|
+
{ "name": "EXECUTE_PROVE", "status": "pending", "outputs": [] },
|
|
48
|
+
{ "name": "EVIDENCE_PACKAGE", "status": "pending", "outputs": [] },
|
|
49
|
+
{ "name": "EVIDENCE_VALIDATE", "status": "pending", "outputs": [] }
|
|
95
50
|
],
|
|
96
51
|
"resume": {
|
|
97
52
|
"last_valid_stage": null,
|
|
@@ -41,7 +41,7 @@ Run exactly these states in order:
|
|
|
41
41
|
8. `EVIDENCE_PACKAGE`
|
|
42
42
|
9. `EVIDENCE_VALIDATE`
|
|
43
43
|
|
|
44
|
-
Each manifest stage records `name`, `status`,
|
|
44
|
+
Each manifest stage records `name`, `status`, and output paths. A stage may be `not_applicable` only with a concrete rationale. A later state cannot complete until every applicable predecessor output exists and validates.
|
|
45
45
|
|
|
46
46
|
### CHANGE_CLASSIFY
|
|
47
47
|
|
|
@@ -119,7 +119,7 @@ Run progressively:
|
|
|
119
119
|
|
|
120
120
|
Only omit plan-marked non-required layers. Every N/A is structured with rationale, typed evidence, considered signals, and explanations for detected conflicts. The model retains semantic judgment; validation checks structure and obvious signal conflicts, not hidden intent. Append only runner-generated execution receipts containing:
|
|
121
121
|
|
|
122
|
-
- runner-generated receipt format, runner hash, request path/hash, and
|
|
122
|
+
- runner-generated receipt format, runner hash, request path/hash, and unique execution identifier;
|
|
123
123
|
- unique selected execution identifier;
|
|
124
124
|
- layer and exact argv;
|
|
125
125
|
- confined working directory;
|
|
@@ -133,11 +133,11 @@ Do not overwrite failed or superseded attempts. Do not retry until green. Use a
|
|
|
133
133
|
|
|
134
134
|
For each added or changed necessary behavior test, prove the test:
|
|
135
135
|
|
|
136
|
-
1. Prefer
|
|
137
|
-
2. If baseline execution is inapplicable (
|
|
136
|
+
1. Prefer baseline failure plus current success: for existing behavior, the test fails against the pre-change baseline (via `git worktree`) and passes against the current project.
|
|
137
|
+
2. If baseline execution is inapplicable (new behavior), apply a minimal controlled mutation tied to the same matrix risk in a `git worktree`, observe failure, restore it completely, and observe current success.
|
|
138
138
|
3. Record `PROVEN`, justified `NOT_APPLICABLE`, or `UNPROVEN`.
|
|
139
139
|
|
|
140
|
-
Record baseline/
|
|
140
|
+
Record baseline/mutation runner receipt IDs, current runner receipt IDs, request hash, baseline commit, expected failure signal match, and worktree cleanup result. Necessary `UNPROVEN` behavior prevents `TEST_PASS`.
|
|
141
141
|
|
|
142
142
|
### EVIDENCE_PACKAGE
|
|
143
143
|
|
|
@@ -168,7 +168,7 @@ python3 ${SKILL_DIR}/scripts/validate_test_evidence.py \
|
|
|
168
168
|
|
|
169
169
|
When feasible, replay recorded requests through `build_test_evidence.py execute --replay-receipt ...`. Replay performs a new real execution and emits a new linked receipt; it does not pretend the attestation establishes what historically ran.
|
|
170
170
|
|
|
171
|
-
The validator checks all shipped JSON schemas, evidence-directory identity, live target-project inventories and aggregate hashes, changed-file/module-root/exclusion/Regression Ring cross-links, patch/diff binding, stage
|
|
171
|
+
The validator checks all shipped JSON schemas, evidence-directory identity, live target-project inventories and aggregate hashes, changed-file/module-root/exclusion/Regression Ring cross-links, patch/diff binding, stage ordering and output ownership, generated-test snapshot existence, planned test/inventory/execution linkage, module/matrix completeness, successful selected runner receipts with request/raw-output binding, structured N/A and obvious signal conflicts, production/unknown external-target safety, strict verdict semantics, and honest mocked-versus-real claims. `TEST_PASS` requires deterministic validator success.
|
|
172
172
|
|
|
173
173
|
## Resume and Invalidation
|
|
174
174
|
|
|
@@ -11,7 +11,7 @@ Use existing project-native tests first. The Main Agent supplies only semantic t
|
|
|
11
11
|
`finalize` creates the manifest, stage dependency hashes, verdict, evidence identity, and deterministic report inputs. `render-report` only renders the derived Markdown view. The validator alone creates `validation.json` during `--attest`; never hand-author a validation attestation or rebind a finalized package after identity inputs change.
|
|
12
12
|
|
|
13
13
|
|
|
14
|
-
The model writes project-semantic requests after inspecting manifests, runner configuration, contracts, test conventions, changed files, and module boundaries. The bundled builder owns process execution, raw output, receipts, isolation,
|
|
14
|
+
The model writes project-semantic requests after inspecting manifests, runner configuration, contracts, test conventions, changed files, and module boundaries. The bundled builder owns process execution, raw output, receipts, worktree-based isolation, and invalidation calculations.
|
|
15
15
|
|
|
16
16
|
Never hand-author `executions.json` receipt fields. Never copy a command's claimed output into `raw/`. Invoke the builder so provenance is linked to the actual process.
|
|
17
17
|
|
|
@@ -52,7 +52,7 @@ Record:
|
|
|
52
52
|
- mutation patch path when applicable;
|
|
53
53
|
- concrete expected failure signals.
|
|
54
54
|
|
|
55
|
-
The builder
|
|
55
|
+
The builder uses `git worktree add --detach` for the failing side (baseline commit or controlled mutation) and executes the current side directly in the project root. Differential proofs compare receipt outcomes (exit codes, expected failure signals) between the failing side and the current side. Worktrees are cleaned up with `git worktree remove --force` after execution. Source drift between capture and validation is detected through change-capture freshness checks (inventory changed_files vs canonical capture) and resume invalidation (live target hashes vs manifest).
|
|
56
56
|
|
|
57
57
|
## Structured N/A
|
|
58
58
|
|
|
@@ -71,7 +71,7 @@ Evidence: .prizmkit/test/evidence/<evidence-id>/
|
|
|
71
71
|
|
|
72
72
|
## 6. Baseline Inapplicable — Controlled Mutation
|
|
73
73
|
|
|
74
|
-
A newly added CLI command has no pre-change interface, so baseline execution is justified `NOT_APPLICABLE` as a direct proof method.
|
|
74
|
+
A newly added CLI command has no pre-change interface, so baseline execution is justified `NOT_APPLICABLE` as a direct proof method. The run creates a `git worktree` for the mutation, applies a minimal patch that removes the command registration, observes the command test fail for that risk, cleans up the worktree, and observes current success. The behavior is `PROVEN` by `controlled-mutation`.
|
|
75
75
|
|
|
76
76
|
If restoration cannot be proven, the verdict is `TEST_BLOCKED`.
|
|
77
77
|
|
|
@@ -101,8 +101,8 @@ Run selected tests in order through the bundled evidence builder: focused, modul
|
|
|
101
101
|
|
|
102
102
|
For each new/changed necessary behavior test:
|
|
103
103
|
|
|
104
|
-
1.
|
|
105
|
-
2. If baseline proof is genuinely inapplicable, apply one model-selected minimal controlled mutation tied to the risk
|
|
104
|
+
1. For existing behavior, run it against the pre-change baseline (via `git worktree`) and require failure for the intended recorded signal, then current success against the project root.
|
|
105
|
+
2. If baseline proof is genuinely inapplicable, apply one model-selected minimal controlled mutation tied to the risk in a `git worktree`, require failure, restore and clean up the worktree, and verify current success against the project root.
|
|
106
106
|
3. Record `PROVEN`, structured `NOT_APPLICABLE`, or `UNPROVEN` with runner receipt IDs.
|
|
107
107
|
|
|
108
108
|
Necessary `UNPROVEN` evidence blocks. Store complete stdout/stderr in `raw/`; snapshots in `generated-tests/`; do not overwrite previous attempts.
|
|
@@ -27,9 +27,9 @@ The protocol fixes only what trustworthy, replayable packaging needs:
|
|
|
27
27
|
- actual child-process execution without a shell;
|
|
28
28
|
- complete stdout/stderr capture and output hashes;
|
|
29
29
|
- complete effective environment and tool-probe capture;
|
|
30
|
-
- runner-generated UUID receipt, request hash,
|
|
30
|
+
- runner-generated UUID receipt, request hash, and runner hash;
|
|
31
31
|
- append-only execution history;
|
|
32
|
-
- differential isolation
|
|
32
|
+
- differential isolation via git worktree and cleanup;
|
|
33
33
|
- deterministic resume invalidation calculation;
|
|
34
34
|
- strict schema and cross-record validation.
|
|
35
35
|
|
|
@@ -821,9 +821,6 @@ def append_receipt(evidence_dir: Path, receipt: dict[str, Any]) -> None:
|
|
|
821
821
|
existing: Any = load_json(path) if path.exists() else []
|
|
822
822
|
if not isinstance(existing, list):
|
|
823
823
|
raise RequestError("executions.json must remain an array")
|
|
824
|
-
previous = canonical_sha256(existing[-1]) if existing else None
|
|
825
|
-
if receipt["previous_receipt_sha256"] != previous:
|
|
826
|
-
raise RequestError("receipt chain mismatch")
|
|
827
824
|
existing.append(receipt)
|
|
828
825
|
write_json_atomic(path, existing)
|
|
829
826
|
|
|
@@ -900,7 +897,6 @@ def execute_request(
|
|
|
900
897
|
existing = load_json(evidence_dir / "executions.json") if (evidence_dir / "executions.json").exists() else []
|
|
901
898
|
if not isinstance(existing, list):
|
|
902
899
|
raise RequestError("executions.json must remain an array")
|
|
903
|
-
previous = canonical_sha256(existing[-1]) if existing else None
|
|
904
900
|
same_request_attempts = sum(1 for item in existing if isinstance(item, dict) and item.get("request_sha256") == file_sha256(request_path))
|
|
905
901
|
receipt = {
|
|
906
902
|
"receipt_format": "prizmkit-runner-generated-v1",
|
|
@@ -927,7 +923,6 @@ def execute_request(
|
|
|
927
923
|
"reliable": probes_reliable and not timed_out and not execution_error,
|
|
928
924
|
"started_at": started,
|
|
929
925
|
"finished_at": finished,
|
|
930
|
-
"previous_receipt_sha256": previous,
|
|
931
926
|
"replay_of": replay_of,
|
|
932
927
|
"isolation": isolation,
|
|
933
928
|
}
|
|
@@ -935,47 +930,6 @@ def execute_request(
|
|
|
935
930
|
return receipt
|
|
936
931
|
|
|
937
932
|
|
|
938
|
-
def copy_project_tree(project_root: Path, destination: Path, evidence_dir: Path) -> None:
|
|
939
|
-
def ignore(directory: str, names: list[str]) -> set[str]:
|
|
940
|
-
directory_relative = Path(directory).resolve().relative_to(project_root.resolve())
|
|
941
|
-
ignored: set[str] = set()
|
|
942
|
-
for name in names:
|
|
943
|
-
relative = directory_relative / name
|
|
944
|
-
if any(part in _IGNORED_DIR_NAMES for part in relative.parts):
|
|
945
|
-
ignored.add(name)
|
|
946
|
-
continue
|
|
947
|
-
if any(relative == Path(prefix) or Path(prefix) in relative.parents for prefix in INVENTORY_RESERVED_ROOTS):
|
|
948
|
-
ignored.add(name)
|
|
949
|
-
elif (project_root / relative).resolve() == evidence_dir.resolve() or evidence_dir.resolve() in (project_root / relative).resolve().parents:
|
|
950
|
-
ignored.add(name)
|
|
951
|
-
return ignored
|
|
952
|
-
|
|
953
|
-
shutil.copytree(project_root, destination, ignore=ignore)
|
|
954
|
-
|
|
955
|
-
|
|
956
|
-
def _iso_tree_sha256(root: Path) -> str:
|
|
957
|
-
"""Hash an isolated temp tree (not the live project)."""
|
|
958
|
-
entries: list[dict[str, str]] = []
|
|
959
|
-
for candidate in sorted(root.rglob("*")):
|
|
960
|
-
if not candidate.is_file():
|
|
961
|
-
continue
|
|
962
|
-
entries.append({"path": candidate.relative_to(root).as_posix(), "sha256": file_sha256(candidate)})
|
|
963
|
-
return canonical_sha256(entries)
|
|
964
|
-
|
|
965
|
-
|
|
966
|
-
def git_archive_baseline(project_root: Path, baseline_commit: str, destination: Path) -> None:
|
|
967
|
-
result = subprocess.run(
|
|
968
|
-
["git", "-C", str(project_root), "archive", "--format=tar", baseline_commit],
|
|
969
|
-
capture_output=True, check=False,
|
|
970
|
-
)
|
|
971
|
-
if result.returncode != 0:
|
|
972
|
-
raise RequestError(f"cannot archive baseline commit {baseline_commit}: {result.stderr.decode(errors='replace')}")
|
|
973
|
-
destination.mkdir(parents=True)
|
|
974
|
-
extracted = subprocess.run(["tar", "-xf", "-", "-C", str(destination)], input=result.stdout, capture_output=True, check=False)
|
|
975
|
-
if extracted.returncode != 0:
|
|
976
|
-
raise RequestError(f"cannot extract baseline archive: {extracted.stderr.decode(errors='replace')}")
|
|
977
|
-
|
|
978
|
-
|
|
979
933
|
def materialize_request(evidence_dir: Path, name: str, request: dict[str, Any]) -> Path:
|
|
980
934
|
path = evidence_dir / "requests" / name
|
|
981
935
|
write_json_atomic(path, request)
|
|
@@ -1027,7 +981,6 @@ def run_differential(project_root: Path, evidence_dir: Path, request_path: Path)
|
|
|
1027
981
|
"differential_request_sha256": file_sha256(request_path), "baseline_commit": None,
|
|
1028
982
|
"baseline_execution_id": None, "mutation_execution_id": None,
|
|
1029
983
|
"current_execution_id": None, "failure_reason_matched": False, "cleanup_succeeded": True,
|
|
1030
|
-
"isolation_tree_sha256": None, "mutation_apply_sha256": None, "mutation_restore_sha256": None,
|
|
1031
984
|
"not_applicable": not_applicable,
|
|
1032
985
|
}
|
|
1033
986
|
append_proof(evidence_dir, proof)
|
|
@@ -1051,58 +1004,67 @@ def run_differential(project_root: Path, evidence_dir: Path, request_path: Path)
|
|
|
1051
1004
|
baseline_receipt = None
|
|
1052
1005
|
mutation_receipt = None
|
|
1053
1006
|
current_receipt = None
|
|
1054
|
-
isolation_tree = None
|
|
1055
|
-
mutation_apply_sha = None
|
|
1056
|
-
mutation_restore_sha = None
|
|
1057
1007
|
cleanup_succeeded = False
|
|
1008
|
+
worktree_path = None
|
|
1058
1009
|
try:
|
|
1059
|
-
|
|
1060
|
-
|
|
1061
|
-
|
|
1062
|
-
|
|
1063
|
-
|
|
1064
|
-
|
|
1065
|
-
|
|
1066
|
-
|
|
1067
|
-
|
|
1068
|
-
|
|
1069
|
-
|
|
1070
|
-
|
|
1071
|
-
|
|
1072
|
-
|
|
1073
|
-
|
|
1074
|
-
|
|
1075
|
-
|
|
1076
|
-
|
|
1077
|
-
|
|
1078
|
-
|
|
1079
|
-
|
|
1080
|
-
|
|
1081
|
-
|
|
1082
|
-
raise RequestError("controlled mutation requires mutation_patch_path")
|
|
1083
|
-
patch_path = request_file(evidence_dir, patch_value)
|
|
1084
|
-
applied = subprocess.run(["git", "apply", "--whitespace=nowarn", str(patch_path)], cwd=failing_root, capture_output=True, check=False)
|
|
1085
|
-
if applied.returncode != 0:
|
|
1086
|
-
raise RequestError(f"controlled mutation could not be applied: {applied.stderr.decode(errors='replace')}")
|
|
1087
|
-
mutation_apply_sha = _iso_tree_sha256(failing_root)
|
|
1088
|
-
mutation_receipt = execute_request(
|
|
1089
|
-
project_root, evidence_dir, current_request_path, execution_root=failing_root,
|
|
1090
|
-
isolation={"kind": "controlled-mutation", "tree_sha256": _iso_tree_sha256(failing_root), "patch_sha256": file_sha256(patch_path)},
|
|
1091
|
-
selected_execution=False,
|
|
1092
|
-
)
|
|
1093
|
-
mutation_restore_sha = isolation_tree
|
|
1094
|
-
else:
|
|
1095
|
-
raise RequestError("differential method must be baseline or controlled-mutation")
|
|
1096
|
-
current_receipt = execute_request(
|
|
1097
|
-
project_root, evidence_dir, current_request_path, execution_root=current_root,
|
|
1098
|
-
isolation={"kind": "current", "tree_sha256": isolation_tree},
|
|
1010
|
+
if request["method"] == "baseline":
|
|
1011
|
+
worktree_path = tempfile.mkdtemp(prefix="prizmkit-baseline-")
|
|
1012
|
+
result = subprocess.run(
|
|
1013
|
+
["git", "-C", str(project_root), "worktree", "add", "--detach", str(worktree_path), request["baseline_commit"]],
|
|
1014
|
+
capture_output=True, check=False,
|
|
1015
|
+
)
|
|
1016
|
+
if result.returncode != 0:
|
|
1017
|
+
raise RequestError(f"cannot create baseline worktree: {result.stderr.decode(errors='replace')}")
|
|
1018
|
+
for relative in overlay_paths:
|
|
1019
|
+
source = confined(project_root, relative, must_exist=True)
|
|
1020
|
+
destination = Path(worktree_path) / relative
|
|
1021
|
+
destination.parent.mkdir(parents=True, exist_ok=True)
|
|
1022
|
+
shutil.copy2(source, destination)
|
|
1023
|
+
baseline_receipt = execute_request(
|
|
1024
|
+
project_root, evidence_dir, current_request_path, execution_root=Path(worktree_path),
|
|
1025
|
+
isolation={"kind": "baseline", "baseline_commit": request["baseline_commit"]},
|
|
1026
|
+
selected_execution=False,
|
|
1027
|
+
)
|
|
1028
|
+
elif request["method"] == "controlled-mutation":
|
|
1029
|
+
worktree_path = tempfile.mkdtemp(prefix="prizmkit-mutation-")
|
|
1030
|
+
result = subprocess.run(
|
|
1031
|
+
["git", "-C", str(project_root), "worktree", "add", "--detach", str(worktree_path), "HEAD"],
|
|
1032
|
+
capture_output=True, check=False,
|
|
1099
1033
|
)
|
|
1034
|
+
if result.returncode != 0:
|
|
1035
|
+
raise RequestError(f"cannot create mutation worktree: {result.stderr.decode(errors='replace')}")
|
|
1036
|
+
patch_value = request["mutation_patch_path"]
|
|
1037
|
+
if not isinstance(patch_value, str) or not patch_value:
|
|
1038
|
+
raise RequestError("controlled mutation requires mutation_patch_path")
|
|
1039
|
+
patch_path = request_file(evidence_dir, patch_value)
|
|
1040
|
+
applied = subprocess.run(
|
|
1041
|
+
["git", "apply", "--whitespace=nowarn", str(patch_path)],
|
|
1042
|
+
cwd=worktree_path, capture_output=True, check=False,
|
|
1043
|
+
)
|
|
1044
|
+
if applied.returncode != 0:
|
|
1045
|
+
raise RequestError(f"controlled mutation could not be applied: {applied.stderr.decode(errors='replace')}")
|
|
1046
|
+
mutation_receipt = execute_request(
|
|
1047
|
+
project_root, evidence_dir, current_request_path, execution_root=Path(worktree_path),
|
|
1048
|
+
isolation={"kind": "controlled-mutation", "patch_sha256": file_sha256(patch_path)},
|
|
1049
|
+
selected_execution=False,
|
|
1050
|
+
)
|
|
1051
|
+
else:
|
|
1052
|
+
raise RequestError("differential method must be baseline or controlled-mutation")
|
|
1053
|
+
current_receipt = execute_request(
|
|
1054
|
+
project_root, evidence_dir, current_request_path, execution_root=project_root,
|
|
1055
|
+
isolation={"kind": "project-direct"},
|
|
1056
|
+
)
|
|
1100
1057
|
cleanup_succeeded = True
|
|
1101
1058
|
finally:
|
|
1102
|
-
|
|
1103
|
-
|
|
1104
|
-
|
|
1105
|
-
|
|
1059
|
+
if worktree_path and Path(worktree_path).exists():
|
|
1060
|
+
subprocess.run(
|
|
1061
|
+
["git", "-C", str(project_root), "worktree", "remove", "--force", str(worktree_path)],
|
|
1062
|
+
capture_output=True, check=False,
|
|
1063
|
+
)
|
|
1064
|
+
try:
|
|
1065
|
+
shutil.rmtree(worktree_path, ignore_errors=True)
|
|
1066
|
+
except OSError:
|
|
1067
|
+
pass
|
|
1106
1068
|
failing_receipt = baseline_receipt or mutation_receipt
|
|
1107
1069
|
failing_output = b""
|
|
1108
1070
|
if failing_receipt:
|
|
@@ -1125,9 +1087,6 @@ def run_differential(project_root: Path, evidence_dir: Path, request_path: Path)
|
|
|
1125
1087
|
"current_execution_id": current_receipt["execution_id"] if current_receipt else None,
|
|
1126
1088
|
"failure_reason_matched": failure_reason_matched,
|
|
1127
1089
|
"cleanup_succeeded": cleanup_succeeded,
|
|
1128
|
-
"isolation_tree_sha256": isolation_tree,
|
|
1129
|
-
"mutation_apply_sha256": mutation_apply_sha,
|
|
1130
|
-
"mutation_restore_sha256": mutation_restore_sha,
|
|
1131
1090
|
"not_applicable": None,
|
|
1132
1091
|
}
|
|
1133
1092
|
append_proof(evidence_dir, proof)
|
|
@@ -1481,25 +1440,15 @@ def build_manifest(evidence_dir: Path, *, evidence_id: str, identity: dict[str,
|
|
|
1481
1440
|
outputs_by_stage = {stage: [] for stage in STAGES}
|
|
1482
1441
|
for entry in files:
|
|
1483
1442
|
outputs_by_stage.setdefault(entry["produced_by"], []).append(entry["path"])
|
|
1484
|
-
predecessor_files: list[dict[str, str]] = []
|
|
1485
1443
|
stages = []
|
|
1486
1444
|
for stage in STAGES:
|
|
1487
1445
|
outputs = sorted(outputs_by_stage.get(stage, []))
|
|
1488
1446
|
stages.append({
|
|
1489
1447
|
"name": stage,
|
|
1490
1448
|
"status": "complete",
|
|
1491
|
-
"input_sha256": canonical_sha256({
|
|
1492
|
-
"stage": stage,
|
|
1493
|
-
"target_hashes": target_hashes,
|
|
1494
|
-
"predecessor_outputs": sorted(predecessor_files, key=lambda item: item["path"]),
|
|
1495
|
-
}),
|
|
1496
1449
|
"outputs": outputs,
|
|
1497
1450
|
"not_applicable": None,
|
|
1498
1451
|
})
|
|
1499
|
-
predecessor_files.extend(
|
|
1500
|
-
{"path": output, "sha256": next(item["sha256"] for item in files if item["path"] == output)}
|
|
1501
|
-
for output in outputs if output not in FINAL_RECORDS
|
|
1502
|
-
)
|
|
1503
1452
|
return {
|
|
1504
1453
|
"protocol_version": "1.0",
|
|
1505
1454
|
"evidence_id": evidence_id,
|
|
@@ -445,14 +445,6 @@ def validate_external_targets(value: Any, location: str, errors: list[str]) -> N
|
|
|
445
445
|
require(isinstance(deny_evidence, list) and not deny_evidence, f"{target_location}: external target has deny evidence", errors)
|
|
446
446
|
|
|
447
447
|
|
|
448
|
-
def stage_input_sha(stage_name: str, target_hashes: dict[str, str], predecessor_files: list[dict[str, str]]) -> str:
|
|
449
|
-
return canonical_sha256({
|
|
450
|
-
"stage": stage_name,
|
|
451
|
-
"target_hashes": target_hashes,
|
|
452
|
-
"predecessor_outputs": sorted(predecessor_files, key=lambda item: item["path"]),
|
|
453
|
-
})
|
|
454
|
-
|
|
455
|
-
|
|
456
448
|
def validate_manifest(
|
|
457
449
|
root: Path,
|
|
458
450
|
manifest: dict[str, Any],
|
|
@@ -507,7 +499,6 @@ def validate_manifest(
|
|
|
507
499
|
return entry_map
|
|
508
500
|
names = [stage.get("name") for stage in stages if isinstance(stage, dict)]
|
|
509
501
|
require(names == STAGES, "stages are missing or out of order", errors)
|
|
510
|
-
predecessor_files: list[dict[str, str]] = []
|
|
511
502
|
for stage in stages:
|
|
512
503
|
if not isinstance(stage, dict):
|
|
513
504
|
continue
|
|
@@ -525,13 +516,6 @@ def validate_manifest(
|
|
|
525
516
|
require(entry is not None, f"stage output is not a hashed evidence file: {name}/{output}", errors)
|
|
526
517
|
if entry is not None:
|
|
527
518
|
require(entry.get("produced_by") == name, f"stage output producer mismatch: {name}/{output}", errors)
|
|
528
|
-
if isinstance(name, str) and isinstance(manifest.get("target_hashes"), dict):
|
|
529
|
-
expected_input = stage_input_sha(name, manifest["target_hashes"], predecessor_files)
|
|
530
|
-
require(stage.get("input_sha256") == expected_input, f"stage input dependency hash mismatch: {name}", errors)
|
|
531
|
-
predecessor_files.extend(
|
|
532
|
-
{"path": output, "sha256": entry_map[output]["sha256"]}
|
|
533
|
-
for output in outputs if isinstance(outputs, list) and output in entry_map and output not in FINAL_RECORDS
|
|
534
|
-
)
|
|
535
519
|
return entry_map
|
|
536
520
|
|
|
537
521
|
|
|
@@ -822,7 +806,6 @@ def validate_requests_and_receipts(
|
|
|
822
806
|
receipt_by_id: dict[str, dict[str, Any]] = {}
|
|
823
807
|
seen_receipt_paths: set[str] = set()
|
|
824
808
|
attempts_by_request: dict[str, int] = {}
|
|
825
|
-
previous_receipt: dict[str, Any] | None = None
|
|
826
809
|
for index, receipt in enumerate(executions):
|
|
827
810
|
location = f"executions[{index}]"
|
|
828
811
|
if not isinstance(receipt, dict):
|
|
@@ -894,8 +877,6 @@ def validate_requests_and_receipts(
|
|
|
894
877
|
errors.append(str(exc))
|
|
895
878
|
seen_receipt_paths.add(recorded_receipt_path)
|
|
896
879
|
|
|
897
|
-
expected_previous = canonical_sha256(previous_receipt) if previous_receipt is not None else None
|
|
898
|
-
require(receipt.get("previous_receipt_sha256") == expected_previous, f"{location}: receipt chain mismatch", errors)
|
|
899
880
|
request_hash = receipt.get("request_sha256")
|
|
900
881
|
if isinstance(request_hash, str):
|
|
901
882
|
attempts_by_request[request_hash] = attempts_by_request.get(request_hash, 0) + 1
|
|
@@ -906,7 +887,6 @@ def validate_requests_and_receipts(
|
|
|
906
887
|
require(previous is not None, f"{location}: replay source is missing", errors)
|
|
907
888
|
if previous is not None:
|
|
908
889
|
require(previous.get("request_sha256") == request_hash, f"{location}: replay request differs from source", errors)
|
|
909
|
-
previous_receipt = receipt
|
|
910
890
|
|
|
911
891
|
receipt_files = {
|
|
912
892
|
path.relative_to(root).as_posix()
|
|
@@ -1225,10 +1205,6 @@ def validate_matrix_risks(
|
|
|
1225
1205
|
errors.append(f"TEST_PASS has unresolved behavior risk: {behavior_id}/{risk_name}")
|
|
1226
1206
|
|
|
1227
1207
|
|
|
1228
|
-
def require_sha_or_null(value: Any, location: str, errors: list[str]) -> None:
|
|
1229
|
-
require(value is None or is_sha256(value), f"{location} must be a sha256 or null", errors)
|
|
1230
|
-
|
|
1231
|
-
|
|
1232
1208
|
def validate_differential_proofs(
|
|
1233
1209
|
root: Path,
|
|
1234
1210
|
project_root: Path,
|
|
@@ -1262,7 +1238,6 @@ def validate_differential_proofs(
|
|
|
1262
1238
|
for key in (
|
|
1263
1239
|
"method", "differential_request_path", "differential_request_sha256", "baseline_commit",
|
|
1264
1240
|
"baseline_execution_id", "mutation_execution_id", "current_execution_id",
|
|
1265
|
-
"isolation_tree_sha256", "mutation_apply_sha256", "mutation_restore_sha256",
|
|
1266
1241
|
):
|
|
1267
1242
|
require(item.get(key) is None, f"N/A differential proof must clear {key}: {behavior_id}", errors)
|
|
1268
1243
|
require(item.get("failure_reason_matched") is False, f"N/A differential proof cannot claim a failure match: {behavior_id}", errors)
|
|
@@ -1280,10 +1255,6 @@ def validate_differential_proofs(
|
|
|
1280
1255
|
method = item.get("method")
|
|
1281
1256
|
require(method == request.get("method") and method in {"baseline", "controlled-mutation"}, f"differential method mismatch: {behavior_id}", errors)
|
|
1282
1257
|
require(item.get("baseline_commit") == request.get("baseline_commit") == manifest.get("baseline_commit"), f"differential baseline commit mismatch: {behavior_id}", errors)
|
|
1283
|
-
for key in (
|
|
1284
|
-
"differential_request_sha256", "isolation_tree_sha256", "mutation_apply_sha256", "mutation_restore_sha256",
|
|
1285
|
-
):
|
|
1286
|
-
require_sha_or_null(item.get(key), f"differential proof {behavior_id}.{key}", errors)
|
|
1287
1258
|
require(item.get("cleanup_succeeded") is True, f"differential cleanup failed: {behavior_id}", errors)
|
|
1288
1259
|
|
|
1289
1260
|
current = receipts.get(item.get("current_execution_id"))
|
|
@@ -1301,10 +1272,8 @@ def validate_differential_proofs(
|
|
|
1301
1272
|
require(failing.get("selected_execution") is False and failing.get("reliable") is True and failing.get("exit_code") != 0, f"differential failing side is not a reliable failure: {behavior_id}", errors)
|
|
1302
1273
|
current_isolation = current.get("isolation")
|
|
1303
1274
|
failing_isolation = failing.get("isolation")
|
|
1304
|
-
require(isinstance(current_isolation, dict) and current_isolation.get("kind") == "
|
|
1275
|
+
require(isinstance(current_isolation, dict) and current_isolation.get("kind") == "project-direct", f"differential current receipt lacks project-direct isolation: {behavior_id}", errors)
|
|
1305
1276
|
require(isinstance(failing_isolation, dict) and failing_isolation.get("kind") == method, f"differential failing receipt isolation mismatch: {behavior_id}", errors)
|
|
1306
|
-
if isinstance(current_isolation, dict):
|
|
1307
|
-
require(current_isolation.get("tree_sha256") == item.get("isolation_tree_sha256"), f"differential current isolation hash mismatch: {behavior_id}", errors)
|
|
1308
1277
|
if method == "baseline" and isinstance(failing_isolation, dict):
|
|
1309
1278
|
require(failing_isolation.get("baseline_commit") == request.get("baseline_commit"), f"baseline receipt commit mismatch: {behavior_id}", errors)
|
|
1310
1279
|
if method == "controlled-mutation" and isinstance(failing_isolation, dict):
|
|
@@ -1314,12 +1283,9 @@ def validate_differential_proofs(
|
|
|
1314
1283
|
if patch_path is not None:
|
|
1315
1284
|
patch_hash = file_sha256(patch_path)
|
|
1316
1285
|
require(failing_isolation.get("patch_sha256") == patch_hash, f"controlled mutation receipt patch hash mismatch: {behavior_id}", errors)
|
|
1317
|
-
require(item.get("mutation_apply_sha256") == failing_isolation.get("tree_sha256"), f"controlled mutation apply hash mismatch: {behavior_id}", errors)
|
|
1318
|
-
require(item.get("mutation_restore_sha256") == item.get("isolation_tree_sha256"), f"controlled mutation restore hash mismatch: {behavior_id}", errors)
|
|
1319
1286
|
if method == "baseline":
|
|
1320
1287
|
require(item.get("mutation_execution_id") is None, f"baseline proof has mutation receipt: {behavior_id}", errors)
|
|
1321
1288
|
require(request.get("mutation_patch_path") is None, f"baseline request has mutation patch: {behavior_id}", errors)
|
|
1322
|
-
require(item.get("mutation_apply_sha256") is None and item.get("mutation_restore_sha256") is None, f"baseline proof has mutation hashes: {behavior_id}", errors)
|
|
1323
1289
|
elif method == "controlled-mutation":
|
|
1324
1290
|
require(item.get("baseline_execution_id") is None, f"mutation proof has baseline receipt: {behavior_id}", errors)
|
|
1325
1291
|
require(isinstance(request.get("mutation_patch_path"), str) and bool(request.get("mutation_patch_path")), f"mutation request lacks patch: {behavior_id}", errors)
|