prizmkit 1.1.135 → 1.1.137
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bundled/VERSION.json +3 -3
- package/bundled/dev-pipeline/scripts/generate-bootstrap-prompt.py +0 -1
- package/bundled/dev-pipeline/tests/test_generate_bootstrap_prompt.py +1 -1
- package/bundled/skills/_metadata.json +1 -1
- package/bundled/skills/prizmkit-test/SKILL.md +21 -10
- package/bundled/skills/prizmkit-test/assets/authoritative-records.schema.json +7 -16
- package/bundled/skills/prizmkit-test/assets/evidence-manifest.schema.json +1 -2
- package/bundled/skills/prizmkit-test/assets/evidence-package-template.json +9 -54
- package/bundled/skills/prizmkit-test/references/evidence-protocol.md +6 -7
- package/bundled/skills/prizmkit-test/references/evidence-request-protocol.md +3 -3
- package/bundled/skills/prizmkit-test/references/examples.md +1 -1
- package/bundled/skills/prizmkit-test/references/test-generation-steps.md +2 -2
- package/bundled/skills/prizmkit-test/references/trusted-evidence-execution.md +2 -2
- package/bundled/skills/prizmkit-test/scripts/build_test_evidence.py +77 -129
- package/bundled/skills/prizmkit-test/scripts/validate_test_evidence.py +2 -80
- package/package.json +1 -1
package/bundled/VERSION.json
CHANGED
|
@@ -993,7 +993,6 @@ def validate_feature_test_evidence(project_root, artifact_dir):
|
|
|
993
993
|
return None
|
|
994
994
|
for stage in stages:
|
|
995
995
|
if (not isinstance(stage, dict) or stage.get("status") not in {"complete", "not_applicable"} or
|
|
996
|
-
not _is_sha256(stage.get("input_sha256")) or
|
|
997
996
|
not isinstance(stage.get("outputs"), list)):
|
|
998
997
|
return None
|
|
999
998
|
if (not _is_sha256(manifest.get("working_diff_sha256")) or
|
|
@@ -246,7 +246,7 @@ def write_fresh_structured_evidence(project_root):
|
|
|
246
246
|
"EVIDENCE_VALIDATE",
|
|
247
247
|
]
|
|
248
248
|
for name in stage_names:
|
|
249
|
-
stages.append({"name": name, "status": "complete", "
|
|
249
|
+
stages.append({"name": name, "status": "complete", "outputs": []})
|
|
250
250
|
manifest = {
|
|
251
251
|
"protocol_version": "1.0",
|
|
252
252
|
"evidence_id": evidence_id,
|
|
@@ -13,7 +13,7 @@ This skill orchestrates each target project's native runners and conventions. It
|
|
|
13
13
|
|
|
14
14
|
Choose project-semantic parameters from the target project: framework, commands, cwd, timeout, retry/attempt policy, concurrency, Mock tools, test layers, module boundary, inventory patterns, mutation technology, and environment classification. Do not hard-code ecosystem-specific defaults when the project can decide them.
|
|
15
15
|
|
|
16
|
-
Fix only the protocol mechanisms needed for safe, replayable evidence: project/evidence/request locators, schema-shaped requests, path confinement, real process execution, complete raw capture, runner-generated
|
|
16
|
+
Fix only the protocol mechanisms needed for safe, replayable evidence: project/evidence/request locators, schema-shaped requests, path confinement, real process execution, complete raw capture, runner-generated receipts, append-only history, and resume invalidation. Read `${SKILL_DIR}/references/trusted-evidence-execution.md` before inventory, execution, differential proof, resume, or replay; it defines the trust boundary and builder commands. Read `${SKILL_DIR}/references/evidence-request-protocol.md` when authoring any request or authoritative JSON record; it defines request ownership, structured N/A, and the schema map.
|
|
17
17
|
|
|
18
18
|
## Canonical Evidence Builder Lifecycle
|
|
19
19
|
|
|
@@ -174,15 +174,14 @@ If a valid test reveals a business/contract defect, do not fix production behavi
|
|
|
174
174
|
|
|
175
175
|
### 7. EXECUTE_PROVE
|
|
176
176
|
|
|
177
|
-
Run required layers in plan order through `${SKILL_DIR}/scripts/build_test_evidence.py execute`. Use `test_paths` or `auto_bind=true` for new requests; the builder resolves and persists canonical test IDs before the native command runs. The model selects project-native argv, cwd, timeout, attempts, concurrency, layer, environment classification, and tool probes in schema-shaped requests. The runner performs the actual process execution and appends complete environment/tool versions, raw stdout/stderr hashes, request hash, runner hash, unique receipt
|
|
177
|
+
Run required layers in plan order through `${SKILL_DIR}/scripts/build_test_evidence.py execute`. Use `test_paths` or `auto_bind=true` for new requests; the builder resolves and persists canonical test IDs before the native command runs. The model selects project-native argv, cwd, timeout, attempts, concurrency, layer, environment classification, and tool probes in schema-shaped requests. The runner performs the actual process execution and appends complete environment/tool versions, raw stdout/stderr hashes, request hash, runner hash, and unique receipt to `executions.json`. Do not accept caller-authored execution JSON and do not retry until green.
|
|
178
178
|
|
|
179
179
|
Differentially prove each added/changed necessary behavior test:
|
|
180
180
|
|
|
181
|
-
- Prefer baseline failure plus current success
|
|
182
|
-
- If baseline is inapplicable, use a minimal controlled mutation tied to the same risk
|
|
183
|
-
-
|
|
184
|
-
-
|
|
185
|
-
- Completely remove mutations and verify the stable source snapshot is unchanged. Never rebind completed proof to the mutable live project tree.
|
|
181
|
+
- Prefer baseline failure plus current success: for existing behavior, the test fails against the pre-change baseline (via `git worktree`) and passes against the current project.
|
|
182
|
+
- If baseline is inapplicable (new behavior), use a minimal controlled mutation tied to the same risk, applied in a `git worktree`.
|
|
183
|
+
- The current (success) side executes directly in the project root — tests and behavior under proof belong in the main repository, not an isolated copy.
|
|
184
|
+
- Bind proof to runner receipts, baseline commit, and worktree cleanup. Never rebind completed proof to the mutable live project tree.
|
|
186
185
|
- Classify each proof as `PROVEN`, structured `NOT_APPLICABLE`, or `UNPROVEN`.
|
|
187
186
|
|
|
188
187
|
Necessary `UNPROVEN` behavior, required flakiness, unavailable infrastructure after bounded recovery, failed cleanup, unreliable execution, or budget truncation yields `TEST_BLOCKED`.
|
|
@@ -206,11 +205,23 @@ Create `.prizmkit/test/evidence/<evidence-id>/` from the package template with:
|
|
|
206
205
|
- `contracts/` snapshots
|
|
207
206
|
- derived `test-report.md`
|
|
208
207
|
|
|
209
|
-
Hash baseline, diff,
|
|
208
|
+
Hash baseline, diff, inventoried source/tests/contracts/lockfiles, environment, plan, and every evidence file. Evidence files remain individually content-addressed through `manifest.json`. Set `environment_claim=mocked-code-level-only`.
|
|
210
209
|
|
|
211
210
|
### 9. EVIDENCE_VALIDATE
|
|
212
211
|
|
|
213
|
-
|
|
212
|
+
The validator runs in two passes:
|
|
213
|
+
|
|
214
|
+
**First pass (pre-check, no `--attest`):**
|
|
215
|
+
|
|
216
|
+
```bash
|
|
217
|
+
python3 ${SKILL_DIR}/scripts/validate_test_evidence.py \
|
|
218
|
+
--evidence-dir .prizmkit/test/evidence/<evidence-id> \
|
|
219
|
+
--project-root <target-project-root>
|
|
220
|
+
```
|
|
221
|
+
|
|
222
|
+
If this pre-check finds errors, fix them in the package before the second pass. The validator checks every authoritative record against its shipped schema, live target inventories, patch/diff binding, changed-file/module-root/exclusion/Regression Ring cross-links, generated-test snapshots, planned-test execution links, runner-generated receipts/request/raw-output hashes, successful behavior-risk mappings, production/unknown external-target safety, and package integrity.
|
|
223
|
+
|
|
224
|
+
**Second pass (attestation, with `--attest`):**
|
|
214
225
|
|
|
215
226
|
```bash
|
|
216
227
|
python3 ${SKILL_DIR}/scripts/validate_test_evidence.py \
|
|
@@ -219,7 +230,7 @@ python3 ${SKILL_DIR}/scripts/validate_test_evidence.py \
|
|
|
219
230
|
--attest
|
|
220
231
|
```
|
|
221
232
|
|
|
222
|
-
|
|
233
|
+
`--attest` writes an integrity/protocol attestation into `validation.json`, re-derives `test-report.md`, re-hashes final records in `manifest.json`, then runs a **second** strict validation against the refreshed package. A package is not a `TEST_PASS` until this second strict validation succeeds. Use builder replay when feasible to rerun recorded requests and create new linked receipts. Store validation output. Tampering, schema/identity/hash mismatch, source/test/lockfile drift, missing matrix mappings, wrong execution order, invalid differential proof, cleanup failure, unjustified structured N/A, unresolved risk, or a false real-environment claim fails validation.
|
|
223
234
|
|
|
224
235
|
## Resume and Idempotency
|
|
225
236
|
|
|
@@ -20,7 +20,7 @@
|
|
|
20
20
|
"additionalProperties": false,
|
|
21
21
|
"required": ["kind", "source", "observations"],
|
|
22
22
|
"properties": {
|
|
23
|
-
"kind": { "enum": ["manifest", "runner-config", "filesystem", "contract", "lockfile", "import-graph", "caller-analysis", "project-doc", "model-analysis"] },
|
|
23
|
+
"kind": { "enum": ["manifest", "runner-config", "filesystem", "contract", "lockfile", "import-graph", "caller-analysis", "project-doc", "model-analysis", "shared-contract", "source-inspection", "field-analysis", "runner-inspection", "dependency-scan", "audit-log", "api-spec", "tool-probe"] },
|
|
24
24
|
"source": { "type": "string", "minLength": 1 },
|
|
25
25
|
"observations": { "$ref": "#/$defs/stringArray" }
|
|
26
26
|
}
|
|
@@ -104,7 +104,7 @@
|
|
|
104
104
|
"required": ["name", "kind", "files", "planned_test_ids", "discovery_evidence"],
|
|
105
105
|
"properties": {
|
|
106
106
|
"name": { "type": "string", "minLength": 1 },
|
|
107
|
-
"kind": { "enum": ["caller", "consumer", "shared-contract", "state-dependency"] },
|
|
107
|
+
"kind": { "enum": ["caller", "consumer", "shared-contract", "state-dependency", "api-spec", "documentation"] },
|
|
108
108
|
"files": { "$ref": "#/$defs/stringArray" },
|
|
109
109
|
"planned_test_ids": { "$ref": "#/$defs/stringArray" },
|
|
110
110
|
"discovery_evidence": { "type": "array", "minItems": 1, "items": { "$ref": "#/$defs/discoveryEvidence" } }
|
|
@@ -218,7 +218,7 @@
|
|
|
218
218
|
"changeCapture": {
|
|
219
219
|
"type": "object",
|
|
220
220
|
"additionalProperties": false,
|
|
221
|
-
"required": ["capture_format", "baseline_commit", "patch_path", "patch_sha256", "changed_files"
|
|
221
|
+
"required": ["capture_format", "baseline_commit", "patch_path", "patch_sha256", "changed_files"],
|
|
222
222
|
"properties": {
|
|
223
223
|
"capture_format": { "const": "prizmkit-canonical-change-v1" },
|
|
224
224
|
"baseline_commit": { "type": "string", "minLength": 1 },
|
|
@@ -237,8 +237,7 @@
|
|
|
237
237
|
"role": { "const": "source" }
|
|
238
238
|
}
|
|
239
239
|
}
|
|
240
|
-
}
|
|
241
|
-
"source_tree_sha256": { "$ref": "#/$defs/sha256" }
|
|
240
|
+
}
|
|
242
241
|
}
|
|
243
242
|
},
|
|
244
243
|
"testPlan": {
|
|
@@ -358,7 +357,7 @@
|
|
|
358
357
|
"executionReceipt": {
|
|
359
358
|
"type": "object",
|
|
360
359
|
"additionalProperties": false,
|
|
361
|
-
"required": ["receipt_format", "runner_path", "runner_sha256", "execution_id", "attempt_index", "request_path", "request_sha256", "purpose", "command", "cwd", "environment", "tool_versions", "layer", "test_ids", "external_targets", "exit_code", "stdout_path", "stderr_path", "stdout_sha256", "stderr_sha256", "selected_execution", "reliable", "started_at", "finished_at"
|
|
360
|
+
"required": ["receipt_format", "runner_path", "runner_sha256", "execution_id", "attempt_index", "request_path", "request_sha256", "purpose", "command", "cwd", "environment", "tool_versions", "layer", "test_ids", "external_targets", "exit_code", "stdout_path", "stderr_path", "stdout_sha256", "stderr_sha256", "selected_execution", "reliable", "started_at", "finished_at"],
|
|
362
361
|
"properties": {
|
|
363
362
|
"receipt_format": { "const": "prizmkit-runner-generated-v1" },
|
|
364
363
|
"runner_path": { "$ref": "#/$defs/path" },
|
|
@@ -386,7 +385,6 @@
|
|
|
386
385
|
"reliable": { "type": "boolean" },
|
|
387
386
|
"started_at": { "type": "string", "minLength": 1 },
|
|
388
387
|
"finished_at": { "type": "string", "minLength": 1 },
|
|
389
|
-
"previous_receipt_sha256": { "type": ["string", "null"] },
|
|
390
388
|
"replay_of": { "type": ["string", "null"] },
|
|
391
389
|
"isolation": { "type": ["object", "null"] }
|
|
392
390
|
}
|
|
@@ -398,14 +396,13 @@
|
|
|
398
396
|
{
|
|
399
397
|
"type": "object",
|
|
400
398
|
"additionalProperties": false,
|
|
401
|
-
"required": ["request_version", "behavior_id", "method", "execution_request", "baseline_commit", "
|
|
399
|
+
"required": ["request_version", "behavior_id", "method", "execution_request", "baseline_commit", "mutation_patch_path", "test_overlay_paths", "expected_failure_signals"],
|
|
402
400
|
"properties": {
|
|
403
401
|
"request_version": { "const": "1.0" },
|
|
404
402
|
"behavior_id": { "type": "string", "minLength": 1 },
|
|
405
403
|
"method": { "enum": ["baseline", "controlled-mutation"] },
|
|
406
404
|
"execution_request": { "$ref": "#/$defs/executionRequest" },
|
|
407
405
|
"baseline_commit": { "type": "string", "minLength": 1 },
|
|
408
|
-
"current_tree_sha256": { "$ref": "#/$defs/sha256" },
|
|
409
406
|
"mutation_patch_path": { "type": ["string", "null"] },
|
|
410
407
|
"test_overlay_paths": { "$ref": "#/$defs/stringArray" },
|
|
411
408
|
"expected_failure_signals": { "$ref": "#/$defs/stringArray" }
|
|
@@ -447,7 +444,7 @@
|
|
|
447
444
|
"items": {
|
|
448
445
|
"type": "object",
|
|
449
446
|
"additionalProperties": false,
|
|
450
|
-
"required": ["behavior_id", "classification", "method", "differential_request_path", "differential_request_sha256", "baseline_commit", "
|
|
447
|
+
"required": ["behavior_id", "classification", "method", "differential_request_path", "differential_request_sha256", "baseline_commit", "baseline_execution_id", "mutation_execution_id", "current_execution_id", "failure_reason_matched", "cleanup_succeeded", "not_applicable"],
|
|
451
448
|
"properties": {
|
|
452
449
|
"behavior_id": { "type": "string", "minLength": 1 },
|
|
453
450
|
"classification": { "enum": ["PROVEN", "NOT_APPLICABLE", "UNPROVEN"] },
|
|
@@ -455,17 +452,11 @@
|
|
|
455
452
|
"differential_request_path": { "type": ["string", "null"] },
|
|
456
453
|
"differential_request_sha256": { "type": ["string", "null"] },
|
|
457
454
|
"baseline_commit": { "type": ["string", "null"] },
|
|
458
|
-
"current_tree_sha256": { "type": ["string", "null"] },
|
|
459
455
|
"baseline_execution_id": { "type": ["string", "null"] },
|
|
460
456
|
"mutation_execution_id": { "type": ["string", "null"] },
|
|
461
457
|
"current_execution_id": { "type": ["string", "null"] },
|
|
462
458
|
"failure_reason_matched": { "type": "boolean" },
|
|
463
459
|
"cleanup_succeeded": { "type": "boolean" },
|
|
464
|
-
"project_tree_before_sha256": { "type": ["string", "null"] },
|
|
465
|
-
"project_tree_after_sha256": { "type": ["string", "null"] },
|
|
466
|
-
"isolation_tree_sha256": { "type": ["string", "null"] },
|
|
467
|
-
"mutation_apply_sha256": { "type": ["string", "null"] },
|
|
468
|
-
"mutation_restore_sha256": { "type": ["string", "null"] },
|
|
469
460
|
"not_applicable": { "type": ["object", "null"] }
|
|
470
461
|
}
|
|
471
462
|
}
|
|
@@ -67,7 +67,7 @@
|
|
|
67
67
|
"stage": {
|
|
68
68
|
"type": "object",
|
|
69
69
|
"additionalProperties": false,
|
|
70
|
-
"required": ["name", "status", "
|
|
70
|
+
"required": ["name", "status", "outputs"],
|
|
71
71
|
"properties": {
|
|
72
72
|
"name": {
|
|
73
73
|
"enum": [
|
|
@@ -83,7 +83,6 @@
|
|
|
83
83
|
]
|
|
84
84
|
},
|
|
85
85
|
"status": { "enum": ["complete", "not_applicable"] },
|
|
86
|
-
"input_sha256": { "$ref": "#/$defs/sha256" },
|
|
87
86
|
"outputs": { "type": "array", "items": { "type": "string", "minLength": 1 } },
|
|
88
87
|
"not_applicable": {
|
|
89
88
|
"type": ["object", "null"],
|
|
@@ -38,60 +38,15 @@
|
|
|
38
38
|
"contract_snapshots": "contracts/"
|
|
39
39
|
},
|
|
40
40
|
"stage_statuses": [
|
|
41
|
-
{
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
},
|
|
47
|
-
{
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
"input_sha256": "<sha256>",
|
|
51
|
-
"outputs": []
|
|
52
|
-
},
|
|
53
|
-
{
|
|
54
|
-
"name": "CONTRACT_MODEL",
|
|
55
|
-
"status": "pending",
|
|
56
|
-
"input_sha256": "<sha256>",
|
|
57
|
-
"outputs": []
|
|
58
|
-
},
|
|
59
|
-
{
|
|
60
|
-
"name": "TEST_PLAN",
|
|
61
|
-
"status": "pending",
|
|
62
|
-
"input_sha256": "<sha256>",
|
|
63
|
-
"outputs": []
|
|
64
|
-
},
|
|
65
|
-
{
|
|
66
|
-
"name": "INFRA_READY",
|
|
67
|
-
"status": "pending",
|
|
68
|
-
"input_sha256": "<sha256>",
|
|
69
|
-
"outputs": []
|
|
70
|
-
},
|
|
71
|
-
{
|
|
72
|
-
"name": "TEST_BUILD",
|
|
73
|
-
"status": "pending",
|
|
74
|
-
"input_sha256": "<sha256>",
|
|
75
|
-
"outputs": []
|
|
76
|
-
},
|
|
77
|
-
{
|
|
78
|
-
"name": "EXECUTE_PROVE",
|
|
79
|
-
"status": "pending",
|
|
80
|
-
"input_sha256": "<sha256>",
|
|
81
|
-
"outputs": []
|
|
82
|
-
},
|
|
83
|
-
{
|
|
84
|
-
"name": "EVIDENCE_PACKAGE",
|
|
85
|
-
"status": "pending",
|
|
86
|
-
"input_sha256": "<sha256>",
|
|
87
|
-
"outputs": []
|
|
88
|
-
},
|
|
89
|
-
{
|
|
90
|
-
"name": "EVIDENCE_VALIDATE",
|
|
91
|
-
"status": "pending",
|
|
92
|
-
"input_sha256": "<sha256>",
|
|
93
|
-
"outputs": []
|
|
94
|
-
}
|
|
41
|
+
{ "name": "CHANGE_CLASSIFY", "status": "pending", "outputs": [] },
|
|
42
|
+
{ "name": "SCOPE_DISCOVER", "status": "pending", "outputs": [] },
|
|
43
|
+
{ "name": "CONTRACT_MODEL", "status": "pending", "outputs": [] },
|
|
44
|
+
{ "name": "TEST_PLAN", "status": "pending", "outputs": [] },
|
|
45
|
+
{ "name": "INFRA_READY", "status": "pending", "outputs": [] },
|
|
46
|
+
{ "name": "TEST_BUILD", "status": "pending", "outputs": [] },
|
|
47
|
+
{ "name": "EXECUTE_PROVE", "status": "pending", "outputs": [] },
|
|
48
|
+
{ "name": "EVIDENCE_PACKAGE", "status": "pending", "outputs": [] },
|
|
49
|
+
{ "name": "EVIDENCE_VALIDATE", "status": "pending", "outputs": [] }
|
|
95
50
|
],
|
|
96
51
|
"resume": {
|
|
97
52
|
"last_valid_stage": null,
|
|
@@ -14,7 +14,6 @@ Store those values as `evidence_id_inputs` in `manifest.json`. Record separate a
|
|
|
14
14
|
|
|
15
15
|
Use two separate integrity domains:
|
|
16
16
|
|
|
17
|
-
- **Stable source snapshot:** hash behavior-bearing project files while excluding runtime-managed `.prizmkit/state/`, evidence output under `.prizmkit/test/evidence/`, linked `.claude/worktrees/`, Git metadata, and Python bytecode caches. Builder and validator use the same deterministic exclusion policy, and isolated current execution is copied from that same snapshot scope.
|
|
18
17
|
- **Evidence package:** hash each evidence record through `manifest.json`; any report, receipt, raw output, request, or proof mutation remains detectable even though evidence output is outside source identity.
|
|
19
18
|
|
|
20
19
|
Runtime-only churn must not invalidate a source proof. Drift in an inventoried source, test, contract, or lockfile must still fail live inventory validation. Never replace a completed proof's stable snapshot identity with a later hash of the mutable parent workspace.
|
|
@@ -42,7 +41,7 @@ Run exactly these states in order:
|
|
|
42
41
|
8. `EVIDENCE_PACKAGE`
|
|
43
42
|
9. `EVIDENCE_VALIDATE`
|
|
44
43
|
|
|
45
|
-
Each manifest stage records `name`, `status`,
|
|
44
|
+
Each manifest stage records `name`, `status`, and output paths. A stage may be `not_applicable` only with a concrete rationale. A later state cannot complete until every applicable predecessor output exists and validates.
|
|
46
45
|
|
|
47
46
|
### CHANGE_CLASSIFY
|
|
48
47
|
|
|
@@ -120,7 +119,7 @@ Run progressively:
|
|
|
120
119
|
|
|
121
120
|
Only omit plan-marked non-required layers. Every N/A is structured with rationale, typed evidence, considered signals, and explanations for detected conflicts. The model retains semantic judgment; validation checks structure and obvious signal conflicts, not hidden intent. Append only runner-generated execution receipts containing:
|
|
122
121
|
|
|
123
|
-
- runner-generated receipt format, runner hash, request path/hash, and
|
|
122
|
+
- runner-generated receipt format, runner hash, request path/hash, and unique execution identifier;
|
|
124
123
|
- unique selected execution identifier;
|
|
125
124
|
- layer and exact argv;
|
|
126
125
|
- confined working directory;
|
|
@@ -134,11 +133,11 @@ Do not overwrite failed or superseded attempts. Do not retry until green. Use a
|
|
|
134
133
|
|
|
135
134
|
For each added or changed necessary behavior test, prove the test:
|
|
136
135
|
|
|
137
|
-
1. Prefer
|
|
138
|
-
2. If baseline execution is inapplicable (
|
|
136
|
+
1. Prefer baseline failure plus current success: for existing behavior, the test fails against the pre-change baseline (via `git worktree`) and passes against the current project.
|
|
137
|
+
2. If baseline execution is inapplicable (new behavior), apply a minimal controlled mutation tied to the same matrix risk in a `git worktree`, observe failure, restore it completely, and observe current success.
|
|
139
138
|
3. Record `PROVEN`, justified `NOT_APPLICABLE`, or `UNPROVEN`.
|
|
140
139
|
|
|
141
|
-
Record baseline/
|
|
140
|
+
Record baseline/mutation runner receipt IDs, current runner receipt IDs, request hash, baseline commit, expected failure signal match, and worktree cleanup result. Necessary `UNPROVEN` behavior prevents `TEST_PASS`.
|
|
142
141
|
|
|
143
142
|
### EVIDENCE_PACKAGE
|
|
144
143
|
|
|
@@ -169,7 +168,7 @@ python3 ${SKILL_DIR}/scripts/validate_test_evidence.py \
|
|
|
169
168
|
|
|
170
169
|
When feasible, replay recorded requests through `build_test_evidence.py execute --replay-receipt ...`. Replay performs a new real execution and emits a new linked receipt; it does not pretend the attestation establishes what historically ran.
|
|
171
170
|
|
|
172
|
-
The validator checks all shipped JSON schemas, evidence-directory identity, live target-project inventories and aggregate hashes,
|
|
171
|
+
The validator checks all shipped JSON schemas, evidence-directory identity, live target-project inventories and aggregate hashes, changed-file/module-root/exclusion/Regression Ring cross-links, patch/diff binding, stage ordering and output ownership, generated-test snapshot existence, planned test/inventory/execution linkage, module/matrix completeness, successful selected runner receipts with request/raw-output binding, structured N/A and obvious signal conflicts, production/unknown external-target safety, strict verdict semantics, and honest mocked-versus-real claims. `TEST_PASS` requires deterministic validator success.
|
|
173
172
|
|
|
174
173
|
## Resume and Invalidation
|
|
175
174
|
|
|
@@ -11,7 +11,7 @@ Use existing project-native tests first. The Main Agent supplies only semantic t
|
|
|
11
11
|
`finalize` creates the manifest, stage dependency hashes, verdict, evidence identity, and deterministic report inputs. `render-report` only renders the derived Markdown view. The validator alone creates `validation.json` during `--attest`; never hand-author a validation attestation or rebind a finalized package after identity inputs change.
|
|
12
12
|
|
|
13
13
|
|
|
14
|
-
The model writes project-semantic requests after inspecting manifests, runner configuration, contracts, test conventions, changed files, and module boundaries. The bundled builder owns process execution, raw output, receipts, isolation,
|
|
14
|
+
The model writes project-semantic requests after inspecting manifests, runner configuration, contracts, test conventions, changed files, and module boundaries. The bundled builder owns process execution, raw output, receipts, worktree-based isolation, and invalidation calculations.
|
|
15
15
|
|
|
16
16
|
Never hand-author `executions.json` receipt fields. Never copy a command's claimed output into `raw/`. Invoke the builder so provenance is linked to the actual process.
|
|
17
17
|
|
|
@@ -48,11 +48,11 @@ Record:
|
|
|
48
48
|
|
|
49
49
|
- behavior ID and model-selected `baseline` or `controlled-mutation` method;
|
|
50
50
|
- the nested execution request;
|
|
51
|
-
- actual baseline commit
|
|
51
|
+
- actual baseline commit;
|
|
52
52
|
- mutation patch path when applicable;
|
|
53
53
|
- concrete expected failure signals.
|
|
54
54
|
|
|
55
|
-
The builder
|
|
55
|
+
The builder uses `git worktree add --detach` for the failing side (baseline commit or controlled mutation) and executes the current side directly in the project root. Differential proofs compare receipt outcomes (exit codes, expected failure signals) between the failing side and the current side. Worktrees are cleaned up with `git worktree remove --force` after execution. Source drift between capture and validation is detected through change-capture freshness checks (inventory changed_files vs canonical capture) and resume invalidation (live target hashes vs manifest).
|
|
56
56
|
|
|
57
57
|
## Structured N/A
|
|
58
58
|
|
|
@@ -71,7 +71,7 @@ Evidence: .prizmkit/test/evidence/<evidence-id>/
|
|
|
71
71
|
|
|
72
72
|
## 6. Baseline Inapplicable — Controlled Mutation
|
|
73
73
|
|
|
74
|
-
A newly added CLI command has no pre-change interface, so baseline execution is justified `NOT_APPLICABLE` as a direct proof method.
|
|
74
|
+
A newly added CLI command has no pre-change interface, so baseline execution is justified `NOT_APPLICABLE` as a direct proof method. The run creates a `git worktree` for the mutation, applies a minimal patch that removes the command registration, observes the command test fail for that risk, cleans up the worktree, and observes current success. The behavior is `PROVEN` by `controlled-mutation`.
|
|
75
75
|
|
|
76
76
|
If restoration cannot be proven, the verdict is `TEST_BLOCKED`.
|
|
77
77
|
|
|
@@ -101,8 +101,8 @@ Run selected tests in order through the bundled evidence builder: focused, modul
|
|
|
101
101
|
|
|
102
102
|
For each new/changed necessary behavior test:
|
|
103
103
|
|
|
104
|
-
1.
|
|
105
|
-
2. If baseline proof is genuinely inapplicable, apply one model-selected minimal controlled mutation tied to the risk
|
|
104
|
+
1. For existing behavior, run it against the pre-change baseline (via `git worktree`) and require failure for the intended recorded signal, then current success against the project root.
|
|
105
|
+
2. If baseline proof is genuinely inapplicable, apply one model-selected minimal controlled mutation tied to the risk in a `git worktree`, require failure, restore and clean up the worktree, and verify current success against the project root.
|
|
106
106
|
3. Record `PROVEN`, structured `NOT_APPLICABLE`, or `UNPROVEN` with runner receipt IDs.
|
|
107
107
|
|
|
108
108
|
Necessary `UNPROVEN` evidence blocks. Store complete stdout/stderr in `raw/`; snapshots in `generated-tests/`; do not overwrite previous attempts.
|
|
@@ -27,9 +27,9 @@ The protocol fixes only what trustworthy, replayable packaging needs:
|
|
|
27
27
|
- actual child-process execution without a shell;
|
|
28
28
|
- complete stdout/stderr capture and output hashes;
|
|
29
29
|
- complete effective environment and tool-probe capture;
|
|
30
|
-
- runner-generated UUID receipt, request hash,
|
|
30
|
+
- runner-generated UUID receipt, request hash, and runner hash;
|
|
31
31
|
- append-only execution history;
|
|
32
|
-
- differential isolation
|
|
32
|
+
- differential isolation via git worktree and cleanup;
|
|
33
33
|
- deterministic resume invalidation calculation;
|
|
34
34
|
- strict schema and cross-record validation.
|
|
35
35
|
|
|
@@ -46,7 +46,7 @@ LAYERS = (
|
|
|
46
46
|
"affected-module-regression", "regression-ring",
|
|
47
47
|
)
|
|
48
48
|
BLOCKED_EXTERNAL_CLASSES = {"production", "unknown"}
|
|
49
|
-
|
|
49
|
+
INVENTORY_RESERVED_ROOTS = (
|
|
50
50
|
".prizmkit/state",
|
|
51
51
|
".prizmkit/test/evidence",
|
|
52
52
|
".prizmkit/" + "dev-" + "pipeline",
|
|
@@ -55,8 +55,8 @@ SNAPSHOT_VOLATILE_ROOTS = (
|
|
|
55
55
|
".codebuddy/worktrees",
|
|
56
56
|
".codex/worktrees",
|
|
57
57
|
)
|
|
58
|
-
|
|
59
|
-
|
|
58
|
+
|
|
59
|
+
_IGNORED_DIR_NAMES = {".git", "__pycache__", ".pytest_cache", ".mypy_cache"}
|
|
60
60
|
FINAL_RECORDS = {"manifest.json", "validation.json", "verdict.json", "test-report.md"}
|
|
61
61
|
|
|
62
62
|
|
|
@@ -152,7 +152,7 @@ def normalize_relative(value: str) -> str:
|
|
|
152
152
|
|
|
153
153
|
def is_reserved_inventory_path(relative: str) -> bool:
|
|
154
154
|
path = Path(relative)
|
|
155
|
-
if any(part in
|
|
155
|
+
if any(part in _IGNORED_DIR_NAMES for part in path.parts):
|
|
156
156
|
return True
|
|
157
157
|
return any(path == Path(prefix) or Path(prefix) in path.parents for prefix in INVENTORY_RESERVED_ROOTS)
|
|
158
158
|
|
|
@@ -355,7 +355,6 @@ def load_change_capture(
|
|
|
355
355
|
if (
|
|
356
356
|
live_patch_sha256 != capture["patch_sha256"]
|
|
357
357
|
or live_entries != capture["changed_files"]
|
|
358
|
-
or tree_sha256(project_root, [evidence_dir]) != capture.get("source_tree_sha256")
|
|
359
358
|
):
|
|
360
359
|
raise RequestError(
|
|
361
360
|
"canonical change capture is stale; rerun capture-change and inventory after repository changes"
|
|
@@ -558,28 +557,6 @@ def resolve_execution_test_ids(project_root: Path, evidence_dir: Path, request:
|
|
|
558
557
|
return request
|
|
559
558
|
|
|
560
559
|
|
|
561
|
-
def is_volatile_snapshot_path(root: Path, candidate: Path, excluded: list[Path] | None = None) -> bool:
|
|
562
|
-
try:
|
|
563
|
-
relative = candidate.relative_to(root)
|
|
564
|
-
except ValueError:
|
|
565
|
-
return True
|
|
566
|
-
if any(part in SNAPSHOT_VOLATILE_NAMES for part in relative.parts):
|
|
567
|
-
return True
|
|
568
|
-
if any(relative == Path(prefix) or Path(prefix) in relative.parents for prefix in SNAPSHOT_VOLATILE_ROOTS):
|
|
569
|
-
return True
|
|
570
|
-
resolved = candidate.resolve()
|
|
571
|
-
return any(resolved == item.resolve() or item.resolve() in resolved.parents for item in (excluded or []))
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
def tree_sha256(root: Path, excluded: list[Path] | None = None) -> str:
|
|
575
|
-
entries: list[dict[str, str]] = []
|
|
576
|
-
for candidate in sorted(root.rglob("*")):
|
|
577
|
-
if not candidate.is_file() or is_volatile_snapshot_path(root, candidate, excluded):
|
|
578
|
-
continue
|
|
579
|
-
entries.append({"path": candidate.relative_to(root).as_posix(), "sha256": file_sha256(candidate)})
|
|
580
|
-
return canonical_sha256(entries)
|
|
581
|
-
|
|
582
|
-
|
|
583
560
|
def collect_matches(project_root: Path, patterns: list[str], exclusions: set[str]) -> list[dict[str, str]]:
|
|
584
561
|
project_resolved = project_root.resolve()
|
|
585
562
|
paths: set[Path] = set()
|
|
@@ -844,9 +821,6 @@ def append_receipt(evidence_dir: Path, receipt: dict[str, Any]) -> None:
|
|
|
844
821
|
existing: Any = load_json(path) if path.exists() else []
|
|
845
822
|
if not isinstance(existing, list):
|
|
846
823
|
raise RequestError("executions.json must remain an array")
|
|
847
|
-
previous = canonical_sha256(existing[-1]) if existing else None
|
|
848
|
-
if receipt["previous_receipt_sha256"] != previous:
|
|
849
|
-
raise RequestError("receipt chain mismatch")
|
|
850
824
|
existing.append(receipt)
|
|
851
825
|
write_json_atomic(path, existing)
|
|
852
826
|
|
|
@@ -923,7 +897,6 @@ def execute_request(
|
|
|
923
897
|
existing = load_json(evidence_dir / "executions.json") if (evidence_dir / "executions.json").exists() else []
|
|
924
898
|
if not isinstance(existing, list):
|
|
925
899
|
raise RequestError("executions.json must remain an array")
|
|
926
|
-
previous = canonical_sha256(existing[-1]) if existing else None
|
|
927
900
|
same_request_attempts = sum(1 for item in existing if isinstance(item, dict) and item.get("request_sha256") == file_sha256(request_path))
|
|
928
901
|
receipt = {
|
|
929
902
|
"receipt_format": "prizmkit-runner-generated-v1",
|
|
@@ -950,7 +923,6 @@ def execute_request(
|
|
|
950
923
|
"reliable": probes_reliable and not timed_out and not execution_error,
|
|
951
924
|
"started_at": started,
|
|
952
925
|
"finished_at": finished,
|
|
953
|
-
"previous_receipt_sha256": previous,
|
|
954
926
|
"replay_of": replay_of,
|
|
955
927
|
"isolation": isolation,
|
|
956
928
|
}
|
|
@@ -958,33 +930,6 @@ def execute_request(
|
|
|
958
930
|
return receipt
|
|
959
931
|
|
|
960
932
|
|
|
961
|
-
def copy_project_tree(project_root: Path, destination: Path, evidence_dir: Path) -> None:
|
|
962
|
-
def ignore(directory: str, names: list[str]) -> set[str]:
|
|
963
|
-
directory_relative = Path(directory).resolve().relative_to(project_root.resolve())
|
|
964
|
-
ignored: set[str] = set()
|
|
965
|
-
for name in names:
|
|
966
|
-
relative = directory_relative / name
|
|
967
|
-
candidate = project_root / relative
|
|
968
|
-
if is_volatile_snapshot_path(project_root, candidate, [evidence_dir]):
|
|
969
|
-
ignored.add(name)
|
|
970
|
-
return ignored
|
|
971
|
-
|
|
972
|
-
shutil.copytree(project_root, destination, ignore=ignore)
|
|
973
|
-
|
|
974
|
-
|
|
975
|
-
def git_archive_baseline(project_root: Path, baseline_commit: str, destination: Path) -> None:
|
|
976
|
-
result = subprocess.run(
|
|
977
|
-
["git", "-C", str(project_root), "archive", "--format=tar", baseline_commit],
|
|
978
|
-
capture_output=True, check=False,
|
|
979
|
-
)
|
|
980
|
-
if result.returncode != 0:
|
|
981
|
-
raise RequestError(f"cannot archive baseline commit {baseline_commit}: {result.stderr.decode(errors='replace')}")
|
|
982
|
-
destination.mkdir(parents=True)
|
|
983
|
-
extracted = subprocess.run(["tar", "-xf", "-", "-C", str(destination)], input=result.stdout, capture_output=True, check=False)
|
|
984
|
-
if extracted.returncode != 0:
|
|
985
|
-
raise RequestError(f"cannot extract baseline archive: {extracted.stderr.decode(errors='replace')}")
|
|
986
|
-
|
|
987
|
-
|
|
988
933
|
def materialize_request(evidence_dir: Path, name: str, request: dict[str, Any]) -> Path:
|
|
989
934
|
path = evidence_dir / "requests" / name
|
|
990
935
|
write_json_atomic(path, request)
|
|
@@ -1024,7 +969,7 @@ def append_proof(evidence_dir: Path, proof: dict[str, Any]) -> None:
|
|
|
1024
969
|
|
|
1025
970
|
def run_differential(project_root: Path, evidence_dir: Path, request_path: Path) -> dict[str, Any]:
|
|
1026
971
|
request = ensure_object(load_json(request_path), "differential request")
|
|
1027
|
-
required = {"request_version", "behavior_id", "method", "execution_request", "baseline_commit", "
|
|
972
|
+
required = {"request_version", "behavior_id", "method", "execution_request", "baseline_commit", "mutation_patch_path", "test_overlay_paths", "expected_failure_signals"}
|
|
1028
973
|
na_required = {"request_version", "behavior_id", "method", "not_applicable"}
|
|
1029
974
|
if set(request) == na_required:
|
|
1030
975
|
if request["request_version"] != "1.0" or request["method"] != "not-applicable":
|
|
@@ -1034,10 +979,9 @@ def run_differential(project_root: Path, evidence_dir: Path, request_path: Path)
|
|
|
1034
979
|
"behavior_id": request["behavior_id"], "classification": "NOT_APPLICABLE",
|
|
1035
980
|
"method": None, "differential_request_path": evidence_relative(evidence_dir, request_path),
|
|
1036
981
|
"differential_request_sha256": file_sha256(request_path), "baseline_commit": None,
|
|
1037
|
-
"
|
|
982
|
+
"baseline_execution_id": None, "mutation_execution_id": None,
|
|
1038
983
|
"current_execution_id": None, "failure_reason_matched": False, "cleanup_succeeded": True,
|
|
1039
|
-
"
|
|
1040
|
-
"mutation_apply_sha256": None, "mutation_restore_sha256": None, "not_applicable": not_applicable,
|
|
984
|
+
"not_applicable": not_applicable,
|
|
1041
985
|
}
|
|
1042
986
|
append_proof(evidence_dir, proof)
|
|
1043
987
|
return proof
|
|
@@ -1056,62 +1000,71 @@ def run_differential(project_root: Path, evidence_dir: Path, request_path: Path)
|
|
|
1056
1000
|
overlay_paths = ensure_string_list(request["test_overlay_paths"], "test_overlay_paths")
|
|
1057
1001
|
for relative in overlay_paths:
|
|
1058
1002
|
confined(project_root, relative, must_exist=True)
|
|
1059
|
-
project_before = tree_sha256(project_root, [evidence_dir])
|
|
1060
|
-
if project_before != request["current_tree_sha256"]:
|
|
1061
|
-
raise RequestError("current project tree does not match differential request")
|
|
1062
1003
|
current_request_path = materialize_request(evidence_dir, f"differential-{uuid.uuid4().hex}.execution.json", execution_request)
|
|
1063
1004
|
baseline_receipt = None
|
|
1064
1005
|
mutation_receipt = None
|
|
1065
1006
|
current_receipt = None
|
|
1066
|
-
isolation_tree = None
|
|
1067
|
-
mutation_apply_sha = None
|
|
1068
|
-
mutation_restore_sha = None
|
|
1069
1007
|
cleanup_succeeded = False
|
|
1008
|
+
worktree_path = None
|
|
1070
1009
|
try:
|
|
1071
|
-
|
|
1072
|
-
|
|
1073
|
-
|
|
1074
|
-
|
|
1075
|
-
|
|
1076
|
-
|
|
1077
|
-
|
|
1078
|
-
|
|
1079
|
-
|
|
1080
|
-
|
|
1081
|
-
|
|
1082
|
-
|
|
1083
|
-
|
|
1084
|
-
|
|
1085
|
-
|
|
1086
|
-
|
|
1087
|
-
|
|
1088
|
-
|
|
1089
|
-
|
|
1090
|
-
|
|
1091
|
-
|
|
1092
|
-
|
|
1093
|
-
|
|
1094
|
-
|
|
1095
|
-
|
|
1096
|
-
|
|
1097
|
-
|
|
1098
|
-
|
|
1099
|
-
|
|
1100
|
-
|
|
1101
|
-
|
|
1102
|
-
|
|
1103
|
-
|
|
1104
|
-
)
|
|
1105
|
-
mutation_restore_sha = tree_sha256(current_root)
|
|
1106
|
-
else:
|
|
1107
|
-
raise RequestError("differential method must be baseline or controlled-mutation")
|
|
1108
|
-
current_receipt = execute_request(
|
|
1109
|
-
project_root, evidence_dir, current_request_path, execution_root=current_root,
|
|
1110
|
-
isolation={"kind": "current", "tree_sha256": tree_sha256(current_root)},
|
|
1010
|
+
if request["method"] == "baseline":
|
|
1011
|
+
worktree_path = tempfile.mkdtemp(prefix="prizmkit-baseline-")
|
|
1012
|
+
result = subprocess.run(
|
|
1013
|
+
["git", "-C", str(project_root), "worktree", "add", "--detach", str(worktree_path), request["baseline_commit"]],
|
|
1014
|
+
capture_output=True, check=False,
|
|
1015
|
+
)
|
|
1016
|
+
if result.returncode != 0:
|
|
1017
|
+
raise RequestError(f"cannot create baseline worktree: {result.stderr.decode(errors='replace')}")
|
|
1018
|
+
for relative in overlay_paths:
|
|
1019
|
+
source = confined(project_root, relative, must_exist=True)
|
|
1020
|
+
destination = Path(worktree_path) / relative
|
|
1021
|
+
destination.parent.mkdir(parents=True, exist_ok=True)
|
|
1022
|
+
shutil.copy2(source, destination)
|
|
1023
|
+
baseline_receipt = execute_request(
|
|
1024
|
+
project_root, evidence_dir, current_request_path, execution_root=Path(worktree_path),
|
|
1025
|
+
isolation={"kind": "baseline", "baseline_commit": request["baseline_commit"]},
|
|
1026
|
+
selected_execution=False,
|
|
1027
|
+
)
|
|
1028
|
+
elif request["method"] == "controlled-mutation":
|
|
1029
|
+
worktree_path = tempfile.mkdtemp(prefix="prizmkit-mutation-")
|
|
1030
|
+
result = subprocess.run(
|
|
1031
|
+
["git", "-C", str(project_root), "worktree", "add", "--detach", str(worktree_path), "HEAD"],
|
|
1032
|
+
capture_output=True, check=False,
|
|
1033
|
+
)
|
|
1034
|
+
if result.returncode != 0:
|
|
1035
|
+
raise RequestError(f"cannot create mutation worktree: {result.stderr.decode(errors='replace')}")
|
|
1036
|
+
patch_value = request["mutation_patch_path"]
|
|
1037
|
+
if not isinstance(patch_value, str) or not patch_value:
|
|
1038
|
+
raise RequestError("controlled mutation requires mutation_patch_path")
|
|
1039
|
+
patch_path = request_file(evidence_dir, patch_value)
|
|
1040
|
+
applied = subprocess.run(
|
|
1041
|
+
["git", "apply", "--whitespace=nowarn", str(patch_path)],
|
|
1042
|
+
cwd=worktree_path, capture_output=True, check=False,
|
|
1111
1043
|
)
|
|
1044
|
+
if applied.returncode != 0:
|
|
1045
|
+
raise RequestError(f"controlled mutation could not be applied: {applied.stderr.decode(errors='replace')}")
|
|
1046
|
+
mutation_receipt = execute_request(
|
|
1047
|
+
project_root, evidence_dir, current_request_path, execution_root=Path(worktree_path),
|
|
1048
|
+
isolation={"kind": "controlled-mutation", "patch_sha256": file_sha256(patch_path)},
|
|
1049
|
+
selected_execution=False,
|
|
1050
|
+
)
|
|
1051
|
+
else:
|
|
1052
|
+
raise RequestError("differential method must be baseline or controlled-mutation")
|
|
1053
|
+
current_receipt = execute_request(
|
|
1054
|
+
project_root, evidence_dir, current_request_path, execution_root=project_root,
|
|
1055
|
+
isolation={"kind": "project-direct"},
|
|
1056
|
+
)
|
|
1112
1057
|
cleanup_succeeded = True
|
|
1113
1058
|
finally:
|
|
1114
|
-
|
|
1059
|
+
if worktree_path and Path(worktree_path).exists():
|
|
1060
|
+
subprocess.run(
|
|
1061
|
+
["git", "-C", str(project_root), "worktree", "remove", "--force", str(worktree_path)],
|
|
1062
|
+
capture_output=True, check=False,
|
|
1063
|
+
)
|
|
1064
|
+
try:
|
|
1065
|
+
shutil.rmtree(worktree_path, ignore_errors=True)
|
|
1066
|
+
except OSError:
|
|
1067
|
+
pass
|
|
1115
1068
|
failing_receipt = baseline_receipt or mutation_receipt
|
|
1116
1069
|
failing_output = b""
|
|
1117
1070
|
if failing_receipt:
|
|
@@ -1120,7 +1073,7 @@ def run_differential(project_root: Path, evidence_dir: Path, request_path: Path)
|
|
|
1120
1073
|
proven = bool(
|
|
1121
1074
|
failing_receipt and current_receipt and failing_receipt["exit_code"] != 0
|
|
1122
1075
|
and current_receipt["exit_code"] == 0 and failure_reason_matched
|
|
1123
|
-
and cleanup_succeeded
|
|
1076
|
+
and cleanup_succeeded
|
|
1124
1077
|
)
|
|
1125
1078
|
proof = {
|
|
1126
1079
|
"behavior_id": request["behavior_id"],
|
|
@@ -1129,17 +1082,11 @@ def run_differential(project_root: Path, evidence_dir: Path, request_path: Path)
|
|
|
1129
1082
|
"differential_request_path": evidence_relative(evidence_dir, request_path),
|
|
1130
1083
|
"differential_request_sha256": file_sha256(request_path),
|
|
1131
1084
|
"baseline_commit": request["baseline_commit"],
|
|
1132
|
-
"current_tree_sha256": request["current_tree_sha256"],
|
|
1133
1085
|
"baseline_execution_id": baseline_receipt["execution_id"] if baseline_receipt else None,
|
|
1134
1086
|
"mutation_execution_id": mutation_receipt["execution_id"] if mutation_receipt else None,
|
|
1135
1087
|
"current_execution_id": current_receipt["execution_id"] if current_receipt else None,
|
|
1136
1088
|
"failure_reason_matched": failure_reason_matched,
|
|
1137
1089
|
"cleanup_succeeded": cleanup_succeeded,
|
|
1138
|
-
"project_tree_before_sha256": project_before,
|
|
1139
|
-
"project_tree_after_sha256": project_after,
|
|
1140
|
-
"isolation_tree_sha256": isolation_tree,
|
|
1141
|
-
"mutation_apply_sha256": mutation_apply_sha,
|
|
1142
|
-
"mutation_restore_sha256": mutation_restore_sha,
|
|
1143
1090
|
"not_applicable": None,
|
|
1144
1091
|
}
|
|
1145
1092
|
append_proof(evidence_dir, proof)
|
|
@@ -1163,7 +1110,6 @@ def run_capture_change(
|
|
|
1163
1110
|
"patch_path": evidence_relative(evidence_dir, patch_path),
|
|
1164
1111
|
"patch_sha256": file_sha256(patch_path),
|
|
1165
1112
|
"changed_files": status_entries,
|
|
1166
|
-
"source_tree_sha256": tree_sha256(project_root, [evidence_dir]),
|
|
1167
1113
|
}
|
|
1168
1114
|
output = confined(evidence_dir, output_name)
|
|
1169
1115
|
write_json_atomic(output, capture)
|
|
@@ -1494,25 +1440,15 @@ def build_manifest(evidence_dir: Path, *, evidence_id: str, identity: dict[str,
|
|
|
1494
1440
|
outputs_by_stage = {stage: [] for stage in STAGES}
|
|
1495
1441
|
for entry in files:
|
|
1496
1442
|
outputs_by_stage.setdefault(entry["produced_by"], []).append(entry["path"])
|
|
1497
|
-
predecessor_files: list[dict[str, str]] = []
|
|
1498
1443
|
stages = []
|
|
1499
1444
|
for stage in STAGES:
|
|
1500
1445
|
outputs = sorted(outputs_by_stage.get(stage, []))
|
|
1501
1446
|
stages.append({
|
|
1502
1447
|
"name": stage,
|
|
1503
1448
|
"status": "complete",
|
|
1504
|
-
"input_sha256": canonical_sha256({
|
|
1505
|
-
"stage": stage,
|
|
1506
|
-
"target_hashes": target_hashes,
|
|
1507
|
-
"predecessor_outputs": sorted(predecessor_files, key=lambda item: item["path"]),
|
|
1508
|
-
}),
|
|
1509
1449
|
"outputs": outputs,
|
|
1510
1450
|
"not_applicable": None,
|
|
1511
1451
|
})
|
|
1512
|
-
predecessor_files.extend(
|
|
1513
|
-
{"path": output, "sha256": next(item["sha256"] for item in files if item["path"] == output)}
|
|
1514
|
-
for output in outputs if output not in FINAL_RECORDS
|
|
1515
|
-
)
|
|
1516
1452
|
return {
|
|
1517
1453
|
"protocol_version": "1.0",
|
|
1518
1454
|
"evidence_id": evidence_id,
|
|
@@ -1641,6 +1577,7 @@ def run_finalize(project_root: Path, evidence_dir: Path, request: dict[str, Any]
|
|
|
1641
1577
|
"finalized": False,
|
|
1642
1578
|
"immutable_identity": identity,
|
|
1643
1579
|
})
|
|
1580
|
+
# First manifest: pinned before render-report so the report can reference it.
|
|
1644
1581
|
manifest = build_manifest(
|
|
1645
1582
|
evidence_dir, evidence_id=evidence_id, identity=identity,
|
|
1646
1583
|
target_hashes=target_hashes, change_class=change_class,
|
|
@@ -1664,6 +1601,7 @@ def run_finalize(project_root: Path, evidence_dir: Path, request: dict[str, Any]
|
|
|
1664
1601
|
"validation_path": "validation.json",
|
|
1665
1602
|
"next": "validate_test_evidence.py --attest",
|
|
1666
1603
|
})
|
|
1604
|
+
# Second manifest: re-hashes after render-report produces test-report.md.
|
|
1667
1605
|
manifest = build_manifest(
|
|
1668
1606
|
evidence_dir, evidence_id=evidence_id, identity=identity,
|
|
1669
1607
|
target_hashes=target_hashes, change_class=change_class,
|
|
@@ -1673,6 +1611,13 @@ def run_finalize(project_root: Path, evidence_dir: Path, request: dict[str, Any]
|
|
|
1673
1611
|
destination = evidence_dir.parent / evidence_id
|
|
1674
1612
|
if destination != evidence_dir:
|
|
1675
1613
|
if destination.exists():
|
|
1614
|
+
try:
|
|
1615
|
+
existing_man = load_json(destination / "manifest.json")
|
|
1616
|
+
if isinstance(existing_man, dict) and existing_man.get("final_verdict") == "TEST_PASS":
|
|
1617
|
+
write_handoff_pointer(destination, lifecycle.get("artifact_dir"), project_root)
|
|
1618
|
+
return destination / output_name
|
|
1619
|
+
except Exception:
|
|
1620
|
+
pass
|
|
1676
1621
|
raise RequestError(
|
|
1677
1622
|
f"evidence identity already exists at {destination}; re-init a new package"
|
|
1678
1623
|
)
|
|
@@ -1796,7 +1741,10 @@ def main() -> int:
|
|
|
1796
1741
|
try:
|
|
1797
1742
|
if not project_root.is_dir():
|
|
1798
1743
|
raise RequestError(f"project root does not exist: {project_root}")
|
|
1799
|
-
|
|
1744
|
+
try:
|
|
1745
|
+
evidence_dir.relative_to(project_root)
|
|
1746
|
+
except ValueError:
|
|
1747
|
+
raise RequestError(f"evidence directory must be under project root: {evidence_dir}")
|
|
1800
1748
|
if args.subcommand == "init":
|
|
1801
1749
|
output = run_init(project_root, evidence_dir, args.baseline, args.output)
|
|
1802
1750
|
print(f"Evidence initialized: {output}")
|
|
@@ -52,16 +52,6 @@ LIGHTWEIGHT_RECORDS = {"lightweight-verification.json"}
|
|
|
52
52
|
SHA256_RE = re.compile(r"^[a-f0-9]{64}$")
|
|
53
53
|
UUID_RE = re.compile(r"^[a-f0-9-]{36}$")
|
|
54
54
|
BLOCKED_EXTERNAL_CLASSES = {"production", "unknown"}
|
|
55
|
-
SNAPSHOT_VOLATILE_ROOTS = (
|
|
56
|
-
".prizmkit/state",
|
|
57
|
-
".prizmkit/test/evidence",
|
|
58
|
-
".prizmkit/" + "dev-" + "pipeline",
|
|
59
|
-
".claude/worktrees",
|
|
60
|
-
".agents/worktrees",
|
|
61
|
-
".codebuddy/worktrees",
|
|
62
|
-
".codex/worktrees",
|
|
63
|
-
)
|
|
64
|
-
SNAPSHOT_VOLATILE_NAMES = {".git", "__pycache__", ".pytest_cache", ".mypy_cache"}
|
|
65
55
|
RISK_CONFLICT_PATTERNS = {
|
|
66
56
|
"permission": re.compile(r"auth|tenant|role|permission|access|acl|rbac", re.I),
|
|
67
57
|
"concurrency": re.compile(r"lock|shared[ _-]?state|worker|queue|concurr|parallel|race", re.I),
|
|
@@ -455,14 +445,6 @@ def validate_external_targets(value: Any, location: str, errors: list[str]) -> N
|
|
|
455
445
|
require(isinstance(deny_evidence, list) and not deny_evidence, f"{target_location}: external target has deny evidence", errors)
|
|
456
446
|
|
|
457
447
|
|
|
458
|
-
def stage_input_sha(stage_name: str, target_hashes: dict[str, str], predecessor_files: list[dict[str, str]]) -> str:
|
|
459
|
-
return canonical_sha256({
|
|
460
|
-
"stage": stage_name,
|
|
461
|
-
"target_hashes": target_hashes,
|
|
462
|
-
"predecessor_outputs": sorted(predecessor_files, key=lambda item: item["path"]),
|
|
463
|
-
})
|
|
464
|
-
|
|
465
|
-
|
|
466
448
|
def validate_manifest(
|
|
467
449
|
root: Path,
|
|
468
450
|
manifest: dict[str, Any],
|
|
@@ -517,7 +499,6 @@ def validate_manifest(
|
|
|
517
499
|
return entry_map
|
|
518
500
|
names = [stage.get("name") for stage in stages if isinstance(stage, dict)]
|
|
519
501
|
require(names == STAGES, "stages are missing or out of order", errors)
|
|
520
|
-
predecessor_files: list[dict[str, str]] = []
|
|
521
502
|
for stage in stages:
|
|
522
503
|
if not isinstance(stage, dict):
|
|
523
504
|
continue
|
|
@@ -535,13 +516,6 @@ def validate_manifest(
|
|
|
535
516
|
require(entry is not None, f"stage output is not a hashed evidence file: {name}/{output}", errors)
|
|
536
517
|
if entry is not None:
|
|
537
518
|
require(entry.get("produced_by") == name, f"stage output producer mismatch: {name}/{output}", errors)
|
|
538
|
-
if isinstance(name, str) and isinstance(manifest.get("target_hashes"), dict):
|
|
539
|
-
expected_input = stage_input_sha(name, manifest["target_hashes"], predecessor_files)
|
|
540
|
-
require(stage.get("input_sha256") == expected_input, f"stage input dependency hash mismatch: {name}", errors)
|
|
541
|
-
predecessor_files.extend(
|
|
542
|
-
{"path": output, "sha256": entry_map[output]["sha256"]}
|
|
543
|
-
for output in outputs if isinstance(outputs, list) and output in entry_map and output not in FINAL_RECORDS
|
|
544
|
-
)
|
|
545
519
|
return entry_map
|
|
546
520
|
|
|
547
521
|
|
|
@@ -832,7 +806,6 @@ def validate_requests_and_receipts(
|
|
|
832
806
|
receipt_by_id: dict[str, dict[str, Any]] = {}
|
|
833
807
|
seen_receipt_paths: set[str] = set()
|
|
834
808
|
attempts_by_request: dict[str, int] = {}
|
|
835
|
-
previous_receipt: dict[str, Any] | None = None
|
|
836
809
|
for index, receipt in enumerate(executions):
|
|
837
810
|
location = f"executions[{index}]"
|
|
838
811
|
if not isinstance(receipt, dict):
|
|
@@ -904,8 +877,6 @@ def validate_requests_and_receipts(
|
|
|
904
877
|
errors.append(str(exc))
|
|
905
878
|
seen_receipt_paths.add(recorded_receipt_path)
|
|
906
879
|
|
|
907
|
-
expected_previous = canonical_sha256(previous_receipt) if previous_receipt is not None else None
|
|
908
|
-
require(receipt.get("previous_receipt_sha256") == expected_previous, f"{location}: receipt chain mismatch", errors)
|
|
909
880
|
request_hash = receipt.get("request_sha256")
|
|
910
881
|
if isinstance(request_hash, str):
|
|
911
882
|
attempts_by_request[request_hash] = attempts_by_request.get(request_hash, 0) + 1
|
|
@@ -916,7 +887,6 @@ def validate_requests_and_receipts(
|
|
|
916
887
|
require(previous is not None, f"{location}: replay source is missing", errors)
|
|
917
888
|
if previous is not None:
|
|
918
889
|
require(previous.get("request_sha256") == request_hash, f"{location}: replay request differs from source", errors)
|
|
919
|
-
previous_receipt = receipt
|
|
920
890
|
|
|
921
891
|
receipt_files = {
|
|
922
892
|
path.relative_to(root).as_posix()
|
|
@@ -985,11 +955,6 @@ def validate_canonical_capture(
|
|
|
985
955
|
require(patch_relative == "source-change.patch", "canonical capture patch_path must be source-change.patch", errors)
|
|
986
956
|
if patch is not None:
|
|
987
957
|
require(file_sha256(patch) == capture.get("patch_sha256"), "canonical capture patch hash mismatch", errors)
|
|
988
|
-
require(
|
|
989
|
-
capture.get("source_tree_sha256") == tree_sha256(project_root, root),
|
|
990
|
-
"canonical change capture source tree is stale",
|
|
991
|
-
errors,
|
|
992
|
-
)
|
|
993
958
|
captured_paths = {
|
|
994
959
|
item.get("path")
|
|
995
960
|
for item in capture.get("changed_files", [])
|
|
@@ -1240,33 +1205,6 @@ def validate_matrix_risks(
|
|
|
1240
1205
|
errors.append(f"TEST_PASS has unresolved behavior risk: {behavior_id}/{risk_name}")
|
|
1241
1206
|
|
|
1242
1207
|
|
|
1243
|
-
def is_volatile_snapshot_path(root: Path, candidate: Path, evidence_root: Path) -> bool:
|
|
1244
|
-
try:
|
|
1245
|
-
relative = candidate.relative_to(root)
|
|
1246
|
-
except ValueError:
|
|
1247
|
-
return True
|
|
1248
|
-
if any(part in SNAPSHOT_VOLATILE_NAMES for part in relative.parts):
|
|
1249
|
-
return True
|
|
1250
|
-
if any(relative == Path(prefix) or Path(prefix) in relative.parents for prefix in SNAPSHOT_VOLATILE_ROOTS):
|
|
1251
|
-
return True
|
|
1252
|
-
resolved = candidate.resolve()
|
|
1253
|
-
excluded = evidence_root.resolve()
|
|
1254
|
-
return resolved == excluded or excluded in resolved.parents
|
|
1255
|
-
|
|
1256
|
-
|
|
1257
|
-
def tree_sha256(root: Path, evidence_root: Path) -> str:
|
|
1258
|
-
entries: list[dict[str, str]] = []
|
|
1259
|
-
for candidate in sorted(root.rglob("*")):
|
|
1260
|
-
if not candidate.is_file() or is_volatile_snapshot_path(root, candidate, evidence_root):
|
|
1261
|
-
continue
|
|
1262
|
-
entries.append({"path": candidate.relative_to(root).as_posix(), "sha256": file_sha256(candidate)})
|
|
1263
|
-
return canonical_sha256(entries)
|
|
1264
|
-
|
|
1265
|
-
|
|
1266
|
-
def require_sha_or_null(value: Any, location: str, errors: list[str]) -> None:
|
|
1267
|
-
require(value is None or is_sha256(value), f"{location} must be a sha256 or null", errors)
|
|
1268
|
-
|
|
1269
|
-
|
|
1270
1208
|
def validate_differential_proofs(
|
|
1271
1209
|
root: Path,
|
|
1272
1210
|
project_root: Path,
|
|
@@ -1283,7 +1221,6 @@ def validate_differential_proofs(
|
|
|
1283
1221
|
receipts = {item.get("execution_id"): item for item in executions if isinstance(item.get("execution_id"), str)}
|
|
1284
1222
|
behavior_ids = {item.get("id") for item in matrix.get("behaviors", []) if isinstance(item, dict)}
|
|
1285
1223
|
proof_by_behavior: dict[str, dict[str, Any]] = {}
|
|
1286
|
-
live_tree_hash = tree_sha256(project_root, root)
|
|
1287
1224
|
proofs = proof.get("proofs")
|
|
1288
1225
|
if not isinstance(proofs, list):
|
|
1289
1226
|
return
|
|
@@ -1300,9 +1237,7 @@ def validate_differential_proofs(
|
|
|
1300
1237
|
validate_differential_na(item.get("not_applicable"), f"differential proof {behavior_id} not_applicable", errors)
|
|
1301
1238
|
for key in (
|
|
1302
1239
|
"method", "differential_request_path", "differential_request_sha256", "baseline_commit",
|
|
1303
|
-
"
|
|
1304
|
-
"project_tree_before_sha256", "project_tree_after_sha256", "isolation_tree_sha256",
|
|
1305
|
-
"mutation_apply_sha256", "mutation_restore_sha256",
|
|
1240
|
+
"baseline_execution_id", "mutation_execution_id", "current_execution_id",
|
|
1306
1241
|
):
|
|
1307
1242
|
require(item.get(key) is None, f"N/A differential proof must clear {key}: {behavior_id}", errors)
|
|
1308
1243
|
require(item.get("failure_reason_matched") is False, f"N/A differential proof cannot claim a failure match: {behavior_id}", errors)
|
|
@@ -1320,14 +1255,6 @@ def validate_differential_proofs(
|
|
|
1320
1255
|
method = item.get("method")
|
|
1321
1256
|
require(method == request.get("method") and method in {"baseline", "controlled-mutation"}, f"differential method mismatch: {behavior_id}", errors)
|
|
1322
1257
|
require(item.get("baseline_commit") == request.get("baseline_commit") == manifest.get("baseline_commit"), f"differential baseline commit mismatch: {behavior_id}", errors)
|
|
1323
|
-
require(item.get("current_tree_sha256") == request.get("current_tree_sha256") == live_tree_hash, f"differential current tree hash mismatch: {behavior_id}", errors)
|
|
1324
|
-
for key in (
|
|
1325
|
-
"differential_request_sha256", "current_tree_sha256", "project_tree_before_sha256",
|
|
1326
|
-
"project_tree_after_sha256", "isolation_tree_sha256", "mutation_apply_sha256", "mutation_restore_sha256",
|
|
1327
|
-
):
|
|
1328
|
-
require_sha_or_null(item.get(key), f"differential proof {behavior_id}.{key}", errors)
|
|
1329
|
-
require(item.get("project_tree_before_sha256") == live_tree_hash, f"differential project tree before mismatch: {behavior_id}", errors)
|
|
1330
|
-
require(item.get("project_tree_after_sha256") == live_tree_hash, f"differential project tree after mismatch: {behavior_id}", errors)
|
|
1331
1258
|
require(item.get("cleanup_succeeded") is True, f"differential cleanup failed: {behavior_id}", errors)
|
|
1332
1259
|
|
|
1333
1260
|
current = receipts.get(item.get("current_execution_id"))
|
|
@@ -1345,10 +1272,8 @@ def validate_differential_proofs(
|
|
|
1345
1272
|
require(failing.get("selected_execution") is False and failing.get("reliable") is True and failing.get("exit_code") != 0, f"differential failing side is not a reliable failure: {behavior_id}", errors)
|
|
1346
1273
|
current_isolation = current.get("isolation")
|
|
1347
1274
|
failing_isolation = failing.get("isolation")
|
|
1348
|
-
require(isinstance(current_isolation, dict) and current_isolation.get("kind") == "
|
|
1275
|
+
require(isinstance(current_isolation, dict) and current_isolation.get("kind") == "project-direct", f"differential current receipt lacks project-direct isolation: {behavior_id}", errors)
|
|
1349
1276
|
require(isinstance(failing_isolation, dict) and failing_isolation.get("kind") == method, f"differential failing receipt isolation mismatch: {behavior_id}", errors)
|
|
1350
|
-
if isinstance(current_isolation, dict):
|
|
1351
|
-
require(current_isolation.get("tree_sha256") == item.get("isolation_tree_sha256") == live_tree_hash, f"differential current isolation hash mismatch: {behavior_id}", errors)
|
|
1352
1277
|
if method == "baseline" and isinstance(failing_isolation, dict):
|
|
1353
1278
|
require(failing_isolation.get("baseline_commit") == request.get("baseline_commit"), f"baseline receipt commit mismatch: {behavior_id}", errors)
|
|
1354
1279
|
if method == "controlled-mutation" and isinstance(failing_isolation, dict):
|
|
@@ -1358,12 +1283,9 @@ def validate_differential_proofs(
|
|
|
1358
1283
|
if patch_path is not None:
|
|
1359
1284
|
patch_hash = file_sha256(patch_path)
|
|
1360
1285
|
require(failing_isolation.get("patch_sha256") == patch_hash, f"controlled mutation receipt patch hash mismatch: {behavior_id}", errors)
|
|
1361
|
-
require(item.get("mutation_apply_sha256") == failing_isolation.get("tree_sha256"), f"controlled mutation apply hash mismatch: {behavior_id}", errors)
|
|
1362
|
-
require(item.get("mutation_restore_sha256") == item.get("isolation_tree_sha256"), f"controlled mutation restore hash mismatch: {behavior_id}", errors)
|
|
1363
1286
|
if method == "baseline":
|
|
1364
1287
|
require(item.get("mutation_execution_id") is None, f"baseline proof has mutation receipt: {behavior_id}", errors)
|
|
1365
1288
|
require(request.get("mutation_patch_path") is None, f"baseline request has mutation patch: {behavior_id}", errors)
|
|
1366
|
-
require(item.get("mutation_apply_sha256") is None and item.get("mutation_restore_sha256") is None, f"baseline proof has mutation hashes: {behavior_id}", errors)
|
|
1367
1289
|
elif method == "controlled-mutation":
|
|
1368
1290
|
require(item.get("baseline_execution_id") is None, f"mutation proof has baseline receipt: {behavior_id}", errors)
|
|
1369
1291
|
require(isinstance(request.get("mutation_patch_path"), str) and bool(request.get("mutation_patch_path")), f"mutation request lacks patch: {behavior_id}", errors)
|