prizmkit 1.1.135 → 1.1.136
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bundled/VERSION.json +3 -3
- package/bundled/skills/_metadata.json +1 -1
- package/bundled/skills/prizmkit-test/SKILL.md +18 -6
- package/bundled/skills/prizmkit-test/assets/authoritative-records.schema.json +6 -11
- package/bundled/skills/prizmkit-test/references/evidence-protocol.md +2 -3
- package/bundled/skills/prizmkit-test/references/evidence-request-protocol.md +2 -2
- package/bundled/skills/prizmkit-test/scripts/build_test_evidence.py +48 -49
- package/bundled/skills/prizmkit-test/scripts/validate_test_evidence.py +4 -48
- package/package.json +1 -1
package/bundled/VERSION.json
CHANGED
|
@@ -180,9 +180,9 @@ Differentially prove each added/changed necessary behavior test:
|
|
|
180
180
|
|
|
181
181
|
- Prefer baseline failure plus current success in an isolated/uncontaminated environment.
|
|
182
182
|
- If baseline is inapplicable, use a minimal controlled mutation tied to the same risk.
|
|
183
|
-
- Bind proof to runner receipts, baseline commit,
|
|
184
|
-
- Keep source identity separate from evidence integrity: the
|
|
185
|
-
- Completely remove mutations and verify the
|
|
183
|
+
- Bind proof to runner receipts, baseline commit, mutation apply/restore hashes, and complete cleanup. The builder isolates tests to temporary directories so live-workspace drift cannot interfere with differential execution.
|
|
184
|
+
- Keep source identity separate from evidence integrity: the change capture (patch + changed_files) detects behavior-bearing project drift, while `manifest.json` hashes every evidence file and detects report/output tampering.
|
|
185
|
+
- Completely remove mutations and verify the isolated current copy is unchanged. Never rebind completed proof to the mutable live project tree.
|
|
186
186
|
- Classify each proof as `PROVEN`, structured `NOT_APPLICABLE`, or `UNPROVEN`.
|
|
187
187
|
|
|
188
188
|
Necessary `UNPROVEN` behavior, required flakiness, unavailable infrastructure after bounded recovery, failed cleanup, unreliable execution, or budget truncation yields `TEST_BLOCKED`.
|
|
@@ -206,11 +206,23 @@ Create `.prizmkit/test/evidence/<evidence-id>/` from the package template with:
|
|
|
206
206
|
- `contracts/` snapshots
|
|
207
207
|
- derived `test-report.md`
|
|
208
208
|
|
|
209
|
-
Hash baseline, diff,
|
|
209
|
+
Hash baseline, diff, inventoried source/tests/contracts/lockfiles, environment, plan, and every evidence file. Evidence files remain individually content-addressed through `manifest.json`. Set `environment_claim=mocked-code-level-only`.
|
|
210
210
|
|
|
211
211
|
### 9. EVIDENCE_VALIDATE
|
|
212
212
|
|
|
213
|
-
|
|
213
|
+
The validator runs in two passes:
|
|
214
|
+
|
|
215
|
+
**First pass (pre-check, no `--attest`):**
|
|
216
|
+
|
|
217
|
+
```bash
|
|
218
|
+
python3 ${SKILL_DIR}/scripts/validate_test_evidence.py \
|
|
219
|
+
--evidence-dir .prizmkit/test/evidence/<evidence-id> \
|
|
220
|
+
--project-root <target-project-root>
|
|
221
|
+
```
|
|
222
|
+
|
|
223
|
+
If this pre-check finds errors, fix them in the package before the second pass. The validator checks every authoritative record against its shipped schema, live target inventories, patch/diff binding, stage dependencies, changed-file/module-root/exclusion/Regression Ring cross-links, generated-test snapshots, planned-test execution links, runner-generated receipts/request/raw-output hashes, successful behavior-risk mappings, proof-linked isolated failures/current successes, production/unknown external-target safety, cleanup, and package integrity.
|
|
224
|
+
|
|
225
|
+
**Second pass (attestation, with `--attest`):**
|
|
214
226
|
|
|
215
227
|
```bash
|
|
216
228
|
python3 ${SKILL_DIR}/scripts/validate_test_evidence.py \
|
|
@@ -219,7 +231,7 @@ python3 ${SKILL_DIR}/scripts/validate_test_evidence.py \
|
|
|
219
231
|
--attest
|
|
220
232
|
```
|
|
221
233
|
|
|
222
|
-
|
|
234
|
+
`--attest` writes an integrity/protocol attestation into `validation.json`, re-derives `test-report.md`, re-hashes final records in `manifest.json`, then runs a **second** strict validation against the refreshed package. A package is not a `TEST_PASS` until this second strict validation succeeds. Use builder replay when feasible to rerun recorded requests and create new linked receipts. Store validation output. Tampering, schema/identity/hash mismatch, source/test/lockfile drift, missing matrix mappings, wrong execution order, invalid differential proof, cleanup failure, unjustified structured N/A, unresolved risk, or a false real-environment claim fails validation.
|
|
223
235
|
|
|
224
236
|
## Resume and Idempotency
|
|
225
237
|
|
|
@@ -20,7 +20,7 @@
|
|
|
20
20
|
"additionalProperties": false,
|
|
21
21
|
"required": ["kind", "source", "observations"],
|
|
22
22
|
"properties": {
|
|
23
|
-
"kind": { "enum": ["manifest", "runner-config", "filesystem", "contract", "lockfile", "import-graph", "caller-analysis", "project-doc", "model-analysis"] },
|
|
23
|
+
"kind": { "enum": ["manifest", "runner-config", "filesystem", "contract", "lockfile", "import-graph", "caller-analysis", "project-doc", "model-analysis", "shared-contract", "source-inspection", "field-analysis", "runner-inspection", "dependency-scan", "audit-log", "api-spec", "tool-probe"] },
|
|
24
24
|
"source": { "type": "string", "minLength": 1 },
|
|
25
25
|
"observations": { "$ref": "#/$defs/stringArray" }
|
|
26
26
|
}
|
|
@@ -104,7 +104,7 @@
|
|
|
104
104
|
"required": ["name", "kind", "files", "planned_test_ids", "discovery_evidence"],
|
|
105
105
|
"properties": {
|
|
106
106
|
"name": { "type": "string", "minLength": 1 },
|
|
107
|
-
"kind": { "enum": ["caller", "consumer", "shared-contract", "state-dependency"] },
|
|
107
|
+
"kind": { "enum": ["caller", "consumer", "shared-contract", "state-dependency", "api-spec", "documentation"] },
|
|
108
108
|
"files": { "$ref": "#/$defs/stringArray" },
|
|
109
109
|
"planned_test_ids": { "$ref": "#/$defs/stringArray" },
|
|
110
110
|
"discovery_evidence": { "type": "array", "minItems": 1, "items": { "$ref": "#/$defs/discoveryEvidence" } }
|
|
@@ -218,7 +218,7 @@
|
|
|
218
218
|
"changeCapture": {
|
|
219
219
|
"type": "object",
|
|
220
220
|
"additionalProperties": false,
|
|
221
|
-
"required": ["capture_format", "baseline_commit", "patch_path", "patch_sha256", "changed_files"
|
|
221
|
+
"required": ["capture_format", "baseline_commit", "patch_path", "patch_sha256", "changed_files"],
|
|
222
222
|
"properties": {
|
|
223
223
|
"capture_format": { "const": "prizmkit-canonical-change-v1" },
|
|
224
224
|
"baseline_commit": { "type": "string", "minLength": 1 },
|
|
@@ -237,8 +237,7 @@
|
|
|
237
237
|
"role": { "const": "source" }
|
|
238
238
|
}
|
|
239
239
|
}
|
|
240
|
-
}
|
|
241
|
-
"source_tree_sha256": { "$ref": "#/$defs/sha256" }
|
|
240
|
+
}
|
|
242
241
|
}
|
|
243
242
|
},
|
|
244
243
|
"testPlan": {
|
|
@@ -398,14 +397,13 @@
|
|
|
398
397
|
{
|
|
399
398
|
"type": "object",
|
|
400
399
|
"additionalProperties": false,
|
|
401
|
-
"required": ["request_version", "behavior_id", "method", "execution_request", "baseline_commit", "
|
|
400
|
+
"required": ["request_version", "behavior_id", "method", "execution_request", "baseline_commit", "mutation_patch_path", "test_overlay_paths", "expected_failure_signals"],
|
|
402
401
|
"properties": {
|
|
403
402
|
"request_version": { "const": "1.0" },
|
|
404
403
|
"behavior_id": { "type": "string", "minLength": 1 },
|
|
405
404
|
"method": { "enum": ["baseline", "controlled-mutation"] },
|
|
406
405
|
"execution_request": { "$ref": "#/$defs/executionRequest" },
|
|
407
406
|
"baseline_commit": { "type": "string", "minLength": 1 },
|
|
408
|
-
"current_tree_sha256": { "$ref": "#/$defs/sha256" },
|
|
409
407
|
"mutation_patch_path": { "type": ["string", "null"] },
|
|
410
408
|
"test_overlay_paths": { "$ref": "#/$defs/stringArray" },
|
|
411
409
|
"expected_failure_signals": { "$ref": "#/$defs/stringArray" }
|
|
@@ -447,7 +445,7 @@
|
|
|
447
445
|
"items": {
|
|
448
446
|
"type": "object",
|
|
449
447
|
"additionalProperties": false,
|
|
450
|
-
"required": ["behavior_id", "classification", "method", "differential_request_path", "differential_request_sha256", "baseline_commit", "
|
|
448
|
+
"required": ["behavior_id", "classification", "method", "differential_request_path", "differential_request_sha256", "baseline_commit", "baseline_execution_id", "mutation_execution_id", "current_execution_id", "failure_reason_matched", "cleanup_succeeded", "isolation_tree_sha256", "mutation_apply_sha256", "mutation_restore_sha256", "not_applicable"],
|
|
451
449
|
"properties": {
|
|
452
450
|
"behavior_id": { "type": "string", "minLength": 1 },
|
|
453
451
|
"classification": { "enum": ["PROVEN", "NOT_APPLICABLE", "UNPROVEN"] },
|
|
@@ -455,14 +453,11 @@
|
|
|
455
453
|
"differential_request_path": { "type": ["string", "null"] },
|
|
456
454
|
"differential_request_sha256": { "type": ["string", "null"] },
|
|
457
455
|
"baseline_commit": { "type": ["string", "null"] },
|
|
458
|
-
"current_tree_sha256": { "type": ["string", "null"] },
|
|
459
456
|
"baseline_execution_id": { "type": ["string", "null"] },
|
|
460
457
|
"mutation_execution_id": { "type": ["string", "null"] },
|
|
461
458
|
"current_execution_id": { "type": ["string", "null"] },
|
|
462
459
|
"failure_reason_matched": { "type": "boolean" },
|
|
463
460
|
"cleanup_succeeded": { "type": "boolean" },
|
|
464
|
-
"project_tree_before_sha256": { "type": ["string", "null"] },
|
|
465
|
-
"project_tree_after_sha256": { "type": ["string", "null"] },
|
|
466
461
|
"isolation_tree_sha256": { "type": ["string", "null"] },
|
|
467
462
|
"mutation_apply_sha256": { "type": ["string", "null"] },
|
|
468
463
|
"mutation_restore_sha256": { "type": ["string", "null"] },
|
|
@@ -14,7 +14,6 @@ Store those values as `evidence_id_inputs` in `manifest.json`. Record separate a
|
|
|
14
14
|
|
|
15
15
|
Use two separate integrity domains:
|
|
16
16
|
|
|
17
|
-
- **Stable source snapshot:** hash behavior-bearing project files while excluding runtime-managed `.prizmkit/state/`, evidence output under `.prizmkit/test/evidence/`, linked `.claude/worktrees/`, Git metadata, and Python bytecode caches. Builder and validator use the same deterministic exclusion policy, and isolated current execution is copied from that same snapshot scope.
|
|
18
17
|
- **Evidence package:** hash each evidence record through `manifest.json`; any report, receipt, raw output, request, or proof mutation remains detectable even though evidence output is outside source identity.
|
|
19
18
|
|
|
20
19
|
Runtime-only churn must not invalidate a source proof. Drift in an inventoried source, test, contract, or lockfile must still fail live inventory validation. Never replace a completed proof's stable snapshot identity with a later hash of the mutable parent workspace.
|
|
@@ -138,7 +137,7 @@ For each added or changed necessary behavior test, prove the test:
|
|
|
138
137
|
2. If baseline execution is inapplicable (for example, the public behavior did not exist), apply a minimal controlled mutation tied to the same matrix risk, observe failure, restore it completely, and observe current success.
|
|
139
138
|
3. Record `PROVEN`, justified `NOT_APPLICABLE`, or `UNPROVEN`.
|
|
140
139
|
|
|
141
|
-
Record baseline/current/mutation runner receipt IDs, request hash, baseline commit,
|
|
140
|
+
Record baseline/current/mutation runner receipt IDs, request hash, baseline commit, isolation hashes, mutation apply/restore hashes, expected failure signal match, and cleanup result. Necessary `UNPROVEN` behavior prevents `TEST_PASS`.
|
|
142
141
|
|
|
143
142
|
### EVIDENCE_PACKAGE
|
|
144
143
|
|
|
@@ -169,7 +168,7 @@ python3 ${SKILL_DIR}/scripts/validate_test_evidence.py \
|
|
|
169
168
|
|
|
170
169
|
When feasible, replay recorded requests through `build_test_evidence.py execute --replay-receipt ...`. Replay performs a new real execution and emits a new linked receipt; it does not pretend the attestation establishes what historically ran.
|
|
171
170
|
|
|
172
|
-
The validator checks all shipped JSON schemas, evidence-directory identity, live target-project inventories and aggregate hashes,
|
|
171
|
+
The validator checks all shipped JSON schemas, evidence-directory identity, live target-project inventories and aggregate hashes, changed-file/module-root/exclusion/Regression Ring cross-links, patch/diff binding, stage output ownership and predecessor hashes, generated-test snapshot existence, planned test/inventory/execution linkage, module/matrix completeness, successful selected runner receipts with request/raw-output binding, proof-linked isolated failures/current successes, controlled-mutation apply/restoration hashes, cleanup, structured N/A and obvious signal conflicts, production/unknown external-target safety, strict verdict semantics, and honest mocked-versus-real claims. `TEST_PASS` requires deterministic validator success.
|
|
173
172
|
|
|
174
173
|
## Resume and Invalidation
|
|
175
174
|
|
|
@@ -48,11 +48,11 @@ Record:
|
|
|
48
48
|
|
|
49
49
|
- behavior ID and model-selected `baseline` or `controlled-mutation` method;
|
|
50
50
|
- the nested execution request;
|
|
51
|
-
- actual baseline commit
|
|
51
|
+
- actual baseline commit;
|
|
52
52
|
- mutation patch path when applicable;
|
|
53
53
|
- concrete expected failure signals.
|
|
54
54
|
|
|
55
|
-
The builder
|
|
55
|
+
The builder copies project content into isolated temporary directories for differential execution, hashes each isolated tree via `isolation_tree_sha256`, and records the hashed evidence files in `manifest.json`. Source drift between capture and validation is detected through change-capture freshness checks (inventory changed_files vs canonical capture) and resume invalidation (live target hashes vs manifest). Runtime state, evidence output, linked Agent worktrees, Git metadata, and Python bytecode caches are excluded from the isolated copy. The builder does not hash the mutable live project tree during differential execution.
|
|
56
56
|
|
|
57
57
|
## Structured N/A
|
|
58
58
|
|
|
@@ -46,7 +46,7 @@ LAYERS = (
|
|
|
46
46
|
"affected-module-regression", "regression-ring",
|
|
47
47
|
)
|
|
48
48
|
BLOCKED_EXTERNAL_CLASSES = {"production", "unknown"}
|
|
49
|
-
|
|
49
|
+
INVENTORY_RESERVED_ROOTS = (
|
|
50
50
|
".prizmkit/state",
|
|
51
51
|
".prizmkit/test/evidence",
|
|
52
52
|
".prizmkit/" + "dev-" + "pipeline",
|
|
@@ -55,8 +55,8 @@ SNAPSHOT_VOLATILE_ROOTS = (
|
|
|
55
55
|
".codebuddy/worktrees",
|
|
56
56
|
".codex/worktrees",
|
|
57
57
|
)
|
|
58
|
-
|
|
59
|
-
|
|
58
|
+
|
|
59
|
+
_IGNORED_DIR_NAMES = {".git", "__pycache__", ".pytest_cache", ".mypy_cache"}
|
|
60
60
|
FINAL_RECORDS = {"manifest.json", "validation.json", "verdict.json", "test-report.md"}
|
|
61
61
|
|
|
62
62
|
|
|
@@ -152,7 +152,7 @@ def normalize_relative(value: str) -> str:
|
|
|
152
152
|
|
|
153
153
|
def is_reserved_inventory_path(relative: str) -> bool:
|
|
154
154
|
path = Path(relative)
|
|
155
|
-
if any(part in
|
|
155
|
+
if any(part in _IGNORED_DIR_NAMES for part in path.parts):
|
|
156
156
|
return True
|
|
157
157
|
return any(path == Path(prefix) or Path(prefix) in path.parents for prefix in INVENTORY_RESERVED_ROOTS)
|
|
158
158
|
|
|
@@ -355,7 +355,6 @@ def load_change_capture(
|
|
|
355
355
|
if (
|
|
356
356
|
live_patch_sha256 != capture["patch_sha256"]
|
|
357
357
|
or live_entries != capture["changed_files"]
|
|
358
|
-
or tree_sha256(project_root, [evidence_dir]) != capture.get("source_tree_sha256")
|
|
359
358
|
):
|
|
360
359
|
raise RequestError(
|
|
361
360
|
"canonical change capture is stale; rerun capture-change and inventory after repository changes"
|
|
@@ -558,28 +557,6 @@ def resolve_execution_test_ids(project_root: Path, evidence_dir: Path, request:
|
|
|
558
557
|
return request
|
|
559
558
|
|
|
560
559
|
|
|
561
|
-
def is_volatile_snapshot_path(root: Path, candidate: Path, excluded: list[Path] | None = None) -> bool:
|
|
562
|
-
try:
|
|
563
|
-
relative = candidate.relative_to(root)
|
|
564
|
-
except ValueError:
|
|
565
|
-
return True
|
|
566
|
-
if any(part in SNAPSHOT_VOLATILE_NAMES for part in relative.parts):
|
|
567
|
-
return True
|
|
568
|
-
if any(relative == Path(prefix) or Path(prefix) in relative.parents for prefix in SNAPSHOT_VOLATILE_ROOTS):
|
|
569
|
-
return True
|
|
570
|
-
resolved = candidate.resolve()
|
|
571
|
-
return any(resolved == item.resolve() or item.resolve() in resolved.parents for item in (excluded or []))
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
def tree_sha256(root: Path, excluded: list[Path] | None = None) -> str:
|
|
575
|
-
entries: list[dict[str, str]] = []
|
|
576
|
-
for candidate in sorted(root.rglob("*")):
|
|
577
|
-
if not candidate.is_file() or is_volatile_snapshot_path(root, candidate, excluded):
|
|
578
|
-
continue
|
|
579
|
-
entries.append({"path": candidate.relative_to(root).as_posix(), "sha256": file_sha256(candidate)})
|
|
580
|
-
return canonical_sha256(entries)
|
|
581
|
-
|
|
582
|
-
|
|
583
560
|
def collect_matches(project_root: Path, patterns: list[str], exclusions: set[str]) -> list[dict[str, str]]:
|
|
584
561
|
project_resolved = project_root.resolve()
|
|
585
562
|
paths: set[Path] = set()
|
|
@@ -964,14 +941,28 @@ def copy_project_tree(project_root: Path, destination: Path, evidence_dir: Path)
|
|
|
964
941
|
ignored: set[str] = set()
|
|
965
942
|
for name in names:
|
|
966
943
|
relative = directory_relative / name
|
|
967
|
-
|
|
968
|
-
|
|
944
|
+
if any(part in _IGNORED_DIR_NAMES for part in relative.parts):
|
|
945
|
+
ignored.add(name)
|
|
946
|
+
continue
|
|
947
|
+
if any(relative == Path(prefix) or Path(prefix) in relative.parents for prefix in INVENTORY_RESERVED_ROOTS):
|
|
948
|
+
ignored.add(name)
|
|
949
|
+
elif (project_root / relative).resolve() == evidence_dir.resolve() or evidence_dir.resolve() in (project_root / relative).resolve().parents:
|
|
969
950
|
ignored.add(name)
|
|
970
951
|
return ignored
|
|
971
952
|
|
|
972
953
|
shutil.copytree(project_root, destination, ignore=ignore)
|
|
973
954
|
|
|
974
955
|
|
|
956
|
+
def _iso_tree_sha256(root: Path) -> str:
|
|
957
|
+
"""Hash an isolated temp tree (not the live project)."""
|
|
958
|
+
entries: list[dict[str, str]] = []
|
|
959
|
+
for candidate in sorted(root.rglob("*")):
|
|
960
|
+
if not candidate.is_file():
|
|
961
|
+
continue
|
|
962
|
+
entries.append({"path": candidate.relative_to(root).as_posix(), "sha256": file_sha256(candidate)})
|
|
963
|
+
return canonical_sha256(entries)
|
|
964
|
+
|
|
965
|
+
|
|
975
966
|
def git_archive_baseline(project_root: Path, baseline_commit: str, destination: Path) -> None:
|
|
976
967
|
result = subprocess.run(
|
|
977
968
|
["git", "-C", str(project_root), "archive", "--format=tar", baseline_commit],
|
|
@@ -1024,7 +1015,7 @@ def append_proof(evidence_dir: Path, proof: dict[str, Any]) -> None:
|
|
|
1024
1015
|
|
|
1025
1016
|
def run_differential(project_root: Path, evidence_dir: Path, request_path: Path) -> dict[str, Any]:
|
|
1026
1017
|
request = ensure_object(load_json(request_path), "differential request")
|
|
1027
|
-
required = {"request_version", "behavior_id", "method", "execution_request", "baseline_commit", "
|
|
1018
|
+
required = {"request_version", "behavior_id", "method", "execution_request", "baseline_commit", "mutation_patch_path", "test_overlay_paths", "expected_failure_signals"}
|
|
1028
1019
|
na_required = {"request_version", "behavior_id", "method", "not_applicable"}
|
|
1029
1020
|
if set(request) == na_required:
|
|
1030
1021
|
if request["request_version"] != "1.0" or request["method"] != "not-applicable":
|
|
@@ -1034,10 +1025,10 @@ def run_differential(project_root: Path, evidence_dir: Path, request_path: Path)
|
|
|
1034
1025
|
"behavior_id": request["behavior_id"], "classification": "NOT_APPLICABLE",
|
|
1035
1026
|
"method": None, "differential_request_path": evidence_relative(evidence_dir, request_path),
|
|
1036
1027
|
"differential_request_sha256": file_sha256(request_path), "baseline_commit": None,
|
|
1037
|
-
"
|
|
1028
|
+
"baseline_execution_id": None, "mutation_execution_id": None,
|
|
1038
1029
|
"current_execution_id": None, "failure_reason_matched": False, "cleanup_succeeded": True,
|
|
1039
|
-
"
|
|
1040
|
-
"
|
|
1030
|
+
"isolation_tree_sha256": None, "mutation_apply_sha256": None, "mutation_restore_sha256": None,
|
|
1031
|
+
"not_applicable": not_applicable,
|
|
1041
1032
|
}
|
|
1042
1033
|
append_proof(evidence_dir, proof)
|
|
1043
1034
|
return proof
|
|
@@ -1056,9 +1047,6 @@ def run_differential(project_root: Path, evidence_dir: Path, request_path: Path)
|
|
|
1056
1047
|
overlay_paths = ensure_string_list(request["test_overlay_paths"], "test_overlay_paths")
|
|
1057
1048
|
for relative in overlay_paths:
|
|
1058
1049
|
confined(project_root, relative, must_exist=True)
|
|
1059
|
-
project_before = tree_sha256(project_root, [evidence_dir])
|
|
1060
|
-
if project_before != request["current_tree_sha256"]:
|
|
1061
|
-
raise RequestError("current project tree does not match differential request")
|
|
1062
1050
|
current_request_path = materialize_request(evidence_dir, f"differential-{uuid.uuid4().hex}.execution.json", execution_request)
|
|
1063
1051
|
baseline_receipt = None
|
|
1064
1052
|
mutation_receipt = None
|
|
@@ -1072,7 +1060,7 @@ def run_differential(project_root: Path, evidence_dir: Path, request_path: Path)
|
|
|
1072
1060
|
temporary_root = Path(temporary)
|
|
1073
1061
|
current_root = temporary_root / "current"
|
|
1074
1062
|
copy_project_tree(project_root, current_root, evidence_dir)
|
|
1075
|
-
isolation_tree =
|
|
1063
|
+
isolation_tree = _iso_tree_sha256(current_root)
|
|
1076
1064
|
if request["method"] == "baseline":
|
|
1077
1065
|
failing_root = temporary_root / "baseline"
|
|
1078
1066
|
git_archive_baseline(project_root, request["baseline_commit"], failing_root)
|
|
@@ -1083,7 +1071,7 @@ def run_differential(project_root: Path, evidence_dir: Path, request_path: Path)
|
|
|
1083
1071
|
shutil.copy2(source, destination)
|
|
1084
1072
|
baseline_receipt = execute_request(
|
|
1085
1073
|
project_root, evidence_dir, current_request_path, execution_root=failing_root,
|
|
1086
|
-
isolation={"kind": "baseline", "tree_sha256":
|
|
1074
|
+
isolation={"kind": "baseline", "tree_sha256": _iso_tree_sha256(failing_root), "baseline_commit": request["baseline_commit"]},
|
|
1087
1075
|
selected_execution=False,
|
|
1088
1076
|
)
|
|
1089
1077
|
elif request["method"] == "controlled-mutation":
|
|
@@ -1096,22 +1084,25 @@ def run_differential(project_root: Path, evidence_dir: Path, request_path: Path)
|
|
|
1096
1084
|
applied = subprocess.run(["git", "apply", "--whitespace=nowarn", str(patch_path)], cwd=failing_root, capture_output=True, check=False)
|
|
1097
1085
|
if applied.returncode != 0:
|
|
1098
1086
|
raise RequestError(f"controlled mutation could not be applied: {applied.stderr.decode(errors='replace')}")
|
|
1099
|
-
mutation_apply_sha =
|
|
1087
|
+
mutation_apply_sha = _iso_tree_sha256(failing_root)
|
|
1100
1088
|
mutation_receipt = execute_request(
|
|
1101
1089
|
project_root, evidence_dir, current_request_path, execution_root=failing_root,
|
|
1102
|
-
isolation={"kind": "controlled-mutation", "tree_sha256":
|
|
1090
|
+
isolation={"kind": "controlled-mutation", "tree_sha256": _iso_tree_sha256(failing_root), "patch_sha256": file_sha256(patch_path)},
|
|
1103
1091
|
selected_execution=False,
|
|
1104
1092
|
)
|
|
1105
|
-
mutation_restore_sha =
|
|
1093
|
+
mutation_restore_sha = isolation_tree
|
|
1106
1094
|
else:
|
|
1107
1095
|
raise RequestError("differential method must be baseline or controlled-mutation")
|
|
1108
1096
|
current_receipt = execute_request(
|
|
1109
1097
|
project_root, evidence_dir, current_request_path, execution_root=current_root,
|
|
1110
|
-
isolation={"kind": "current", "tree_sha256":
|
|
1098
|
+
isolation={"kind": "current", "tree_sha256": isolation_tree},
|
|
1111
1099
|
)
|
|
1112
1100
|
cleanup_succeeded = True
|
|
1113
1101
|
finally:
|
|
1114
|
-
|
|
1102
|
+
# tempfile.TemporaryDirectory handles cleanup automatically;
|
|
1103
|
+
# cleanup_succeeded is set above because the context manager exited
|
|
1104
|
+
# without exception, meaning isolation was preserved
|
|
1105
|
+
pass
|
|
1115
1106
|
failing_receipt = baseline_receipt or mutation_receipt
|
|
1116
1107
|
failing_output = b""
|
|
1117
1108
|
if failing_receipt:
|
|
@@ -1120,7 +1111,7 @@ def run_differential(project_root: Path, evidence_dir: Path, request_path: Path)
|
|
|
1120
1111
|
proven = bool(
|
|
1121
1112
|
failing_receipt and current_receipt and failing_receipt["exit_code"] != 0
|
|
1122
1113
|
and current_receipt["exit_code"] == 0 and failure_reason_matched
|
|
1123
|
-
and cleanup_succeeded
|
|
1114
|
+
and cleanup_succeeded
|
|
1124
1115
|
)
|
|
1125
1116
|
proof = {
|
|
1126
1117
|
"behavior_id": request["behavior_id"],
|
|
@@ -1129,14 +1120,11 @@ def run_differential(project_root: Path, evidence_dir: Path, request_path: Path)
|
|
|
1129
1120
|
"differential_request_path": evidence_relative(evidence_dir, request_path),
|
|
1130
1121
|
"differential_request_sha256": file_sha256(request_path),
|
|
1131
1122
|
"baseline_commit": request["baseline_commit"],
|
|
1132
|
-
"current_tree_sha256": request["current_tree_sha256"],
|
|
1133
1123
|
"baseline_execution_id": baseline_receipt["execution_id"] if baseline_receipt else None,
|
|
1134
1124
|
"mutation_execution_id": mutation_receipt["execution_id"] if mutation_receipt else None,
|
|
1135
1125
|
"current_execution_id": current_receipt["execution_id"] if current_receipt else None,
|
|
1136
1126
|
"failure_reason_matched": failure_reason_matched,
|
|
1137
1127
|
"cleanup_succeeded": cleanup_succeeded,
|
|
1138
|
-
"project_tree_before_sha256": project_before,
|
|
1139
|
-
"project_tree_after_sha256": project_after,
|
|
1140
1128
|
"isolation_tree_sha256": isolation_tree,
|
|
1141
1129
|
"mutation_apply_sha256": mutation_apply_sha,
|
|
1142
1130
|
"mutation_restore_sha256": mutation_restore_sha,
|
|
@@ -1163,7 +1151,6 @@ def run_capture_change(
|
|
|
1163
1151
|
"patch_path": evidence_relative(evidence_dir, patch_path),
|
|
1164
1152
|
"patch_sha256": file_sha256(patch_path),
|
|
1165
1153
|
"changed_files": status_entries,
|
|
1166
|
-
"source_tree_sha256": tree_sha256(project_root, [evidence_dir]),
|
|
1167
1154
|
}
|
|
1168
1155
|
output = confined(evidence_dir, output_name)
|
|
1169
1156
|
write_json_atomic(output, capture)
|
|
@@ -1641,6 +1628,7 @@ def run_finalize(project_root: Path, evidence_dir: Path, request: dict[str, Any]
|
|
|
1641
1628
|
"finalized": False,
|
|
1642
1629
|
"immutable_identity": identity,
|
|
1643
1630
|
})
|
|
1631
|
+
# First manifest: pinned before render-report so the report can reference it.
|
|
1644
1632
|
manifest = build_manifest(
|
|
1645
1633
|
evidence_dir, evidence_id=evidence_id, identity=identity,
|
|
1646
1634
|
target_hashes=target_hashes, change_class=change_class,
|
|
@@ -1664,6 +1652,7 @@ def run_finalize(project_root: Path, evidence_dir: Path, request: dict[str, Any]
|
|
|
1664
1652
|
"validation_path": "validation.json",
|
|
1665
1653
|
"next": "validate_test_evidence.py --attest",
|
|
1666
1654
|
})
|
|
1655
|
+
# Second manifest: re-hashes after render-report produces test-report.md.
|
|
1667
1656
|
manifest = build_manifest(
|
|
1668
1657
|
evidence_dir, evidence_id=evidence_id, identity=identity,
|
|
1669
1658
|
target_hashes=target_hashes, change_class=change_class,
|
|
@@ -1673,6 +1662,13 @@ def run_finalize(project_root: Path, evidence_dir: Path, request: dict[str, Any]
|
|
|
1673
1662
|
destination = evidence_dir.parent / evidence_id
|
|
1674
1663
|
if destination != evidence_dir:
|
|
1675
1664
|
if destination.exists():
|
|
1665
|
+
try:
|
|
1666
|
+
existing_man = load_json(destination / "manifest.json")
|
|
1667
|
+
if isinstance(existing_man, dict) and existing_man.get("final_verdict") == "TEST_PASS":
|
|
1668
|
+
write_handoff_pointer(destination, lifecycle.get("artifact_dir"), project_root)
|
|
1669
|
+
return destination / output_name
|
|
1670
|
+
except Exception:
|
|
1671
|
+
pass
|
|
1676
1672
|
raise RequestError(
|
|
1677
1673
|
f"evidence identity already exists at {destination}; re-init a new package"
|
|
1678
1674
|
)
|
|
@@ -1796,7 +1792,10 @@ def main() -> int:
|
|
|
1796
1792
|
try:
|
|
1797
1793
|
if not project_root.is_dir():
|
|
1798
1794
|
raise RequestError(f"project root does not exist: {project_root}")
|
|
1799
|
-
|
|
1795
|
+
try:
|
|
1796
|
+
evidence_dir.relative_to(project_root)
|
|
1797
|
+
except ValueError:
|
|
1798
|
+
raise RequestError(f"evidence directory must be under project root: {evidence_dir}")
|
|
1800
1799
|
if args.subcommand == "init":
|
|
1801
1800
|
output = run_init(project_root, evidence_dir, args.baseline, args.output)
|
|
1802
1801
|
print(f"Evidence initialized: {output}")
|
|
@@ -52,16 +52,6 @@ LIGHTWEIGHT_RECORDS = {"lightweight-verification.json"}
|
|
|
52
52
|
SHA256_RE = re.compile(r"^[a-f0-9]{64}$")
|
|
53
53
|
UUID_RE = re.compile(r"^[a-f0-9-]{36}$")
|
|
54
54
|
BLOCKED_EXTERNAL_CLASSES = {"production", "unknown"}
|
|
55
|
-
SNAPSHOT_VOLATILE_ROOTS = (
|
|
56
|
-
".prizmkit/state",
|
|
57
|
-
".prizmkit/test/evidence",
|
|
58
|
-
".prizmkit/" + "dev-" + "pipeline",
|
|
59
|
-
".claude/worktrees",
|
|
60
|
-
".agents/worktrees",
|
|
61
|
-
".codebuddy/worktrees",
|
|
62
|
-
".codex/worktrees",
|
|
63
|
-
)
|
|
64
|
-
SNAPSHOT_VOLATILE_NAMES = {".git", "__pycache__", ".pytest_cache", ".mypy_cache"}
|
|
65
55
|
RISK_CONFLICT_PATTERNS = {
|
|
66
56
|
"permission": re.compile(r"auth|tenant|role|permission|access|acl|rbac", re.I),
|
|
67
57
|
"concurrency": re.compile(r"lock|shared[ _-]?state|worker|queue|concurr|parallel|race", re.I),
|
|
@@ -985,11 +975,6 @@ def validate_canonical_capture(
|
|
|
985
975
|
require(patch_relative == "source-change.patch", "canonical capture patch_path must be source-change.patch", errors)
|
|
986
976
|
if patch is not None:
|
|
987
977
|
require(file_sha256(patch) == capture.get("patch_sha256"), "canonical capture patch hash mismatch", errors)
|
|
988
|
-
require(
|
|
989
|
-
capture.get("source_tree_sha256") == tree_sha256(project_root, root),
|
|
990
|
-
"canonical change capture source tree is stale",
|
|
991
|
-
errors,
|
|
992
|
-
)
|
|
993
978
|
captured_paths = {
|
|
994
979
|
item.get("path")
|
|
995
980
|
for item in capture.get("changed_files", [])
|
|
@@ -1240,29 +1225,6 @@ def validate_matrix_risks(
|
|
|
1240
1225
|
errors.append(f"TEST_PASS has unresolved behavior risk: {behavior_id}/{risk_name}")
|
|
1241
1226
|
|
|
1242
1227
|
|
|
1243
|
-
def is_volatile_snapshot_path(root: Path, candidate: Path, evidence_root: Path) -> bool:
|
|
1244
|
-
try:
|
|
1245
|
-
relative = candidate.relative_to(root)
|
|
1246
|
-
except ValueError:
|
|
1247
|
-
return True
|
|
1248
|
-
if any(part in SNAPSHOT_VOLATILE_NAMES for part in relative.parts):
|
|
1249
|
-
return True
|
|
1250
|
-
if any(relative == Path(prefix) or Path(prefix) in relative.parents for prefix in SNAPSHOT_VOLATILE_ROOTS):
|
|
1251
|
-
return True
|
|
1252
|
-
resolved = candidate.resolve()
|
|
1253
|
-
excluded = evidence_root.resolve()
|
|
1254
|
-
return resolved == excluded or excluded in resolved.parents
|
|
1255
|
-
|
|
1256
|
-
|
|
1257
|
-
def tree_sha256(root: Path, evidence_root: Path) -> str:
|
|
1258
|
-
entries: list[dict[str, str]] = []
|
|
1259
|
-
for candidate in sorted(root.rglob("*")):
|
|
1260
|
-
if not candidate.is_file() or is_volatile_snapshot_path(root, candidate, evidence_root):
|
|
1261
|
-
continue
|
|
1262
|
-
entries.append({"path": candidate.relative_to(root).as_posix(), "sha256": file_sha256(candidate)})
|
|
1263
|
-
return canonical_sha256(entries)
|
|
1264
|
-
|
|
1265
|
-
|
|
1266
1228
|
def require_sha_or_null(value: Any, location: str, errors: list[str]) -> None:
|
|
1267
1229
|
require(value is None or is_sha256(value), f"{location} must be a sha256 or null", errors)
|
|
1268
1230
|
|
|
@@ -1283,7 +1245,6 @@ def validate_differential_proofs(
|
|
|
1283
1245
|
receipts = {item.get("execution_id"): item for item in executions if isinstance(item.get("execution_id"), str)}
|
|
1284
1246
|
behavior_ids = {item.get("id") for item in matrix.get("behaviors", []) if isinstance(item, dict)}
|
|
1285
1247
|
proof_by_behavior: dict[str, dict[str, Any]] = {}
|
|
1286
|
-
live_tree_hash = tree_sha256(project_root, root)
|
|
1287
1248
|
proofs = proof.get("proofs")
|
|
1288
1249
|
if not isinstance(proofs, list):
|
|
1289
1250
|
return
|
|
@@ -1300,9 +1261,8 @@ def validate_differential_proofs(
|
|
|
1300
1261
|
validate_differential_na(item.get("not_applicable"), f"differential proof {behavior_id} not_applicable", errors)
|
|
1301
1262
|
for key in (
|
|
1302
1263
|
"method", "differential_request_path", "differential_request_sha256", "baseline_commit",
|
|
1303
|
-
"
|
|
1304
|
-
"
|
|
1305
|
-
"mutation_apply_sha256", "mutation_restore_sha256",
|
|
1264
|
+
"baseline_execution_id", "mutation_execution_id", "current_execution_id",
|
|
1265
|
+
"isolation_tree_sha256", "mutation_apply_sha256", "mutation_restore_sha256",
|
|
1306
1266
|
):
|
|
1307
1267
|
require(item.get(key) is None, f"N/A differential proof must clear {key}: {behavior_id}", errors)
|
|
1308
1268
|
require(item.get("failure_reason_matched") is False, f"N/A differential proof cannot claim a failure match: {behavior_id}", errors)
|
|
@@ -1320,14 +1280,10 @@ def validate_differential_proofs(
|
|
|
1320
1280
|
method = item.get("method")
|
|
1321
1281
|
require(method == request.get("method") and method in {"baseline", "controlled-mutation"}, f"differential method mismatch: {behavior_id}", errors)
|
|
1322
1282
|
require(item.get("baseline_commit") == request.get("baseline_commit") == manifest.get("baseline_commit"), f"differential baseline commit mismatch: {behavior_id}", errors)
|
|
1323
|
-
require(item.get("current_tree_sha256") == request.get("current_tree_sha256") == live_tree_hash, f"differential current tree hash mismatch: {behavior_id}", errors)
|
|
1324
1283
|
for key in (
|
|
1325
|
-
"differential_request_sha256", "
|
|
1326
|
-
"project_tree_after_sha256", "isolation_tree_sha256", "mutation_apply_sha256", "mutation_restore_sha256",
|
|
1284
|
+
"differential_request_sha256", "isolation_tree_sha256", "mutation_apply_sha256", "mutation_restore_sha256",
|
|
1327
1285
|
):
|
|
1328
1286
|
require_sha_or_null(item.get(key), f"differential proof {behavior_id}.{key}", errors)
|
|
1329
|
-
require(item.get("project_tree_before_sha256") == live_tree_hash, f"differential project tree before mismatch: {behavior_id}", errors)
|
|
1330
|
-
require(item.get("project_tree_after_sha256") == live_tree_hash, f"differential project tree after mismatch: {behavior_id}", errors)
|
|
1331
1287
|
require(item.get("cleanup_succeeded") is True, f"differential cleanup failed: {behavior_id}", errors)
|
|
1332
1288
|
|
|
1333
1289
|
current = receipts.get(item.get("current_execution_id"))
|
|
@@ -1348,7 +1304,7 @@ def validate_differential_proofs(
|
|
|
1348
1304
|
require(isinstance(current_isolation, dict) and current_isolation.get("kind") == "current", f"differential current receipt lacks current isolation: {behavior_id}", errors)
|
|
1349
1305
|
require(isinstance(failing_isolation, dict) and failing_isolation.get("kind") == method, f"differential failing receipt isolation mismatch: {behavior_id}", errors)
|
|
1350
1306
|
if isinstance(current_isolation, dict):
|
|
1351
|
-
require(current_isolation.get("tree_sha256") == item.get("isolation_tree_sha256")
|
|
1307
|
+
require(current_isolation.get("tree_sha256") == item.get("isolation_tree_sha256"), f"differential current isolation hash mismatch: {behavior_id}", errors)
|
|
1352
1308
|
if method == "baseline" and isinstance(failing_isolation, dict):
|
|
1353
1309
|
require(failing_isolation.get("baseline_commit") == request.get("baseline_commit"), f"baseline receipt commit mismatch: {behavior_id}", errors)
|
|
1354
1310
|
if method == "controlled-mutation" and isinstance(failing_isolation, dict):
|