prizmkit 1.1.134 → 1.1.136

package/bundled/skills/prizmkit-test/scripts/validate_test_evidence.py

@@ -45,23 +45,13 @@ COMMON_RECORDS = {
     "source-change.patch", "test-report.md",
 }
 BEHAVIOR_RECORDS = {
-    "behavior-risk-matrix.json", "test-plan.json", "infrastructure-changes.json",
-    "differential-proof.json",
+    "behavior-risk-matrix.json", "test-plan.json",
+    "infrastructure-changes.json", "differential-proof.json",
 }
 LIGHTWEIGHT_RECORDS = {"lightweight-verification.json"}
 SHA256_RE = re.compile(r"^[a-f0-9]{64}$")
 UUID_RE = re.compile(r"^[a-f0-9-]{36}$")
 BLOCKED_EXTERNAL_CLASSES = {"production", "unknown"}
-SNAPSHOT_VOLATILE_ROOTS = (
-    ".prizmkit/state",
-    ".prizmkit/test/evidence",
-    ".prizmkit/" + "dev-" + "pipeline",
-    ".claude/worktrees",
-    ".agents/worktrees",
-    ".codebuddy/worktrees",
-    ".codex/worktrees",
-)
-SNAPSHOT_VOLATILE_NAMES = {".git", "__pycache__", ".pytest_cache", ".mypy_cache"}
 RISK_CONFLICT_PATTERNS = {
     "permission": re.compile(r"auth|tenant|role|permission|access|acl|rbac", re.I),
     "concurrency": re.compile(r"lock|shared[ _-]?state|worker|queue|concurr|parallel|race", re.I),
@@ -776,7 +766,9 @@ def validate_requests_and_receipts(
             if not isinstance(request, dict):
                 errors.append(f"request must be an object: {relative}")
                 continue
-            if request.get("method") == "not-applicable":
+            if set(request) == {"request_version", "layers", "tests"}:
+                validate_definition(request, "testPreparationRequest", records_schema, relative, errors)
+            elif request.get("method") == "not-applicable":
                 validate_definition(request, "differentialRequest", records_schema, relative, errors)
                 not_applicable = request.get("not_applicable")
                 if isinstance(not_applicable, dict):
@@ -935,13 +927,98 @@ def validate_execution_request_semantics(
     environment = value.get("environment")
     require(isinstance(environment, dict) and all(isinstance(key, str) and isinstance(item, str) for key, item in environment.items()), f"{location}: environment must contain complete string values", errors)
     test_ids = value.get("test_ids")
-    require(isinstance(test_ids, list) and all(isinstance(item, str) and item for item in test_ids), f"{location}: test_ids must be string IDs", errors)
+    test_paths = value.get("test_paths")
+    auto_bind = value.get("auto_bind") is True
+    require(
+        (isinstance(test_ids, list) and all(isinstance(item, str) and item for item in test_ids))
+        or (isinstance(test_paths, list) and all(isinstance(item, str) and item for item in test_paths))
+        or auto_bind,
+        f"{location}: execution must provide test_ids, test_paths, or auto_bind",
+        errors,
+    )
     if receipt:
         require(type(value.get("selected_execution")) is bool, f"{location}: selected_execution must be boolean", errors)
         require(type(value.get("reliable")) is bool, f"{location}: reliable must be boolean", errors)
         require(type(value.get("exit_code")) is int and not isinstance(value.get("exit_code"), bool), f"{location}: exit_code must be an integer", errors)
 
 
+def changed_test_statuses(root: Path) -> dict[str, str]:
+    capture_path = root / "change-capture.json"
+    if not capture_path.is_file():
+        return {}
+    try:
+        capture = load_json(capture_path)
+    except EvidenceError:
+        return {}
+    statuses: dict[str, str] = {}
+    for item in capture.get("changed_files", []) if isinstance(capture, dict) else []:
+        if not isinstance(item, dict) or not isinstance(item.get("path"), str) or item.get("role") == "source":
+            continue
+        status = str(item.get("status", "")).strip()
+        if status == "??" or status.startswith("A"):
+            statuses[item["path"]] = "added"
+        elif status.startswith("M") or status.endswith("M") or status.startswith(("R", "C")):
+            statuses[item["path"]] = "modified"
+    return statuses
+
+
+def validate_canonical_capture(
+    root: Path,
+    project_root: Path,
+    capture: dict[str, Any],
+    inventory: dict[str, Any],
+    scope: dict[str, Any],
+    errors: list[str],
+) -> None:
+    patch_relative = capture.get("patch_path")
+    patch = safe_path(root, patch_relative, errors, "canonical change patch") if isinstance(patch_relative, str) else None
+    require(patch_relative == "source-change.patch", "canonical capture patch_path must be source-change.patch", errors)
+    if patch is not None:
+        require(file_sha256(patch) == capture.get("patch_sha256"), "canonical capture patch hash mismatch", errors)
+    captured_paths = {
+        item.get("path")
+        for item in capture.get("changed_files", [])
+        if isinstance(item, dict) and isinstance(item.get("path"), str)
+    }
+    inventory_changed = set(inventory.get("changed_files", [])) if isinstance(inventory.get("changed_files"), list) else set()
+    scope_changed = set(scope.get("changed_files", [])) if isinstance(scope.get("changed_files"), list) else set()
+    require(captured_paths == inventory_changed, "canonical capture and inventory changed_files do not match", errors)
+    require(captured_paths == scope_changed, "canonical capture and scope changed_files do not match", errors)
+
+
+def validate_test_preparation(
+    root: Path,
+    project_root: Path,
+    preparation: dict[str, Any],
+    plan: dict[str, Any],
+    inventory_by_category: dict[str, dict[str, str]],
+    entry_map: dict[str, dict[str, Any]],
+    records_schema: dict[str, Any],
+    errors: list[str],
+) -> None:
+    validate_definition(preparation, "testPreparation", records_schema, "test-preparation", errors)
+    request_relative = preparation.get("request_path")
+    request_path = require_manifest_entry(request_relative, entry_map, root, errors, "test preparation request") if isinstance(request_relative, str) else None
+    if request_path is not None:
+        require(file_sha256(request_path) == preparation.get("request_sha256"), "test preparation request hash mismatch", errors)
+    prepared_tests = preparation.get("tests")
+    require(prepared_tests == plan.get("tests"), "test preparation tests do not match test plan", errors)
+    statuses = preparation.get("changed_test_statuses")
+    require(isinstance(statuses, dict), "test preparation changed_test_statuses must be an object", errors)
+    if not isinstance(statuses, dict):
+        return
+    for relative, status in statuses.items():
+        require(relative in inventory_by_category.get("tests", {}), f"test preparation status is not in test inventory: {relative}", errors)
+        require(status in {"added", "modified"}, f"test preparation has invalid changed test status: {relative}", errors)
+    for test in prepared_tests if isinstance(prepared_tests, list) else []:
+        if not isinstance(test, dict):
+            continue
+        relative = test.get("project_path")
+        if isinstance(relative, str):
+            expected = statuses.get(relative, "existing")
+            require(test.get("change_status") == expected, f"test preparation status mismatch: {relative}", errors)
+
+
 def validate_plan_and_scope_bindings(
     root: Path,
     project_root: Path,
@@ -1003,6 +1080,8 @@ def validate_plan_and_scope_bindings(
             for test_id in receipt.get("test_ids", []):
                 if isinstance(test_id, str):
                     selected_by_test.setdefault(test_id, []).append(receipt)
+    changed_tests = changed_test_statuses(root)
+    planned_test_paths: set[str] = set()
     known_behavior_ids = {item.get("id") for item in matrix.get("behaviors", []) if isinstance(item, dict)}
     for test in tests:
         if not isinstance(test, dict):
@@ -1018,9 +1097,13 @@ def validate_plan_and_scope_bindings(
         for behavior_id in behavior_ids if isinstance(behavior_ids, list) else []:
             require(behavior_id in known_behavior_ids, f"planned test maps unknown behavior: {test_id}/{behavior_id}", errors)
         project_relative = test.get("project_path")
+        planned_test_paths.add(project_relative)
         inventory_relative = test.get("inventory_path")
         snapshot_relative = test.get("snapshot_path")
+        inferred_status = changed_tests.get(project_relative, "existing")
         change_status = test.get("change_status", "modified" if isinstance(snapshot_relative, str) else "existing")
+        if changed_tests:
+            require(change_status == inferred_status, f"planned test change_status does not match canonical capture: {test_id}", errors)
         require(change_status in {"existing", "added", "modified"}, f"planned test has invalid change_status: {test_id}", errors)
         project_path = safe_path(project_root, project_relative, errors, f"planned test {test_id} project_path")
         require(project_path is not None and project_path.is_file(), f"planned test live path is missing: {test_id}", errors)
@@ -1043,6 +1126,19 @@ def validate_plan_and_scope_bindings(
         for receipt in matching_receipts:
             require(receipt.get("layer") in test_layers, f"planned test receipt uses an unplanned layer: {test_id}", errors)
 
+    for relative, status in changed_tests.items():
+        if relative in inventory_by_category.get("tests", {}) and relative not in planned_test_paths:
+            errors.append(f"changed native test is not bound by test plan: {relative}")
+    generated_dir = root / "generated-tests"
+    referenced_snapshots = {
+        test.get("snapshot_path") for test in tests if isinstance(test, dict) and isinstance(test.get("snapshot_path"), str)
+    }
+    if generated_dir.is_dir():
+        actual_snapshots = {
+            path.relative_to(root).as_posix() for path in generated_dir.rglob("*") if path.is_file()
+        }
+        for relative in sorted(actual_snapshots - referenced_snapshots):
+            errors.append(f"generated test snapshot is not bound by test plan: {relative}")
     for ring in scope.get("regression_ring", []):
         if not isinstance(ring, dict):
             continue
@@ -1129,29 +1225,6 @@ def validate_matrix_risks(
                     errors.append(f"TEST_PASS has unresolved behavior risk: {behavior_id}/{risk_name}")
 
 
-def is_volatile_snapshot_path(root: Path, candidate: Path, evidence_root: Path) -> bool:
-    try:
-        relative = candidate.relative_to(root)
-    except ValueError:
-        return True
-    if any(part in SNAPSHOT_VOLATILE_NAMES for part in relative.parts):
-        return True
-    if any(relative == Path(prefix) or Path(prefix) in relative.parents for prefix in SNAPSHOT_VOLATILE_ROOTS):
-        return True
-    resolved = candidate.resolve()
-    excluded = evidence_root.resolve()
-    return resolved == excluded or excluded in resolved.parents
-
-
-def tree_sha256(root: Path, evidence_root: Path) -> str:
-    entries: list[dict[str, str]] = []
-    for candidate in sorted(root.rglob("*")):
-        if not candidate.is_file() or is_volatile_snapshot_path(root, candidate, evidence_root):
-            continue
-        entries.append({"path": candidate.relative_to(root).as_posix(), "sha256": file_sha256(candidate)})
-    return canonical_sha256(entries)
-
-
 def require_sha_or_null(value: Any, location: str, errors: list[str]) -> None:
     require(value is None or is_sha256(value), f"{location} must be a sha256 or null", errors)
 
@@ -1172,7 +1245,6 @@ def validate_differential_proofs(
     receipts = {item.get("execution_id"): item for item in executions if isinstance(item.get("execution_id"), str)}
     behavior_ids = {item.get("id") for item in matrix.get("behaviors", []) if isinstance(item, dict)}
     proof_by_behavior: dict[str, dict[str, Any]] = {}
-    live_tree_hash = tree_sha256(project_root, root)
     proofs = proof.get("proofs")
     if not isinstance(proofs, list):
         return
@@ -1189,9 +1261,8 @@ def validate_differential_proofs(
             validate_differential_na(item.get("not_applicable"), f"differential proof {behavior_id} not_applicable", errors)
             for key in (
                 "method", "differential_request_path", "differential_request_sha256", "baseline_commit",
-                "current_tree_sha256", "baseline_execution_id", "mutation_execution_id", "current_execution_id",
-                "project_tree_before_sha256", "project_tree_after_sha256", "isolation_tree_sha256",
-                "mutation_apply_sha256", "mutation_restore_sha256",
+                "baseline_execution_id", "mutation_execution_id", "current_execution_id",
+                "isolation_tree_sha256", "mutation_apply_sha256", "mutation_restore_sha256",
             ):
                 require(item.get(key) is None, f"N/A differential proof must clear {key}: {behavior_id}", errors)
             require(item.get("failure_reason_matched") is False, f"N/A differential proof cannot claim a failure match: {behavior_id}", errors)
@@ -1209,14 +1280,10 @@ def validate_differential_proofs(
         method = item.get("method")
         require(method == request.get("method") and method in {"baseline", "controlled-mutation"}, f"differential method mismatch: {behavior_id}", errors)
         require(item.get("baseline_commit") == request.get("baseline_commit") == manifest.get("baseline_commit"), f"differential baseline commit mismatch: {behavior_id}", errors)
-        require(item.get("current_tree_sha256") == request.get("current_tree_sha256") == live_tree_hash, f"differential current tree hash mismatch: {behavior_id}", errors)
         for key in (
-            "differential_request_sha256", "current_tree_sha256", "project_tree_before_sha256",
-            "project_tree_after_sha256", "isolation_tree_sha256", "mutation_apply_sha256", "mutation_restore_sha256",
+            "differential_request_sha256", "isolation_tree_sha256", "mutation_apply_sha256", "mutation_restore_sha256",
         ):
             require_sha_or_null(item.get(key), f"differential proof {behavior_id}.{key}", errors)
-        require(item.get("project_tree_before_sha256") == live_tree_hash, f"differential project tree before mismatch: {behavior_id}", errors)
-        require(item.get("project_tree_after_sha256") == live_tree_hash, f"differential project tree after mismatch: {behavior_id}", errors)
         require(item.get("cleanup_succeeded") is True, f"differential cleanup failed: {behavior_id}", errors)
 
         current = receipts.get(item.get("current_execution_id"))
@@ -1237,7 +1304,7 @@ def validate_differential_proofs(
             require(isinstance(current_isolation, dict) and current_isolation.get("kind") == "current", f"differential current receipt lacks current isolation: {behavior_id}", errors)
             require(isinstance(failing_isolation, dict) and failing_isolation.get("kind") == method, f"differential failing receipt isolation mismatch: {behavior_id}", errors)
             if isinstance(current_isolation, dict):
-                require(current_isolation.get("tree_sha256") == item.get("isolation_tree_sha256") == live_tree_hash, f"differential current isolation hash mismatch: {behavior_id}", errors)
+                require(current_isolation.get("tree_sha256") == item.get("isolation_tree_sha256"), f"differential current isolation hash mismatch: {behavior_id}", errors)
             if method == "baseline" and isinstance(failing_isolation, dict):
                 require(failing_isolation.get("baseline_commit") == request.get("baseline_commit"), f"baseline receipt commit mismatch: {behavior_id}", errors)
             if method == "controlled-mutation" and isinstance(failing_isolation, dict):
@@ -1415,11 +1482,25 @@ def validate_evidence(
     entry_map = validate_manifest(root, manifest, manifest_schema, records_schema, errors, ignored_hash_paths)
     change_class = manifest.get("change_class")
     final_verdict = manifest.get("final_verdict")
+    canonical_lifecycle = (root / "lifecycle.json").is_file()
     required = COMMON_RECORDS
+    if canonical_lifecycle:
+        required = required | {"change-capture.json"}
     if final_verdict != "TEST_BLOCKED":
         required = required | (BEHAVIOR_RECORDS if change_class == "behavior" else LIGHTWEIGHT_RECORDS)
+        if canonical_lifecycle and change_class == "behavior":
+            required = required | {"test-preparation.json"}
     for relative in required:
         require(relative in entry_map, f"required evidence record not hashed: {relative}", errors)
+    if canonical_lifecycle:
+        capture_path = root / "change-capture.json"
+        if capture_path.is_file():
+            try:
+                capture = load_json(capture_path)
+                validate_definition(capture, "changeCapture", records_schema, "change-capture", errors)
+                validate_canonical_capture(root, project_root, capture, inventory, scope, errors)
+            except EvidenceError as exc:
+                errors.append(str(exc))
     validate_definition(classification, "changeClassification", records_schema, "change-classification", errors)
     validate_definition(scope, "scope", records_schema, "scope", errors)
     validate_definition(inventory, "targetInventory", records_schema, "target-inventory", errors)
@@ -1427,18 +1508,32 @@ def validate_evidence(
     require(classification.get("full_protocol_required") is (change_class == "behavior"), "change classification protocol decision is inconsistent", errors)
 
     plan: dict[str, Any] | None = None
+    preparation: dict[str, Any] | None = None
     if change_class == "behavior":
         try:
             plan = load_json(root / "test-plan.json")
+            if canonical_lifecycle:
+                preparation = load_json(root / "test-preparation.json")
         except EvidenceError as exc:
             errors.append(str(exc))
+            plan = preparation = None
         if isinstance(plan, dict):
             validate_definition(plan, "testPlan", records_schema, "test-plan", errors)
         else:
             errors.append("test-plan.json must be an object")
             plan = None
+        if canonical_lifecycle:
+            if isinstance(preparation, dict):
+                pass
+            else:
+                errors.append("test-preparation.json must be an object")
 
     inventory_by_category, _ = validate_inventory(root, project_root, manifest, inventory, scope, plan, entry_map, errors)
+    if canonical_lifecycle and change_class == "behavior" and isinstance(preparation, dict):
+        validate_test_preparation(
+            root, project_root, preparation, plan or {}, inventory_by_category,
+            entry_map, records_schema, errors,
+        )
     require(manifest.get("evidence_id_inputs", {}).get("scope_sha256") == canonical_sha256(scope), "scope identity hash mismatch", errors)
     validate_scope(
         scope, inventory, inventory_by_category, project_root, root,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "prizmkit",
-  "version": "1.1.134",
+  "version": "1.1.136",
   "description": "Create a new PrizmKit-powered project with clean initialization — no framework dev files, just what you need.",
   "type": "module",
   "bin": {