prizmkit 1.1.139 → 1.1.141

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (74) hide show
  1. package/bundled/VERSION.json +3 -3
  2. package/bundled/dev-pipeline/prizmkit_runtime/checkpoint_state.py +41 -121
  3. package/bundled/dev-pipeline/prizmkit_runtime/runner_models.py +0 -6
  4. package/bundled/dev-pipeline/prizmkit_runtime/runners.py +1 -2
  5. package/bundled/dev-pipeline/prizmkit_runtime/test_result.py +150 -0
  6. package/bundled/dev-pipeline/scripts/generate-bootstrap-prompt.py +21 -804
  7. package/bundled/dev-pipeline/scripts/generate-bugfix-prompt.py +4 -6
  8. package/bundled/dev-pipeline/scripts/generate-recovery-prompt.py +1 -1
  9. package/bundled/dev-pipeline/scripts/generate-refactor-prompt.py +4 -6
  10. package/bundled/dev-pipeline/scripts/prompt_framework.py +56 -55
  11. package/bundled/dev-pipeline/scripts/update-bug-status.py +1 -2
  12. package/bundled/dev-pipeline/scripts/update-checkpoint.py +14 -96
  13. package/bundled/dev-pipeline/scripts/update-feature-status.py +1 -2
  14. package/bundled/dev-pipeline/scripts/update-refactor-status.py +1 -2
  15. package/bundled/dev-pipeline/templates/bootstrap-prompt.md +3 -3
  16. package/bundled/dev-pipeline/templates/bootstrap-tier3.md +7 -18
  17. package/bundled/dev-pipeline/templates/bugfix-bootstrap-prompt.md +6 -6
  18. package/bundled/dev-pipeline/templates/refactor-bootstrap-prompt.md +5 -5
  19. package/bundled/dev-pipeline/templates/sections/bugfix-phase-commit-report.md +1 -1
  20. package/bundled/dev-pipeline/templates/sections/bugfix-phase-diagnose-plan.md +1 -1
  21. package/bundled/dev-pipeline/templates/sections/bugfix-phase-manual-verification.md +1 -1
  22. package/bundled/dev-pipeline/templates/sections/bugfix-phase-test.md +13 -10
  23. package/bundled/dev-pipeline/templates/sections/bugfix-reminders.md +1 -1
  24. package/bundled/dev-pipeline/templates/sections/checkpoint-system.md +14 -9
  25. package/bundled/dev-pipeline/templates/sections/headless-commit-authorization.md +1 -1
  26. package/bundled/dev-pipeline/templates/sections/phase-commit-full.md +1 -1
  27. package/bundled/dev-pipeline/templates/sections/phase-commit.md +1 -1
  28. package/bundled/dev-pipeline/templates/sections/phase-context-snapshot-agent-suffix.md +2 -2
  29. package/bundled/dev-pipeline/templates/sections/phase-context-snapshot-base.md +1 -1
  30. package/bundled/dev-pipeline/templates/sections/phase-context-snapshot-lite-suffix.md +1 -1
  31. package/bundled/dev-pipeline/templates/sections/phase-implement-lite.md +1 -1
  32. package/bundled/dev-pipeline/templates/sections/phase-prizmkit-test.md +14 -26
  33. package/bundled/dev-pipeline/templates/sections/phase-specify-plan-full.md +3 -3
  34. package/bundled/dev-pipeline/templates/sections/refactor-phase-test.md +13 -10
  35. package/bundled/dev-pipeline/templates/sections/refactor-reminders.md +1 -1
  36. package/bundled/dev-pipeline/tests/test_checkpoint_state.py +287 -384
  37. package/bundled/dev-pipeline/tests/test_generate_bootstrap_prompt.py +119 -684
  38. package/bundled/dev-pipeline/tests/test_generate_bugfix_prompt.py +17 -40
  39. package/bundled/dev-pipeline/tests/test_generate_refactor_prompt.py +20 -23
  40. package/bundled/dev-pipeline/tests/test_python_runner_parity.py +29 -32
  41. package/bundled/skills/_metadata.json +6 -6
  42. package/bundled/skills/prizmkit/SKILL.md +8 -13
  43. package/bundled/skills/prizmkit/references/workflow-state-protocol.md +61 -80
  44. package/bundled/skills/prizmkit-code-review/SKILL.md +3 -3
  45. package/bundled/skills/prizmkit-code-review/references/workflow-state-protocol.md +61 -80
  46. package/bundled/skills/prizmkit-committer/SKILL.md +3 -3
  47. package/bundled/skills/prizmkit-committer/references/workflow-state-protocol.md +61 -80
  48. package/bundled/skills/prizmkit-implement/references/workflow-state-protocol.md +61 -80
  49. package/bundled/skills/prizmkit-plan/references/workflow-state-protocol.md +61 -80
  50. package/bundled/skills/prizmkit-retrospective/SKILL.md +2 -2
  51. package/bundled/skills/prizmkit-retrospective/references/workflow-state-protocol.md +61 -80
  52. package/bundled/skills/prizmkit-test/SKILL.md +217 -187
  53. package/bundled/skills/prizmkit-test/references/external-contract-mock-guidance.md +119 -0
  54. package/bundled/skills/prizmkit-test/references/independent-test-review.md +269 -0
  55. package/bundled/skills/prizmkit-test/references/test-coverage-model.md +193 -0
  56. package/bundled/skills/prizmkit-test/references/test-report-template.md +164 -79
  57. package/bundled/skills/prizmkit-workflow/SKILL.md +15 -23
  58. package/bundled/skills/prizmkit-workflow/references/workflow-state-protocol.md +61 -80
  59. package/package.json +1 -1
  60. package/bundled/skills/prizmkit-test/assets/authoritative-records.schema.json +0 -476
  61. package/bundled/skills/prizmkit-test/assets/behavior-risk-matrix.schema.json +0 -116
  62. package/bundled/skills/prizmkit-test/assets/evidence-manifest.schema.json +0 -111
  63. package/bundled/skills/prizmkit-test/assets/evidence-package-template.json +0 -66
  64. package/bundled/skills/prizmkit-test/references/boundary-coverage-protocol.md +0 -104
  65. package/bundled/skills/prizmkit-test/references/contract-mock-protocol.md +0 -65
  66. package/bundled/skills/prizmkit-test/references/evidence-protocol.md +0 -208
  67. package/bundled/skills/prizmkit-test/references/evidence-request-protocol.md +0 -80
  68. package/bundled/skills/prizmkit-test/references/examples.md +0 -100
  69. package/bundled/skills/prizmkit-test/references/service-boundary-test-catalog.md +0 -228
  70. package/bundled/skills/prizmkit-test/references/test-generation-steps.md +0 -118
  71. package/bundled/skills/prizmkit-test/references/trusted-evidence-execution.md +0 -110
  72. package/bundled/skills/prizmkit-test/references/workflow-state-protocol.md +0 -181
  73. package/bundled/skills/prizmkit-test/scripts/build_test_evidence.py +0 -1851
  74. package/bundled/skills/prizmkit-test/scripts/validate_test_evidence.py +0 -1793
@@ -1,1793 +0,0 @@
1
- #!/usr/bin/env python3
2
- """Deterministically validate a PrizmKit structured test-evidence package.
3
-
4
- This validator deliberately checks protocol mechanics and content-addressed evidence;
5
- it does not treat a successful validation as a claim about a deployed environment or
6
- as authorization to commit, release, or repair product behavior.
7
- """
8
-
9
- from __future__ import annotations
10
-
11
- import argparse
12
- import hashlib
13
- import json
14
- import math
15
- import os
16
- import re
17
- import subprocess
18
- import sys
19
- import tempfile
20
- from pathlib import Path
21
- from typing import Any
22
-
23
- STAGES = [
24
- "CHANGE_CLASSIFY", "SCOPE_DISCOVER", "CONTRACT_MODEL", "TEST_PLAN",
25
- "INFRA_READY", "TEST_BUILD", "EXECUTE_PROVE", "EVIDENCE_PACKAGE",
26
- "EVIDENCE_VALIDATE",
27
- ]
28
- CORE_BEHAVIOR_STAGES = set(STAGES)
29
- VERDICTS = {"TEST_PASS", "TEST_FAIL", "TEST_BLOCKED"}
30
- RISK_KEYS = {
31
- "functional", "boundary", "permission", "concurrency",
32
- "idempotency", "time", "dependency", "consumer",
33
- }
34
- LAYER_ORDER = {
35
- "focused": 0,
36
- "module-component": 1,
37
- "contract-integration": 2,
38
- "affected-module-regression": 3,
39
- "regression-ring": 4,
40
- }
41
- FINAL_RECORDS = {"validation.json", "verdict.json", "test-report.md"}
42
- ATTESTATION_MUTABLE = {"validation.json"}
43
- COMMON_RECORDS = {
44
- "change-classification.json", "scope.json", "target-inventory.json",
45
- "environment.json", "executions.json", "verdict.json", "validation.json",
46
- "source-change.patch", "test-report.md",
47
- }
48
- BEHAVIOR_RECORDS = {
49
- "behavior-risk-matrix.json", "test-plan.json",
50
- "infrastructure-changes.json",
51
- }
52
- LIGHTWEIGHT_RECORDS = {"lightweight-verification.json"}
53
- SHA256_RE = re.compile(r"^[a-f0-9]{64}$")
54
- UUID_RE = re.compile(r"^[a-f0-9-]{36}$")
55
- BLOCKED_EXTERNAL_CLASSES = {"production", "unknown"}
56
- INVENTORY_RESERVED_ROOTS = (
57
- ".prizmkit/state",
58
- ".prizmkit/test/evidence",
59
- ".prizmkit/" + "dev-" + "pipeline",
60
- ".claude/worktrees",
61
- ".agents/worktrees",
62
- ".codebuddy/worktrees",
63
- ".codex/worktrees",
64
- )
65
- _IGNORED_DIR_NAMES = {".git", "__pycache__", ".pytest_cache", ".mypy_cache"}
66
- RISK_CONFLICT_PATTERNS = {
67
- "permission": re.compile(r"auth|tenant|role|permission|access|acl|rbac", re.I),
68
- "concurrency": re.compile(r"lock|shared[ _-]?state|worker|queue|concurr|parallel|race", re.I),
69
- "idempotency": re.compile(r"retry|idempot|dedup|duplicate|replay", re.I),
70
- "time": re.compile(r"timeout|ttl|expiry|expire|clock|date|time", re.I),
71
- "dependency": re.compile(r"client|contract|network|https?|api|service|dependenc|external", re.I),
72
- "consumer": re.compile(r"caller|consumer|shared[ _-]?contract|import", re.I),
73
- }
74
-
75
-
76
- class EvidenceError(Exception):
77
- """Raised for malformed or unreadable evidence input."""
78
-
79
-
80
- def canonical_sha256(value: Any) -> str:
81
- payload = json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
82
- return hashlib.sha256(payload.encode("utf-8")).hexdigest()
83
-
84
-
85
- def file_sha256(path: Path) -> str:
86
- digest = hashlib.sha256()
87
- with path.open("rb") as handle:
88
- for chunk in iter(lambda: handle.read(1024 * 1024), b""):
89
- digest.update(chunk)
90
- return digest.hexdigest()
91
-
92
-
93
- def _reject_duplicate_keys(pairs: list[tuple[str, Any]]) -> dict[str, Any]:
94
- result: dict[str, Any] = {}
95
- for key, value in pairs:
96
- if key in result:
97
- raise ValueError(f"duplicate object key: {key}")
98
- result[key] = value
99
- return result
100
-
101
-
102
- def _reject_non_json_number(value: str) -> Any:
103
- raise ValueError(f"non-JSON numeric constant: {value}")
104
-
105
-
106
- def load_json(path: Path) -> Any:
107
- try:
108
- return json.loads(
109
- path.read_text(encoding="utf-8"),
110
- object_pairs_hook=_reject_duplicate_keys,
111
- parse_constant=_reject_non_json_number,
112
- )
113
- except FileNotFoundError as exc:
114
- raise EvidenceError(f"missing required file: {path.name}") from exc
115
- except (json.JSONDecodeError, ValueError) as exc:
116
- raise EvidenceError(f"invalid JSON in {path.name}: {exc}") from exc
117
-
118
-
119
- def atomic_write_json(path: Path, value: Any) -> None:
120
- path.parent.mkdir(parents=True, exist_ok=True)
121
- descriptor, temporary_name = tempfile.mkstemp(prefix=f".{path.name}.", suffix=".tmp", dir=path.parent)
122
- try:
123
- with os.fdopen(descriptor, "w", encoding="utf-8") as handle:
124
- json.dump(value, handle, indent=2, ensure_ascii=False)
125
- handle.write("\n")
126
- os.replace(temporary_name, path)
127
- except BaseException:
128
- try:
129
- os.unlink(temporary_name)
130
- except FileNotFoundError:
131
- pass
132
- raise
133
-
134
-
135
- def require(condition: bool, message: str, errors: list[str]) -> None:
136
- if not condition:
137
- errors.append(message)
138
-
139
-
140
- def is_sha256(value: Any) -> bool:
141
- return isinstance(value, str) and bool(SHA256_RE.fullmatch(value))
142
-
143
-
144
- def is_relative_path(value: Any) -> bool:
145
- return isinstance(value, str) and bool(value) and not Path(value).is_absolute()
146
-
147
-
148
- def safe_path(root: Path, relative: Any, errors: list[str], label: str) -> Path | None:
149
- if not is_relative_path(relative):
150
- errors.append(f"{label} must be a non-empty relative path")
151
- return None
152
- candidate = (root / relative).resolve()
153
- try:
154
- candidate.relative_to(root.resolve())
155
- except ValueError:
156
- errors.append(f"{label} escapes its root: {relative}")
157
- return None
158
- return candidate
159
-
160
-
161
- def path_under(relative: str, directory: str) -> bool:
162
- try:
163
- Path(relative).relative_to(directory)
164
- return True
165
- except ValueError:
166
- return False
167
-
168
-
169
- def path_below_or_equal(path: str, root: str) -> bool:
170
- normalized_path = Path(path).as_posix()
171
- normalized_root = Path(root).as_posix().rstrip("/")
172
- return normalized_path == normalized_root or normalized_path.startswith(f"{normalized_root}/")
173
-
174
-
175
- def normalize_project_relative(value: str) -> str:
176
- normalized = Path(value).as_posix()
177
- if normalized == "." or normalized.startswith("../") or "/../" in normalized:
178
- raise EvidenceError(f"Git returned a non-project-relative path: {value}")
179
- return normalized
180
-
181
-
182
- def is_reserved_inventory_path(relative: str) -> bool:
183
- path = Path(relative)
184
- if any(part in _IGNORED_DIR_NAMES for part in path.parts):
185
- return True
186
- return any(path == Path(prefix) or Path(prefix) in path.parents for prefix in INVENTORY_RESERVED_ROOTS)
187
-
188
-
189
- def project_git(project_root: Path, *args: str) -> subprocess.CompletedProcess[bytes]:
190
- result = subprocess.run(
191
- ["git", "-C", str(project_root), *args],
192
- capture_output=True,
193
- check=False,
194
- )
195
- if result.returncode != 0:
196
- raise EvidenceError(f"git {' '.join(args)} failed: {result.stderr.decode(errors='replace')}")
197
- return result
198
-
199
-
200
- def git_status_entries(project_root: Path) -> list[dict[str, Any]]:
201
- raw = project_git(
202
- project_root, "status", "--porcelain=v1", "-z", "--untracked-files=all",
203
- ).stdout
204
- entries: list[dict[str, Any]] = []
205
- tokens = raw.split(b"\0")
206
- index = 0
207
- while index < len(tokens):
208
- token = tokens[index]
209
- index += 1
210
- if not token:
211
- continue
212
- status = token[:2].decode(errors="replace")
213
- relative = normalize_project_relative(token[3:].decode(errors="replace"))
214
- old_path = None
215
- if "R" in status or "C" in status:
216
- if index >= len(tokens):
217
- raise EvidenceError("git status returned an incomplete rename/copy record")
218
- old_path = tokens[index].decode(errors="replace")
219
- index += 1
220
- if not is_reserved_inventory_path(relative):
221
- entries.append({"path": relative, "status": status, "old_path": old_path})
222
- if old_path is not None:
223
- old_relative = normalize_project_relative(old_path)
224
- if not is_reserved_inventory_path(old_relative):
225
- entries.append({"path": old_relative, "status": status, "old_path": old_relative, "role": "source"})
226
- return sorted(entries, key=lambda item: (item["path"], item.get("old_path") or ""))
227
-
228
-
229
- def git_diff_entries(project_root: Path, baseline: str) -> list[dict[str, Any]]:
230
- raw = project_git(
231
- project_root, "diff", "--name-status", "-z", "--find-renames", "--find-copies",
232
- baseline, "--", ".",
233
- ":!.prizmkit/state/**", ":!.prizmkit/test/evidence/**", ":!.prizmkit/" + "dev-" + "pipeline/**",
234
- ":!.claude/worktrees/**", ":!.agents/worktrees/**", ":!.codebuddy/worktrees/**", ":!.codex/worktrees/**",
235
- ).stdout
236
- tokens = raw.split(b"\0")
237
- entries: list[dict[str, Any]] = []
238
- index = 0
239
- while index < len(tokens):
240
- status_token = tokens[index].decode(errors="replace")
241
- index += 1
242
- if not status_token:
243
- continue
244
- if index >= len(tokens):
245
- raise EvidenceError("git diff returned an incomplete changed-file record")
246
- first = normalize_project_relative(tokens[index].decode(errors="replace"))
247
- index += 1
248
- status = status_token[0]
249
- old_path = None
250
- if status in {"R", "C"}:
251
- if index >= len(tokens):
252
- raise EvidenceError("git diff returned an incomplete rename/copy record")
253
- old_path = first
254
- first = normalize_project_relative(tokens[index].decode(errors="replace"))
255
- index += 1
256
- if not is_reserved_inventory_path(first):
257
- entries.append({"path": first, "status": status, "old_path": old_path})
258
- if old_path and not is_reserved_inventory_path(old_path):
259
- entries.append({"path": old_path, "status": status, "old_path": old_path, "role": "source"})
260
- return entries
261
-
262
-
263
- def merge_changed_file_entries(
264
- baseline_entries: list[dict[str, Any]],
265
- working_entries: list[dict[str, Any]],
266
- ) -> list[dict[str, Any]]:
267
- merged: dict[tuple[str, str | None, str | None], dict[str, Any]] = {}
268
- for entry in baseline_entries + working_entries:
269
- key = (entry["path"], entry.get("old_path"), entry.get("role"))
270
- existing = merged.get(key)
271
- if existing is None:
272
- merged[key] = dict(entry)
273
- continue
274
- statuses = {existing.get("status", "").strip(), entry.get("status", "").strip()}
275
- if "D" in statuses:
276
- existing["status"] = "D"
277
- elif "R" in statuses:
278
- existing["status"] = "R"
279
- elif "C" in statuses:
280
- existing["status"] = "C"
281
- elif statuses:
282
- existing["status"] = "M"
283
- return sorted(merged.values(), key=lambda item: (item["path"], item.get("old_path") or "", item.get("role") or ""))
284
-
285
-
286
- def canonical_change_patch(project_root: Path, baseline: str) -> tuple[bytes, list[dict[str, Any]]]:
287
- status_entries = merge_changed_file_entries(
288
- git_diff_entries(project_root, baseline),
289
- git_status_entries(project_root),
290
- )
291
- tracked = project_git(
292
- project_root, "diff", "--binary", "--full-index", "--find-renames", "--find-copies",
293
- baseline, "--", ".",
294
- ":!.prizmkit/state/**", ":!.prizmkit/test/evidence/**", ":!.prizmkit/" + "dev-" + "pipeline/**",
295
- ":!.claude/worktrees/**", ":!.agents/worktrees/**", ":!.codebuddy/worktrees/**", ":!.codex/worktrees/**",
296
- ).stdout
297
- chunks = [tracked]
298
- untracked = [
299
- item["path"] for item in status_entries
300
- if item["status"].strip() == "??" and item.get("old_path") is None
301
- ]
302
- for relative in sorted(untracked):
303
- candidate = (project_root / relative).resolve()
304
- try:
305
- candidate.relative_to(project_root.resolve())
306
- except ValueError as exc:
307
- raise EvidenceError(f"untracked path escapes project root: {relative}") from exc
308
- if candidate.is_file():
309
- result = subprocess.run(
310
- ["git", "diff", "--no-index", "--binary", "/dev/null", "--", relative],
311
- cwd=project_root,
312
- capture_output=True,
313
- check=False,
314
- )
315
- if result.returncode not in (0, 1):
316
- raise EvidenceError(f"cannot capture untracked file: {relative}")
317
- chunks.append(result.stdout)
318
- return b"".join(chunk for chunk in chunks if chunk), status_entries
319
-
320
-
321
- def resolve_ref(schema: dict[str, Any], root_schema: dict[str, Any]) -> dict[str, Any]:
322
- reference = schema.get("$ref")
323
- if not reference:
324
- return schema
325
- if not isinstance(reference, str) or not reference.startswith("#/"):
326
- raise EvidenceError(f"unsupported schema reference: {reference}")
327
- current: Any = root_schema
328
- for token in reference[2:].split("/"):
329
- if not isinstance(current, dict):
330
- raise EvidenceError(f"invalid schema reference: {reference}")
331
- current = current[token.replace("~1", "/").replace("~0", "~")]
332
- if not isinstance(current, dict):
333
- raise EvidenceError(f"schema reference is not an object: {reference}")
334
- return current
335
-
336
-
337
- def json_type_matches(value: Any, expected: str) -> bool:
338
- if expected == "object":
339
- return isinstance(value, dict)
340
- if expected == "array":
341
- return isinstance(value, list)
342
- if expected == "string":
343
- return isinstance(value, str)
344
- if expected == "boolean":
345
- return isinstance(value, bool)
346
- if expected == "integer":
347
- return isinstance(value, int) and not isinstance(value, bool)
348
- if expected == "number":
349
- return isinstance(value, (int, float)) and not isinstance(value, bool) and math.isfinite(value)
350
- if expected == "null":
351
- return value is None
352
- return False
353
-
354
-
355
- def json_equals(left: Any, right: Any) -> bool:
356
- """JSON equality without Python's bool-is-an-int equivalence."""
357
- if isinstance(left, bool) or isinstance(right, bool):
358
- return type(left) is type(right) and left == right
359
- if isinstance(left, (int, float)) and isinstance(right, (int, float)):
360
- return type(left) is type(right) and left == right
361
- return left == right
362
-
363
-
364
- def validate_schema(
365
- value: Any,
366
- schema: dict[str, Any],
367
- root_schema: dict[str, Any],
368
- location: str,
369
- errors: list[str],
370
- ) -> None:
371
- """Validate the strict JSON-Schema subset used by skill-owned schemas."""
372
- schema = resolve_ref(schema, root_schema)
373
-
374
- one_of = schema.get("oneOf")
375
- if isinstance(one_of, list):
376
- matching = 0
377
- for branch in one_of:
378
- if not isinstance(branch, dict):
379
- continue
380
- branch_errors: list[str] = []
381
- validate_schema(value, branch, root_schema, location, branch_errors)
382
- if not branch_errors:
383
- matching += 1
384
- require(matching == 1, f"{location}: value must match exactly one schema branch", errors)
385
- prohibited = schema.get("not")
386
- if isinstance(prohibited, dict):
387
- prohibited_errors: list[str] = []
388
- validate_schema(value, prohibited, root_schema, location, prohibited_errors)
389
- require(bool(prohibited_errors), f"{location}: value matches a prohibited schema branch", errors)
390
-
391
- for child in schema.get("allOf", []):
392
- if isinstance(child, dict):
393
- validate_schema(value, child, root_schema, location, errors)
394
- if "const" in schema:
395
- require(json_equals(value, schema["const"]), f"{location}: value does not match const", errors)
396
- if "enum" in schema:
397
- allowed = schema["enum"]
398
- require(
399
- isinstance(allowed, list) and any(json_equals(value, item) for item in allowed),
400
- f"{location}: value is outside enum",
401
- errors,
402
- )
403
-
404
- expected = schema.get("type")
405
- if expected is not None:
406
- expected_types = expected if isinstance(expected, list) else [expected]
407
- valid_types = [item for item in expected_types if isinstance(item, str)]
408
- if not any(json_type_matches(value, item) for item in valid_types):
409
- name = " or ".join(valid_types) if valid_types else repr(expected)
410
- errors.append(f"{location}: expected {name}")
411
- return
412
-
413
- if isinstance(value, str):
414
- if "minLength" in schema:
415
- require(len(value) >= schema["minLength"], f"{location}: string is too short", errors)
416
- if "maxLength" in schema:
417
- require(len(value) <= schema["maxLength"], f"{location}: string is too long", errors)
418
- if "pattern" in schema:
419
- try:
420
- matched = re.search(schema["pattern"], value) is not None
421
- except re.error as exc:
422
- raise EvidenceError(f"invalid schema pattern at {location}: {exc}") from exc
423
- require(matched, f"{location}: string does not match pattern", errors)
424
-
425
- if isinstance(value, (int, float)) and not isinstance(value, bool):
426
- if "minimum" in schema:
427
- require(value >= schema["minimum"], f"{location}: number is below minimum", errors)
428
- if "maximum" in schema:
429
- require(value <= schema["maximum"], f"{location}: number is above maximum", errors)
430
- if "exclusiveMinimum" in schema:
431
- require(value > schema["exclusiveMinimum"], f"{location}: number is not above exclusive minimum", errors)
432
- if "exclusiveMaximum" in schema:
433
- require(value < schema["exclusiveMaximum"], f"{location}: number is not below exclusive maximum", errors)
434
-
435
- if isinstance(value, list):
436
- if "minItems" in schema:
437
- require(len(value) >= schema["minItems"], f"{location}: array has too few items", errors)
438
- if "maxItems" in schema:
439
- require(len(value) <= schema["maxItems"], f"{location}: array has too many items", errors)
440
- if schema.get("uniqueItems") is True:
441
- require(len({canonical_sha256(item) for item in value}) == len(value), f"{location}: array items are not unique", errors)
442
- item_schema = schema.get("items")
443
- if isinstance(item_schema, dict):
444
- for index, item in enumerate(value):
445
- validate_schema(item, item_schema, root_schema, f"{location}[{index}]", errors)
446
-
447
- if isinstance(value, dict):
448
- if "minProperties" in schema:
449
- require(len(value) >= schema["minProperties"], f"{location}: object has too few properties", errors)
450
- if "maxProperties" in schema:
451
- require(len(value) <= schema["maxProperties"], f"{location}: object has too many properties", errors)
452
- required = schema.get("required", [])
453
- for key in required:
454
- require(key in value, f"{location}: missing required property {key}", errors)
455
- properties = schema.get("properties", {})
456
- additional = schema.get("additionalProperties", True)
457
- for key, item in value.items():
458
- if key in properties:
459
- continue
460
- if additional is False:
461
- errors.append(f"{location}: unexpected property {key}")
462
- elif isinstance(additional, dict):
463
- validate_schema(item, additional, root_schema, f"{location}.{key}", errors)
464
- for key, child_schema in properties.items():
465
- if key in value and isinstance(child_schema, dict):
466
- validate_schema(value[key], child_schema, root_schema, f"{location}.{key}", errors)
467
-
468
-
469
- def assets_dir() -> Path:
470
- return Path(__file__).resolve().parent.parent / "assets"
471
-
472
-
473
- def load_schema(name: str) -> dict[str, Any]:
474
- schema = load_json(assets_dir() / name)
475
- if not isinstance(schema, dict):
476
- raise EvidenceError(f"schema is not an object: {name}")
477
- return schema
478
-
479
-
480
- def validate_definition(
481
- value: Any,
482
- definition: str,
483
- records_schema: dict[str, Any],
484
- location: str,
485
- errors: list[str],
486
- ) -> None:
487
- definitions = records_schema.get("$defs")
488
- if not isinstance(definitions, dict) or not isinstance(definitions.get(definition), dict):
489
- raise EvidenceError(f"missing authoritative schema definition: {definition}")
490
- validate_schema(value, definitions[definition], records_schema, location, errors)
491
-
492
-
493
- def require_manifest_entry(
494
- relative: str,
495
- entry_map: dict[str, dict[str, Any]],
496
- root: Path,
497
- errors: list[str],
498
- label: str,
499
- ) -> Path | None:
500
- entry = entry_map.get(relative)
501
- require(entry is not None, f"{label} is not manifest-hashed: {relative}", errors)
502
- path = safe_path(root, relative, errors, label)
503
- if path is None:
504
- return None
505
- if not path.is_file():
506
- errors.append(f"{label} is missing: {relative}")
507
- return None
508
- if entry is not None:
509
- require(file_sha256(path) == entry.get("sha256"), f"{label} hash mismatch: {relative}", errors)
510
- return path
511
-
512
-
513
- def structured_na(
514
- value: Any,
515
- records_schema: dict[str, Any],
516
- location: str,
517
- errors: list[str],
518
- risk: str | None = None,
519
- ) -> None:
520
- validate_definition(value, "naDecision", records_schema, location, errors)
521
- if not isinstance(value, dict):
522
- return
523
- signals = value.get("considered_signals")
524
- require(isinstance(signals, list) and bool(signals), f"{location}: N/A needs considered signals", errors)
525
- evidence = value.get("evidence")
526
- require(isinstance(evidence, list) and bool(evidence), f"{location}: N/A needs discovery evidence", errors)
527
- conflicts = value.get("conflicts")
528
- require(isinstance(conflicts, list), f"{location}: N/A conflicts must be an array", errors)
529
- if not isinstance(signals, list) or not isinstance(conflicts, list):
530
- return
531
-
532
- signal_text = "\n".join(
533
- item for item in signals if isinstance(item, str)
534
- )
535
- if isinstance(evidence, list):
536
- for item in evidence:
537
- if isinstance(item, dict):
538
- source = item.get("source")
539
- observations = item.get("observations")
540
- if isinstance(source, str):
541
- signal_text += f"\n{source}"
542
- if isinstance(observations, list):
543
- signal_text += "\n" + "\n".join(value for value in observations if isinstance(value, str))
544
- if risk and risk in RISK_CONFLICT_PATTERNS and RISK_CONFLICT_PATTERNS[risk].search(signal_text):
545
- pattern = RISK_CONFLICT_PATTERNS[risk]
546
- matching_signals = [
547
- item for item in conflicts
548
- if isinstance(item, dict)
549
- and isinstance(item.get("signal"), str)
550
- and pattern.search(item["signal"])
551
- and isinstance(item.get("explanation"), str)
552
- and len(item["explanation"].strip()) >= 8
553
- ]
554
- require(
555
- bool(matching_signals),
556
- f"{location}: {risk} N/A conflicts with discovered signals but has no explained conflict",
557
- errors,
558
- )
559
-
560
-
561
- def validate_external_targets(value: Any, location: str, errors: list[str]) -> None:
562
- if not isinstance(value, list):
563
- errors.append(f"{location}: external targets must be an array")
564
- return
565
- for index, target in enumerate(value):
566
- target_location = f"{location}[{index}]"
567
- if not isinstance(target, dict):
568
- errors.append(f"{target_location}: external target must be an object")
569
- continue
570
- external = target.get("external")
571
- require(type(external) is bool, f"{target_location}: external must be a boolean", errors)
572
- if external is not True:
573
- continue
574
- classification = target.get("classification")
575
- require(classification not in BLOCKED_EXTERNAL_CLASSES, f"{target_location}: external {classification} target is blocked", errors)
576
- require(classification in {"isolated", "test", "staging"}, f"{target_location}: external target classification is unsafe", errors)
577
- endpoint_evidence = target.get("endpoint_evidence")
578
- allow_evidence = target.get("allow_evidence")
579
- deny_evidence = target.get("deny_evidence")
580
- require(isinstance(endpoint_evidence, list) and bool(endpoint_evidence), f"{target_location}: external target lacks endpoint evidence", errors)
581
- require(isinstance(allow_evidence, list) and bool(allow_evidence), f"{target_location}: external target lacks allow evidence", errors)
582
- require(isinstance(deny_evidence, list) and not deny_evidence, f"{target_location}: external target has deny evidence", errors)
583
-
584
-
585
- def validate_manifest(
586
- root: Path,
587
- manifest: dict[str, Any],
588
- manifest_schema: dict[str, Any],
589
- records_schema: dict[str, Any],
590
- errors: list[str],
591
- ignored_hash_paths: set[str] | None = None,
592
- ) -> dict[str, dict[str, Any]]:
593
- validate_schema(manifest, manifest_schema, manifest_schema, "manifest", errors)
594
- require(manifest.get("sensitivity") == "project-controlled", "sensitivity must be project-controlled", errors)
595
- require(manifest.get("environment_claim") == "mocked-code-level-only", "environment claim must be mocked-code-level-only", errors)
596
- require(manifest.get("compatibility") == "legacy-test-report-interface-not-supported", "legacy compatibility boundary missing", errors)
597
-
598
- identity_inputs = manifest.get("evidence_id_inputs")
599
- if isinstance(identity_inputs, dict):
600
- require(manifest.get("evidence_id") == canonical_sha256(identity_inputs), "evidence_id does not match canonical identity inputs", errors)
601
- require(root.name == manifest.get("evidence_id"), "evidence directory name does not match evidence_id", errors)
602
- require(identity_inputs.get("baseline_commit") == manifest.get("baseline_commit"), "baseline commit identity mismatch", errors)
603
- require(identity_inputs.get("working_diff_sha256") == manifest.get("working_diff_sha256"), "working diff identity mismatch", errors)
604
-
605
- entries = manifest.get("files")
606
- entry_map: dict[str, dict[str, Any]] = {}
607
- if not isinstance(entries, list):
608
- return entry_map
609
- for index, entry in enumerate(entries):
610
- entry_location = f"manifest.files[{index}]"
611
- if not isinstance(entry, dict):
612
- continue
613
- relative = entry.get("path")
614
- if not is_relative_path(relative):
615
- continue
616
- require(relative not in entry_map, f"duplicate manifest file entry: {relative}", errors)
617
- entry_map[relative] = entry
618
- candidate = safe_path(root, relative, errors, entry_location)
619
- if candidate is None:
620
- continue
621
- if not candidate.is_file():
622
- errors.append(f"manifest file missing: {relative}")
623
- continue
624
- if relative not in (ignored_hash_paths or set()):
625
- require(file_sha256(candidate) == entry.get("sha256"), f"content hash mismatch: {relative}", errors)
626
-
627
- actual_paths = {
628
- candidate.relative_to(root).as_posix()
629
- for candidate in root.rglob("*")
630
- if candidate.is_file() and candidate.name != "manifest.json"
631
- }
632
- require(set(entry_map) == actual_paths, "manifest must hash every evidence file exactly once", errors)
633
-
634
- stages = manifest.get("stages")
635
- if not isinstance(stages, list):
636
- return entry_map
637
- names = [stage.get("name") for stage in stages if isinstance(stage, dict)]
638
- require(names == STAGES, "stages are missing or out of order", errors)
639
- for stage in stages:
640
- if not isinstance(stage, dict):
641
- continue
642
- name = stage.get("name")
643
- outputs = stage.get("outputs")
644
- status = stage.get("status")
645
- na = stage.get("not_applicable")
646
- if status == "not_applicable":
647
- structured_na(na, records_schema, f"stage {name} not_applicable", errors)
648
- require(outputs == [], f"not-applicable stage must not claim outputs: {name}", errors)
649
- elif status == "complete":
650
- require(na is None, f"complete stage must have null not_applicable: {name}", errors)
651
- for output in outputs if isinstance(outputs, list) else []:
652
- entry = entry_map.get(output)
653
- require(entry is not None, f"stage output is not a hashed evidence file: {name}/{output}", errors)
654
- if entry is not None:
655
- require(entry.get("produced_by") == name, f"stage output producer mismatch: {name}/{output}", errors)
656
- return entry_map
657
-
658
-
659
- def get_captured_deleted_paths(root: Path) -> set[str]:
660
- capture_path = root / "change-capture.json"
661
- if not capture_path.is_file():
662
- return set()
663
- try:
664
- capture = load_json(capture_path)
665
- except EvidenceError:
666
- return set()
667
- if not isinstance(capture, dict):
668
- return set()
669
- return {
670
- item["path"]
671
- for item in capture.get("changed_files", [])
672
- if isinstance(item, dict)
673
- and isinstance(item.get("path"), str)
674
- and (item.get("status", "").strip() == "D" or item.get("role") == "source")
675
- }
676
-
677
-
678
- def validate_inventory(
679
- root: Path,
680
- project_root: Path,
681
- manifest: dict[str, Any],
682
- inventory: dict[str, Any],
683
- scope: dict[str, Any],
684
- plan: dict[str, Any] | None,
685
- entry_map: dict[str, dict[str, Any]],
686
- errors: list[str],
687
- ) -> tuple[dict[str, dict[str, str]], dict[str, str]]:
688
- categories = inventory.get("categories")
689
- expected_categories = {"source", "tests", "contracts", "lockfiles"}
690
- inventory_by_category: dict[str, dict[str, str]] = {}
691
- aggregates: dict[str, str] = {}
692
- if not isinstance(categories, dict) or set(categories) != expected_categories:
693
- errors.append("target inventory categories are incomplete")
694
- return inventory_by_category, aggregates
695
-
696
- all_paths: dict[str, str] = {}
697
- for category in sorted(expected_categories):
698
- entries = categories.get(category)
699
- category_entries: dict[str, str] = {}
700
- if not isinstance(entries, list):
701
- errors.append(f"target inventory category is not an array: {category}")
702
- continue
703
- for entry in entries:
704
- if not isinstance(entry, dict):
705
- errors.append(f"invalid target inventory entry: {category}")
706
- continue
707
- relative = entry.get("path")
708
- expected_hash = entry.get("sha256")
709
- if not is_relative_path(relative) or not is_sha256(expected_hash):
710
- errors.append(f"invalid target inventory entry: {category}")
711
- continue
712
- require(relative not in category_entries, f"duplicate target inventory path: {relative}", errors)
713
- require(relative not in all_paths, f"target inventory path appears in multiple categories: {relative}", errors)
714
- candidate = safe_path(project_root, relative, errors, "target inventory path")
715
- if candidate is None or not candidate.is_file():
716
- errors.append(f"target project file missing: {relative}")
717
- continue
718
- actual = file_sha256(candidate)
719
- require(actual == expected_hash, f"target project file drift: {relative}", errors)
720
- category_entries[relative] = actual
721
- all_paths[relative] = category
722
- inventory_by_category[category] = category_entries
723
- aggregates[category] = canonical_sha256([
724
- {"path": path, "sha256": category_entries[path]}
725
- for path in sorted(category_entries)
726
- ])
727
-
728
- try:
729
- environment = load_json(root / "environment.json")
730
- except EvidenceError as exc:
731
- errors.append(str(exc))
732
- environment = None
733
- aggregates["environment"] = canonical_sha256(environment)
734
- plan_inputs = inventory.get("plan_inputs")
735
- require(isinstance(plan_inputs, dict) and bool(plan_inputs), "target inventory plan_inputs missing", errors)
736
- aggregates["plan"] = canonical_sha256(plan_inputs)
737
- require(manifest.get("target_hashes") == aggregates, "target hashes do not match live project inventory", errors)
738
- require(scope.get("target_hashes") == aggregates, "scope target hashes do not match live inventory", errors)
739
- if plan is not None:
740
- require(plan.get("target_hashes") == aggregates, "test plan target hashes do not match live inventory", errors)
741
- require(plan.get("plan_inputs") == plan_inputs, "test plan inputs do not match target inventory", errors)
742
-
743
- patch = require_manifest_entry("source-change.patch", entry_map, root, errors, "source-change patch")
744
- captured_deleted_paths = get_captured_deleted_paths(root)
745
- if patch is not None:
746
- require(manifest.get("working_diff_sha256") == file_sha256(patch), "working_diff_sha256 does not match source-change.patch", errors)
747
-
748
- request_relative = inventory.get("inventory_request_path")
749
- request_path = require_manifest_entry(request_relative, entry_map, root, errors, "inventory request") if isinstance(request_relative, str) else None
750
- if request_path is not None:
751
- require(file_sha256(request_path) == inventory.get("inventory_request_sha256"), "inventory request hash mismatch", errors)
752
- request = load_json(request_path)
753
- if isinstance(request, dict):
754
- require(request.get("changed_files") == inventory.get("changed_files"), "inventory request changed_files mismatch", errors)
755
- request_roots = request.get("module_roots")
756
- normalized_request_roots = []
757
- if isinstance(request_roots, list):
758
- for item in request_roots:
759
- if isinstance(item, str):
760
- normalized_request_roots.append(item)
761
- elif isinstance(item, dict) and isinstance(item.get("path"), str):
762
- normalized_request_roots.append(item["path"])
763
- require(
764
- normalized_request_roots == inventory.get("module_roots"),
765
- "inventory request module_roots mismatch",
766
- errors,
767
- )
768
- require(request.get("exclusions") == scope.get("exclusions"), "inventory request exclusions mismatch", errors)
769
- require(request.get("plan_inputs") == plan_inputs, "inventory request plan_inputs mismatch", errors)
770
- else:
771
- errors.append("inventory request must be an object")
772
-
773
- return inventory_by_category, aggregates
774
-
775
-
776
- def validate_scope(
777
- scope: dict[str, Any],
778
- inventory: dict[str, Any],
779
- inventory_by_category: dict[str, dict[str, str]],
780
- project_root: Path,
781
- evidence_root: Path,
782
- captured_deleted_paths: set[str],
783
- errors: list[str],
784
- ) -> None:
785
- changed_files = scope.get("changed_files")
786
- require(isinstance(changed_files, list) and bool(changed_files), "scope changed_files is empty", errors)
787
- if not isinstance(changed_files, list):
788
- return
789
- require(len(set(changed_files)) == len(changed_files), "scope changed_files contains duplicates", errors)
790
- require(sorted(changed_files) == sorted(inventory.get("changed_files", [])), "scope and inventory changed_files do not agree", errors)
791
-
792
- exclusions = scope.get("exclusions")
793
- exclusion_paths = {
794
- item.get("path") for item in exclusions if isinstance(item, dict) and isinstance(item.get("path"), str)
795
- } if isinstance(exclusions, list) else set()
796
- require(exclusion_paths == set(inventory.get("exclusions", [])), "scope and inventory exclusions do not agree", errors)
797
- inventory_paths = {path for entries in inventory_by_category.values() for path in entries}
798
- for relative in changed_files:
799
- candidate = safe_path(project_root, relative, errors, "changed file")
800
- if relative in exclusion_paths:
801
- require(relative not in inventory_paths, f"excluded changed file is also inventoried: {relative}", errors)
802
- else:
803
- candidate = safe_path(project_root, relative, errors, "changed file")
804
- deleted = relative in captured_deleted_paths
805
- require(
806
- deleted or (candidate is not None and candidate.is_file()),
807
- f"changed file is not live: {relative}",
808
- errors,
809
- )
810
- require(relative in inventory_paths or deleted, f"changed file is neither inventoried nor excluded: {relative}", errors)
811
-
812
- roots = scope.get("module_roots")
813
- require(isinstance(roots, list) and bool(roots), "scope module_roots is empty", errors)
814
- if not isinstance(roots, list):
815
- return
816
- scope_root_paths = [item.get("path") for item in roots if isinstance(item, dict)]
817
- require(len(set(scope_root_paths)) == len(scope_root_paths), "scope module_roots contains duplicates", errors)
818
- require(sorted(scope_root_paths) == sorted(inventory.get("module_roots", [])), "scope and inventory module_roots do not agree", errors)
819
- inventory_root_files = inventory.get("module_root_files")
820
- require(isinstance(inventory_root_files, dict), "inventory module_root_files is missing", errors)
821
- require(
822
- isinstance(inventory_root_files, dict) and set(inventory_root_files) == set(scope_root_paths),
823
- "inventory module_root_files does not match declared module roots",
824
- errors,
825
- )
826
- evidence_resolved = evidence_root.resolve()
827
- for root_info in roots:
828
- if not isinstance(root_info, dict):
829
- continue
830
- root_relative = root_info.get("path")
831
- root_path = safe_path(project_root, root_relative, errors, "module root")
832
- require(root_path is not None and root_path.exists(), f"module root is not live: {root_relative}", errors)
833
- files = root_info.get("files")
834
- require(isinstance(files, list) and bool(files), f"module root has no files: {root_relative}", errors)
835
- live_root_files: list[str] = []
836
- if root_path is not None and root_path.exists():
837
- candidates = [root_path] if root_path.is_file() else root_path.rglob("*")
838
- for candidate in candidates:
839
- if not candidate.is_file() or ".git" in candidate.parts or "__pycache__" in candidate.parts:
840
- continue
841
- resolved = candidate.resolve()
842
- if resolved == evidence_resolved or evidence_resolved in resolved.parents:
843
- continue
844
- live_root_files.append(resolved.relative_to(project_root).as_posix())
845
- recorded_root_files = inventory_root_files.get(root_relative) if isinstance(inventory_root_files, dict) else None
846
- require(
847
- isinstance(recorded_root_files, list) and sorted(recorded_root_files) == sorted(live_root_files),
848
- f"module root live enumeration differs from inventory: {root_relative}",
849
- errors,
850
- )
851
- require(
852
- isinstance(files, list) and sorted(files) == sorted(live_root_files),
853
- f"scope module root files are incomplete: {root_relative}",
854
- errors,
855
- )
856
- for relative in files if isinstance(files, list) else []:
857
- candidate = safe_path(project_root, relative, errors, "module root file")
858
- require(candidate is not None and candidate.is_file(), f"module root file is not live: {relative}", errors)
859
- require(path_below_or_equal(relative, root_relative), f"module root file lies outside its root: {relative}", errors)
860
- require(relative in inventory_paths or relative in exclusion_paths, f"module root file is neither inventoried nor excluded: {relative}", errors)
861
-
862
- require(
863
- isinstance(scope.get("primary_scope"), list) and bool(scope.get("primary_scope")),
864
- "Primary Scope must contain observable behavior descriptions",
865
- errors,
866
- )
867
-
868
-
869
- def validate_requests_and_receipts(
870
- root: Path,
871
- entry_map: dict[str, dict[str, Any]],
872
- records_schema: dict[str, Any],
873
- errors: list[str],
874
- ) -> tuple[dict[str, dict[str, Any]], list[dict[str, Any]]]:
875
- execution_requests: dict[str, dict[str, Any]] = {}
876
- requests_root = root / "requests"
877
- if requests_root.exists():
878
- for path in sorted(requests_root.rglob("*.json")):
879
- relative = path.relative_to(root).as_posix()
880
- require_manifest_entry(relative, entry_map, root, errors, "request")
881
- try:
882
- request = load_json(path)
883
- except EvidenceError as exc:
884
- errors.append(str(exc))
885
- continue
886
- if not isinstance(request, dict):
887
- errors.append(f"request must be an object: {relative}")
888
- continue
889
- if set(request) == {"request_version", "layers", "tests"}:
890
- validate_definition(request, "testPreparationRequest", records_schema, relative, errors)
891
- elif "command" in request and "layer" in request:
892
- validate_definition(request, "executionRequest", records_schema, relative, errors)
893
- validate_execution_request_semantics(request, relative, errors)
894
- validate_external_targets(request.get("external_targets"), f"{relative}.external_targets", errors)
895
- execution_requests[relative] = request
896
- elif set(request) == {
897
- "request_version", "change_class", "artifact_dir",
898
- } or (
899
- "request_version" in request
900
- and "change_class" in request
901
- and set(request) <= {
902
- "request_version", "change_class", "baseline_commit", "scope_path",
903
- "environment_path", "artifact_dir", "blockers", "repair_scope",
904
- }
905
- ):
906
- require(request.get("request_version", "1.0") == "1.0", f"unsupported finalize request version: {relative}", errors)
907
- elif set(request) == {
908
- "request_version", "categories", "changed_files", "module_roots",
909
- "exclusions", "discovery_evidence", "plan_inputs",
910
- }:
911
- require(request.get("request_version") == "1.0", f"unsupported inventory request version: {relative}", errors)
912
- else:
913
- errors.append(f"unrecognized request record: {relative}")
914
-
915
- runner = Path(__file__).resolve().with_name("build_test_evidence.py")
916
- if not runner.is_file():
917
- errors.append("runner source is missing: build_test_evidence.py")
918
-
919
- executions_path = root / "executions.json"
920
- try:
921
- executions = load_json(executions_path)
922
- except EvidenceError as exc:
923
- errors.append(str(exc))
924
- return execution_requests, []
925
- validate_definition(executions, "executionLog", records_schema, "executions", errors)
926
- if not isinstance(executions, list):
927
- return execution_requests, []
928
-
929
- receipt_by_id: dict[str, dict[str, Any]] = {}
930
- seen_receipt_paths: set[str] = set()
931
- attempts_by_request: dict[str, int] = {}
932
- for index, receipt in enumerate(executions):
933
- location = f"executions[{index}]"
934
- if not isinstance(receipt, dict):
935
- continue
936
- validate_definition(receipt, "executionReceipt", records_schema, location, errors)
937
- receipt_id = receipt.get("execution_id")
938
- require(isinstance(receipt_id, str) and bool(UUID_RE.fullmatch(receipt_id)), f"{location}: invalid execution_id", errors)
939
- require(receipt_id not in receipt_by_id, f"duplicate execution id: {receipt_id}", errors)
940
- if isinstance(receipt_id, str):
941
- receipt_by_id[receipt_id] = receipt
942
-
943
- require(receipt.get("receipt_format") == "prizmkit-runner-generated-v1", f"{location}: receipt is not runner-generated", errors)
944
- runner_relative = receipt.get("runner_path")
945
- runner_snapshot = require_manifest_entry(
946
- runner_relative, entry_map, root, errors, "runner snapshot",
947
- ) if isinstance(runner_relative, str) else None
948
- require(
949
- isinstance(runner_relative, str) and path_under(runner_relative, "runner"),
950
- f"{location}: runner snapshot must be under runner/",
951
- errors,
952
- )
953
- if runner_snapshot is not None:
954
- require(file_sha256(runner_snapshot) == receipt.get("runner_sha256"), f"{location}: runner snapshot hash mismatch", errors)
955
- request_relative = receipt.get("request_path")
956
- request = execution_requests.get(request_relative) if isinstance(request_relative, str) else None
957
- require(request is not None, f"{location}: receipt references an unknown execution request", errors)
958
- request_path = require_manifest_entry(request_relative, entry_map, root, errors, "receipt request") if isinstance(request_relative, str) else None
959
- if request_path is not None:
960
- require(file_sha256(request_path) == receipt.get("request_sha256"), f"{location}: request hash mismatch", errors)
961
- if isinstance(request, dict):
962
- for key in ("purpose", "command", "cwd", "layer", "test_ids", "external_targets"):
963
- require(receipt.get(key) == request.get(key), f"{location}: receipt {key} does not match request", errors)
964
- expected_environment = request.get("environment")
965
- actual_environment = receipt.get("environment")
966
- require(isinstance(actual_environment, dict) and isinstance(actual_environment.get("PATH"), str), f"{location}: receipt lacks complete PATH environment", errors)
967
- if isinstance(expected_environment, dict) and isinstance(actual_environment, dict):
968
- for key, value in expected_environment.items():
969
- require(actual_environment.get(key) == value, f"{location}: receipt environment differs for {key}", errors)
970
- expected_probes = request.get("tool_version_commands")
971
- actual_probes = receipt.get("tool_versions")
972
- require(isinstance(actual_probes, dict), f"{location}: receipt tool_versions must be an object", errors)
973
- if isinstance(expected_probes, dict) and isinstance(actual_probes, dict):
974
- require(set(actual_probes) == set(expected_probes), f"{location}: receipt tool probe names differ from request", errors)
975
- for probe_name, command in expected_probes.items():
976
- probe = actual_probes.get(probe_name)
977
- require(isinstance(probe, dict), f"{location}: tool probe is not an object: {probe_name}", errors)
978
- if isinstance(probe, dict):
979
- require(probe.get("command") == command, f"{location}: tool probe command mismatch: {probe_name}", errors)
980
- require("exit_code" in probe or "error" in probe, f"{location}: tool probe has no result: {probe_name}", errors)
981
-
982
- validate_execution_request_semantics(receipt, location, errors, receipt=True)
983
- isolation = receipt.get("isolation")
984
- require(
985
- isinstance(isolation, dict) and isolation.get("kind") == "project-direct",
986
- f"{location}: receipt did not execute in the current project workspace",
987
- errors,
988
- )
989
- validate_external_targets(receipt.get("external_targets"), f"{location}.external_targets", errors)
990
- for raw_key, hash_key in (("stdout_path", "stdout_sha256"), ("stderr_path", "stderr_sha256")):
991
- relative = receipt.get(raw_key)
992
- raw_path = safe_path(root, relative, errors, f"{location}.{raw_key}")
993
- require(isinstance(relative, str) and path_under(relative, "raw"), f"{location}: {raw_key} must be under raw/", errors)
994
- if raw_path is not None:
995
- require(raw_path.is_file(), f"{location}: raw output is missing: {relative}", errors)
996
- if raw_path.is_file():
997
- require(file_sha256(raw_path) == receipt.get(hash_key), f"{location}: raw output hash mismatch: {relative}", errors)
998
- require_manifest_entry(relative, entry_map, root, errors, "raw output")
999
-
1000
- recorded_receipt_path = f"receipts/{receipt_id}.json" if isinstance(receipt_id, str) else ""
1001
- receipt_path = require_manifest_entry(recorded_receipt_path, entry_map, root, errors, "receipt") if recorded_receipt_path else None
1002
- if receipt_path is not None:
1003
- try:
1004
- require(load_json(receipt_path) == receipt, f"{location}: receipt file does not match execution log", errors)
1005
- except EvidenceError as exc:
1006
- errors.append(str(exc))
1007
- seen_receipt_paths.add(recorded_receipt_path)
1008
-
1009
- request_hash = receipt.get("request_sha256")
1010
- if isinstance(request_hash, str):
1011
- attempts_by_request[request_hash] = attempts_by_request.get(request_hash, 0) + 1
1012
- require(receipt.get("attempt_index") == attempts_by_request[request_hash], f"{location}: attempt_index is not request-local sequence", errors)
1013
- replay_of = receipt.get("replay_of")
1014
- if replay_of is not None:
1015
- previous = receipt_by_id.get(replay_of)
1016
- require(previous is not None, f"{location}: replay source is missing", errors)
1017
- if previous is not None:
1018
- require(previous.get("request_sha256") == request_hash, f"{location}: replay request differs from source", errors)
1019
-
1020
- receipt_files = {
1021
- path.relative_to(root).as_posix()
1022
- for path in (root / "receipts").rglob("*.json")
1023
- } if (root / "receipts").is_dir() else set()
1024
- require(receipt_files == seen_receipt_paths, "receipt directory does not exactly match executions.json", errors)
1025
- return execution_requests, executions
1026
-
1027
-
1028
- def validate_execution_request_semantics(
1029
- value: dict[str, Any],
1030
- location: str,
1031
- errors: list[str],
1032
- receipt: bool = False,
1033
- ) -> None:
1034
- command = value.get("command")
1035
- require(isinstance(command, list) and bool(command) and all(isinstance(item, str) and item for item in command), f"{location}: command must be a non-empty string argv", errors)
1036
- environment = value.get("environment")
1037
- require(isinstance(environment, dict) and all(isinstance(key, str) and isinstance(item, str) for key, item in environment.items()), f"{location}: environment must contain complete string values", errors)
1038
- has_ids = "test_ids" in value
1039
- has_paths = "test_paths" in value
1040
- has_auto_bind = "auto_bind" in value
1041
- auto_bind = value.get("auto_bind") is True
1042
- binding_modes = int(has_ids) + int(has_paths) + int(auto_bind)
1043
- require(
1044
- binding_modes == 1 and not (has_auto_bind and not auto_bind),
1045
- f"{location}: binding modes test_ids, test_paths, and auto_bind=true must be mutually exclusive",
1046
- errors,
1047
- )
1048
- if has_ids:
1049
- test_ids = value.get("test_ids")
1050
- require(
1051
- isinstance(test_ids, list) and bool(test_ids)
1052
- and all(isinstance(item, str) and item for item in test_ids)
1053
- and len(set(test_ids)) == len(test_ids),
1054
- f"{location}: test_ids must be a non-empty unique string array",
1055
- errors,
1056
- )
1057
- if has_paths:
1058
- test_paths = value.get("test_paths")
1059
- require(
1060
- isinstance(test_paths, list) and bool(test_paths)
1061
- and all(isinstance(item, str) and item for item in test_paths)
1062
- and len(set(test_paths)) == len(test_paths),
1063
- f"{location}: test_paths must be a non-empty unique string array",
1064
- errors,
1065
- )
1066
- if receipt:
1067
- require(has_ids and not has_paths and not has_auto_bind, f"{location}: receipt binding must use canonical test_ids only", errors)
1068
- require(type(value.get("selected_execution")) is bool, f"{location}: selected_execution must be boolean", errors)
1069
- require(type(value.get("reliable")) is bool, f"{location}: reliable must be boolean", errors)
1070
- require(type(value.get("exit_code")) is int and not isinstance(value.get("exit_code"), bool), f"{location}: exit_code must be an integer", errors)
1071
-
1072
-
1073
- def changed_test_statuses(root: Path) -> dict[str, str]:
1074
- capture_path = root / "change-capture.json"
1075
- if not capture_path.is_file():
1076
- return {}
1077
- try:
1078
- capture = load_json(capture_path)
1079
- except EvidenceError:
1080
- return {}
1081
- statuses: dict[str, str] = {}
1082
- for item in capture.get("changed_files", []) if isinstance(capture, dict) else []:
1083
- if not isinstance(item, dict) or not isinstance(item.get("path"), str) or item.get("role") == "source":
1084
- continue
1085
- status = str(item.get("status", "")).strip()
1086
- if status == "??" or status.startswith("A"):
1087
- statuses[item["path"]] = "added"
1088
- elif status.startswith("M") or status.endswith("M") or status.startswith(("R", "C")):
1089
- statuses[item["path"]] = "modified"
1090
- return statuses
1091
-
1092
-
1093
- def validate_canonical_capture(
1094
- root: Path,
1095
- project_root: Path,
1096
- capture: dict[str, Any],
1097
- inventory: dict[str, Any],
1098
- scope: dict[str, Any],
1099
- errors: list[str],
1100
- ) -> None:
1101
- patch_relative = capture.get("patch_path")
1102
- patch = safe_path(root, patch_relative, errors, "canonical change patch") if isinstance(patch_relative, str) else None
1103
- require(patch_relative == "source-change.patch", "canonical capture patch_path must be source-change.patch", errors)
1104
- if patch is not None:
1105
- require(file_sha256(patch) == capture.get("patch_sha256"), "canonical capture patch hash mismatch", errors)
1106
- captured_paths = {
1107
- item.get("path")
1108
- for item in capture.get("changed_files", [])
1109
- if isinstance(item, dict) and isinstance(item.get("path"), str)
1110
- }
1111
- inventory_changed = set(inventory.get("changed_files", [])) if isinstance(inventory.get("changed_files"), list) else set()
1112
- scope_changed = set(scope.get("changed_files", [])) if isinstance(scope.get("changed_files"), list) else set()
1113
- require(captured_paths == inventory_changed, "canonical capture and inventory changed_files do not match", errors)
1114
- require(captured_paths == scope_changed, "canonical capture and scope changed_files do not match", errors)
1115
- baseline = capture.get("baseline_commit")
1116
- if not isinstance(baseline, str) or not baseline:
1117
- errors.append("canonical capture baseline_commit is missing")
1118
- return
1119
- try:
1120
- live_patch, live_entries = canonical_change_patch(project_root, baseline)
1121
- except EvidenceError as exc:
1122
- errors.append(f"cannot verify live canonical change: {exc}")
1123
- return
1124
- require(
1125
- hashlib.sha256(live_patch).hexdigest() == capture.get("patch_sha256"),
1126
- "canonical change capture is stale: live patch hash differs",
1127
- errors,
1128
- )
1129
- require(
1130
- live_entries == capture.get("changed_files"),
1131
- "canonical change capture is stale: live changed-file entries differ",
1132
- errors,
1133
- )
1134
-
1135
-
1136
- def validate_test_preparation(
1137
- root: Path,
1138
- project_root: Path,
1139
- preparation: dict[str, Any],
1140
- plan: dict[str, Any],
1141
- inventory_by_category: dict[str, dict[str, str]],
1142
- entry_map: dict[str, dict[str, Any]],
1143
- records_schema: dict[str, Any],
1144
- errors: list[str],
1145
- ) -> None:
1146
- validate_definition(preparation, "testPreparation", records_schema, "test-preparation", errors)
1147
- request_relative = preparation.get("request_path")
1148
- request_path = require_manifest_entry(request_relative, entry_map, root, errors, "test preparation request") if isinstance(request_relative, str) else None
1149
- if request_path is not None:
1150
- require(file_sha256(request_path) == preparation.get("request_sha256"), "test preparation request hash mismatch", errors)
1151
- prepared_tests = preparation.get("tests")
1152
- require(prepared_tests == plan.get("tests"), "test preparation tests do not match test plan", errors)
1153
- statuses = preparation.get("changed_test_statuses")
1154
- require(isinstance(statuses, dict), "test preparation changed_test_statuses must be an object", errors)
1155
- if not isinstance(statuses, dict):
1156
- return
1157
- for relative, status in statuses.items():
1158
- require(relative in inventory_by_category.get("tests", {}), f"test preparation status is not in test inventory: {relative}", errors)
1159
- require(status in {"added", "modified"}, f"test preparation has invalid changed test status: {relative}", errors)
1160
- for test in prepared_tests if isinstance(prepared_tests, list) else []:
1161
- if not isinstance(test, dict):
1162
- continue
1163
- relative = test.get("project_path")
1164
- if isinstance(relative, str):
1165
- expected = statuses.get(relative, "existing")
1166
- require(test.get("change_status") == expected, f"test preparation status mismatch: {relative}", errors)
1167
-
1168
-
1169
- def validate_plan_and_scope_bindings(
1170
- root: Path,
1171
- project_root: Path,
1172
- manifest: dict[str, Any],
1173
- scope: dict[str, Any],
1174
- matrix: dict[str, Any],
1175
- plan: dict[str, Any],
1176
- inventory_by_category: dict[str, dict[str, str]],
1177
- entry_map: dict[str, dict[str, Any]],
1178
- executions: list[dict[str, Any]],
1179
- records_schema: dict[str, Any],
1180
- errors: list[str],
1181
- ) -> dict[str, dict[str, Any]]:
1182
- if not isinstance(scope, dict):
1183
- errors.append("scope must be an object")
1184
- if not isinstance(matrix, dict):
1185
- errors.append("behavior-risk-matrix must be an object")
1186
- scope_module = scope.get("affected_module") if isinstance(scope, dict) else None
1187
- matrix_module = matrix.get("affected_module") if isinstance(matrix, dict) else None
1188
- matrix_module_name = matrix_module.get("name") if isinstance(matrix_module, dict) else None
1189
- matrix_boundary_source = matrix_module.get("boundary_source") if isinstance(matrix_module, dict) else None
1190
- require(
1191
- isinstance(scope_module, str) and scope_module == matrix_module_name,
1192
- "matrix affected module does not match scope",
1193
- errors,
1194
- )
1195
- require(
1196
- isinstance(scope.get("boundary_source") if isinstance(scope, dict) else None, str)
1197
- and scope.get("boundary_source") == matrix_boundary_source,
1198
- "matrix boundary source does not match scope",
1199
- errors,
1200
- )
1201
- require(
1202
- scope.get("primary_scope") == matrix.get("primary_scope") if isinstance(scope, dict) and isinstance(matrix, dict) else False,
1203
- "matrix primary scope does not match scope",
1204
- errors,
1205
- )
1206
- if not isinstance(scope, dict) or not isinstance(matrix, dict) or not isinstance(matrix_module, dict):
1207
- return {}
1208
- scope_ring = sorted((item.get("name"), item.get("kind")) for item in scope.get("regression_ring", []) if isinstance(item, dict))
1209
- matrix_ring = sorted((item.get("name"), item.get("kind")) for item in matrix.get("regression_ring", []) if isinstance(item, dict))
1210
- require(scope_ring == matrix_ring, "scope and matrix regression rings do not agree", errors)
1211
-
1212
- unresolved = scope.get("unresolved_edges")
1213
- if manifest.get("final_verdict") == "TEST_PASS" and isinstance(unresolved, list):
1214
- for edge in unresolved:
1215
- if isinstance(edge, dict) and edge.get("verdict_capable") is True and edge.get("resolved") is not True:
1216
- errors.append("TEST_PASS has a verdict-capable unresolved edge")
1217
-
1218
- layers = plan.get("layers")
1219
- layer_map = {item.get("name"): item for item in layers if isinstance(item, dict)} if isinstance(layers, list) else {}
1220
- require(set(layer_map) == set(LAYER_ORDER), "test plan must assess every risk-adaptive layer exactly once", errors)
1221
- for name, layer in layer_map.items():
1222
- required = layer.get("required")
1223
- na = layer.get("not_applicable")
1224
- require(type(required) is bool, f"layer required flag invalid: {name}", errors)
1225
- if required is True:
1226
- require(na is None, f"required layer cannot be N/A: {name}", errors)
1227
- elif required is False:
1228
- structured_na(na, records_schema, f"layer {name} not_applicable", errors)
1229
- if matrix.get("regression_ring"):
1230
- require(layer_map.get("regression-ring", {}).get("required") is True, "nonempty regression ring must be required", errors)
1231
- if manifest.get("final_verdict") == "TEST_PASS":
1232
- for layer_name, layer in layer_map.items():
1233
- if layer.get("required") is True:
1234
- matches = [
1235
- item for item in executions
1236
- if item.get("selected_execution") is True
1237
- and item.get("layer") == layer_name
1238
- and item.get("reliable") is True
1239
- and item.get("exit_code") == 0
1240
- ]
1241
- require(bool(matches), f"required layer lacks a successful reliable receipt: {layer_name}", errors)
1242
-
1243
- tests = plan.get("tests")
1244
- test_map: dict[str, dict[str, Any]] = {}
1245
- if not isinstance(tests, list):
1246
- return test_map
1247
- selected_by_test: dict[str, list[dict[str, Any]]] = {}
1248
- for receipt in executions:
1249
- if receipt.get("selected_execution") is True:
1250
- for test_id in receipt.get("test_ids", []):
1251
- if isinstance(test_id, str):
1252
- selected_by_test.setdefault(test_id, []).append(receipt)
1253
- changed_tests = changed_test_statuses(root)
1254
- planned_test_paths: set[str] = set()
1255
- known_behavior_ids = {item.get("id") for item in matrix.get("behaviors", []) if isinstance(item, dict)}
1256
- for test in tests:
1257
- if not isinstance(test, dict):
1258
- continue
1259
- test_id = test.get("id")
1260
- require(isinstance(test_id, str) and bool(test_id), "planned test has no ID", errors)
1261
- require(test_id not in test_map, f"duplicate planned test ID: {test_id}", errors)
1262
- if not isinstance(test_id, str):
1263
- continue
1264
- test_map[test_id] = test
1265
- behavior_ids = test.get("behavior_ids")
1266
- require(isinstance(behavior_ids, list) and bool(behavior_ids), f"planned test has no behavior mapping: {test_id}", errors)
1267
- for behavior_id in behavior_ids if isinstance(behavior_ids, list) else []:
1268
- require(behavior_id in known_behavior_ids, f"planned test maps unknown behavior: {test_id}/{behavior_id}", errors)
1269
- project_relative = test.get("project_path")
1270
- planned_test_paths.add(project_relative)
1271
- inventory_relative = test.get("inventory_path")
1272
- snapshot_relative = test.get("snapshot_path")
1273
- inferred_status = changed_tests.get(project_relative, "existing")
1274
- change_status = test.get("change_status", "modified" if isinstance(snapshot_relative, str) else "existing")
1275
- if changed_tests:
1276
- require(change_status == inferred_status, f"planned test change_status does not match canonical capture: {test_id}", errors)
1277
- require(change_status in {"existing", "added", "modified"}, f"planned test has invalid change_status: {test_id}", errors)
1278
- project_path = safe_path(project_root, project_relative, errors, f"planned test {test_id} project_path")
1279
- require(project_path is not None and project_path.is_file(), f"planned test live path is missing: {test_id}", errors)
1280
- require(inventory_relative == project_relative, f"planned test inventory path differs from live project path: {test_id}", errors)
1281
- require(inventory_relative in inventory_by_category.get("tests", {}), f"planned test is not in the test inventory: {test_id}", errors)
1282
- if change_status == "existing":
1283
- require(snapshot_relative is None, f"existing test must not have a generated snapshot: {test_id}", errors)
1284
- snapshot_path = None
1285
- else:
1286
- snapshot_path = require_manifest_entry(snapshot_relative, entry_map, root, errors, f"planned test snapshot {test_id}") if isinstance(snapshot_relative, str) else None
1287
- require(isinstance(snapshot_relative, str) and path_under(snapshot_relative, "generated-tests"), f"planned test snapshot must be under generated-tests/: {test_id}", errors)
1288
- if project_path is not None and project_path.is_file() and snapshot_path is not None and snapshot_path.is_file():
1289
- require(file_sha256(project_path) == file_sha256(snapshot_path), f"generated test snapshot differs from live test: {test_id}", errors)
1290
- test_layers = test.get("layers")
1291
- require(isinstance(test_layers, list) and bool(test_layers), f"planned test has no layers: {test_id}", errors)
1292
- for layer in test_layers if isinstance(test_layers, list) else []:
1293
- require(layer in LAYER_ORDER, f"planned test uses unknown layer: {test_id}/{layer}", errors)
1294
- matching_receipts = selected_by_test.get(test_id, [])
1295
- if manifest.get("final_verdict") == "TEST_PASS":
1296
- passing_receipts = [
1297
- receipt for receipt in matching_receipts
1298
- if receipt.get("reliable") is True and receipt.get("exit_code") == 0
1299
- ]
1300
- require(bool(passing_receipts), f"planned test has no selected reliable zero-exit receipt: {test_id}", errors)
1301
- else:
1302
- require(bool(matching_receipts), f"planned test has no selected receipt: {test_id}", errors)
1303
- for receipt in matching_receipts:
1304
- require(receipt.get("layer") in test_layers, f"planned test receipt uses an unplanned layer: {test_id}", errors)
1305
-
1306
- for relative, status in changed_tests.items():
1307
- if relative in inventory_by_category.get("tests", {}) and relative not in planned_test_paths:
1308
- errors.append(f"changed native test is not bound by test plan: {relative}")
1309
- generated_dir = root / "generated-tests"
1310
- referenced_snapshots = {
1311
- test.get("snapshot_path") for test in tests if isinstance(test, dict) and isinstance(test.get("snapshot_path"), str)
1312
- }
1313
- if generated_dir.is_dir():
1314
- actual_snapshots = {
1315
- path.relative_to(root).as_posix() for path in generated_dir.rglob("*") if path.is_file()
1316
- }
1317
- for relative in sorted(actual_snapshots - referenced_snapshots):
1318
- errors.append(f"generated test snapshot is not bound by test plan: {relative}")
1319
- for ring in scope.get("regression_ring", []):
1320
- if not isinstance(ring, dict):
1321
- continue
1322
- ring_name = ring.get("name")
1323
- ring_files = ring.get("files")
1324
- require(isinstance(ring_files, list) and bool(ring_files), f"regression ring has no files: {ring_name}", errors)
1325
- for relative in ring_files if isinstance(ring_files, list) else []:
1326
- candidate = safe_path(project_root, relative, errors, "regression ring file")
1327
- require(candidate is not None and candidate.is_file(), f"regression ring file is not live: {relative}", errors)
1328
- require(relative in inventory_by_category.get("source", {}) or relative in inventory_by_category.get("tests", {}) or relative in inventory_by_category.get("contracts", {}) or relative in inventory_by_category.get("lockfiles", {}), f"regression ring file is not inventoried: {relative}", errors)
1329
- planned_ids = ring.get("planned_test_ids")
1330
- require(isinstance(planned_ids, list) and bool(planned_ids), f"regression ring has no planned tests: {ring_name}", errors)
1331
- for test_id in planned_ids if isinstance(planned_ids, list) else []:
1332
- test = test_map.get(test_id)
1333
- require(test is not None, f"regression ring maps unknown planned test: {test_id}", errors)
1334
- if test is not None:
1335
- require("regression-ring" in test.get("layers", []), f"regression ring test lacks regression-ring layer: {test_id}", errors)
1336
- return test_map
1337
-
1338
-
1339
- def validate_matrix_risks(
1340
- manifest: dict[str, Any],
1341
- matrix: dict[str, Any],
1342
- test_map: dict[str, dict[str, Any]],
1343
- executions: list[dict[str, Any]],
1344
- records_schema: dict[str, Any],
1345
- errors: list[str],
1346
- ) -> None:
1347
- verdict = manifest.get("final_verdict")
1348
- execution_by_id = {item.get("execution_id"): item for item in executions if isinstance(item.get("execution_id"), str)}
1349
- selected_ranks = [
1350
- LAYER_ORDER[item["layer"]]
1351
- for item in executions
1352
- if item.get("selected_execution") is True and item.get("layer") in LAYER_ORDER
1353
- ]
1354
- require(selected_ranks == sorted(selected_ranks), "selected executions are out of layer order", errors)
1355
-
1356
- reliably_failed = any(
1357
- item.get("reliable") is True
1358
- and item.get("exit_code") != 0
1359
- and item.get("selected_execution") is True
1360
- for item in executions
1361
- )
1362
- if verdict == "TEST_FAIL":
1363
- require(reliably_failed, "TEST_FAIL lacks a reliable reproduced failing execution", errors)
1364
- elif verdict == "TEST_PASS":
1365
- require(
1366
- not reliably_failed,
1367
- "TEST_PASS contains a reliable selected failing execution",
1368
- errors,
1369
- )
1370
-
1371
- for behavior in matrix.get("behaviors", []):
1372
- if not isinstance(behavior, dict):
1373
- continue
1374
- behavior_id = behavior.get("id")
1375
- risks = behavior.get("risks")
1376
- if not isinstance(risks, dict):
1377
- continue
1378
- require(risks.get("functional", {}).get("status") != "not_applicable", f"behavior functional risk cannot be N/A: {behavior_id}", errors)
1379
- for risk_name in RISK_KEYS:
1380
- cell = risks.get(risk_name)
1381
- if not isinstance(cell, dict):
1382
- continue
1383
- status = cell.get("status")
1384
- na = cell.get("not_applicable")
1385
- if status == "applicable":
1386
- require(na is None, f"applicable behavior risk cannot be N/A: {behavior_id}/{risk_name}", errors)
1387
- mapped_tests = cell.get("test_ids")
1388
- mapped_executions = cell.get("execution_ids")
1389
- require(isinstance(mapped_tests, list) and bool(mapped_tests), f"behavior risk has no test mapping: {behavior_id}/{risk_name}", errors)
1390
- require(isinstance(mapped_executions, list) and bool(mapped_executions), f"behavior risk has no receipt mapping: {behavior_id}/{risk_name}", errors)
1391
- for test_id in mapped_tests if isinstance(mapped_tests, list) else []:
1392
- require(test_id in test_map, f"behavior maps unknown planned test: {behavior_id}/{test_id}", errors)
1393
- if test_id in test_map:
1394
- require(behavior_id in test_map[test_id].get("behavior_ids", []), f"behavior test mapping is not bidirectional: {behavior_id}/{test_id}", errors)
1395
- for execution_id in mapped_executions if isinstance(mapped_executions, list) else []:
1396
- receipt = execution_by_id.get(execution_id)
1397
- require(receipt is not None, f"behavior maps unknown execution: {behavior_id}/{execution_id}", errors)
1398
- if receipt is not None:
1399
- require(any(test_id in receipt.get("test_ids", []) for test_id in mapped_tests or []), f"behavior receipt did not run a mapped test: {behavior_id}/{risk_name}", errors)
1400
- if verdict == "TEST_PASS":
1401
- require(receipt.get("selected_execution") is True and receipt.get("reliable") is True and receipt.get("exit_code") == 0, f"TEST_PASS behavior maps a non-passing receipt: {behavior_id}/{risk_name}", errors)
1402
- elif status == "not_applicable":
1403
- require(cell.get("test_ids") == [] and cell.get("execution_ids") == [], f"N/A risk must not map tests or receipts: {behavior_id}/{risk_name}", errors)
1404
- structured_na(na, records_schema, f"behavior {behavior_id} {risk_name} not_applicable", errors, risk_name)
1405
- elif status == "unresolved":
1406
- require(na is None, f"unresolved risk cannot also be N/A: {behavior_id}/{risk_name}", errors)
1407
- if verdict == "TEST_PASS":
1408
- errors.append(f"TEST_PASS has unresolved behavior risk: {behavior_id}/{risk_name}")
1409
-
1410
-
1411
- def validate_infrastructure(
1412
- manifest: dict[str, Any],
1413
- infrastructure: dict[str, Any],
1414
- executions: list[dict[str, Any]],
1415
- entry_map: dict[str, dict[str, Any]],
1416
- root: Path,
1417
- errors: list[str],
1418
- ) -> None:
1419
- verdict = manifest.get("final_verdict")
1420
- require(infrastructure.get("cleanup_succeeded") is True or verdict == "TEST_BLOCKED", "infrastructure cleanup did not succeed", errors)
1421
- external_targets = infrastructure.get("external_targets")
1422
- validate_external_targets(external_targets, "infrastructure external_targets", errors)
1423
- declared = {canonical_sha256(item) for item in external_targets if isinstance(item, dict)} if isinstance(external_targets, list) else set()
1424
- for receipt in executions:
1425
- for target in receipt.get("external_targets", []):
1426
- if isinstance(target, dict):
1427
- require(canonical_sha256(target) in declared, f"receipt external target is absent from infrastructure: {target.get('name')}", errors)
1428
- snapshots = infrastructure.get("contract_snapshots")
1429
- require(isinstance(snapshots, list), "contract snapshot provenance is missing", errors)
1430
- for record in snapshots if isinstance(snapshots, list) else []:
1431
- if not isinstance(record, dict):
1432
- continue
1433
- relative = record.get("path")
1434
- require(isinstance(relative, str) and path_under(relative, "contracts"), "contract snapshot must be under contracts/", errors)
1435
- path = require_manifest_entry(relative, entry_map, root, errors, "contract snapshot") if isinstance(relative, str) else None
1436
- if path is not None:
1437
- require(file_sha256(path) == record.get("sha256"), f"contract snapshot hash/provenance mismatch: {relative}", errors)
1438
-
1439
-
1440
- def validate_lightweight(
1441
- manifest: dict[str, Any],
1442
- record: dict[str, Any],
1443
- executions: list[dict[str, Any]],
1444
- errors: list[str],
1445
- ) -> None:
1446
- execution_by_id = {item.get("execution_id"): item for item in executions if isinstance(item.get("execution_id"), str)}
1447
- checks = record.get("checks")
1448
- require(isinstance(checks, list) and bool(checks), "lightweight deterministic checks are missing", errors)
1449
- for check in checks if isinstance(checks, list) else []:
1450
- if not isinstance(check, dict):
1451
- continue
1452
- receipt = execution_by_id.get(check.get("execution_id"))
1453
- require(receipt is not None and receipt.get("selected_execution") is True and receipt.get("reliable") is True and receipt.get("exit_code") == 0, "lightweight check lacks a successful deterministic receipt", errors)
1454
- if manifest.get("final_verdict") == "TEST_PASS":
1455
- require(record.get("runtime_behavior_claimed") is False, "lightweight TEST_PASS must not claim runtime behavior", errors)
1456
-
1457
-
1458
- def validate_verdict_and_attestation(
1459
- root: Path,
1460
- manifest: dict[str, Any],
1461
- verdict_record: dict[str, Any],
1462
- validation: dict[str, Any],
1463
- report: str,
1464
- entry_map: dict[str, dict[str, Any]],
1465
- executions: list[dict[str, Any]],
1466
- records_schema: dict[str, Any],
1467
- errors: list[str],
1468
- verify_attestation: bool,
1469
- ) -> None:
1470
- validate_definition(verdict_record, "verdict", records_schema, "verdict", errors)
1471
- verdict = manifest.get("final_verdict")
1472
- require(verdict in VERDICTS and verdict_record.get("verdict") == verdict, "verdict record does not match manifest", errors)
1473
- require(verdict_record.get("testing_domain_only") is True, "verdict exceeds testing-domain responsibility", errors)
1474
- require(verdict_record.get("authorizes_commit_or_release") is False, "test verdict must not authorize commit or release", errors)
1475
- require(verdict_record.get("real_environment_validated") is False, "code evidence must not claim real-environment validation", errors)
1476
- require(verdict_record.get("repairs_business_defects") is False, "test skill must not repair business defects", errors)
1477
- if verdict == "TEST_PASS":
1478
- selected = [
1479
- item for item in executions
1480
- if isinstance(item, dict) and item.get("selected_execution") is True
1481
- ]
1482
- require(bool(selected), "TEST_PASS has no selected executions", errors)
1483
- for item in selected:
1484
- execution_id = item.get("execution_id", "unknown-execution")
1485
- require(
1486
- item.get("reliable") is True,
1487
- f"TEST_PASS contains a selected unreliable execution: {execution_id}",
1488
- errors,
1489
- )
1490
- require(
1491
- item.get("exit_code") == 0,
1492
- f"TEST_PASS contains a selected failing execution: {execution_id}",
1493
- errors,
1494
- )
1495
- require(verdict_record.get("code_evidence_replayable") is True, "TEST_PASS must declare replayable code evidence", errors)
1496
- require(not verdict_record.get("blockers"), "TEST_PASS contains blockers", errors)
1497
- require(not verdict_record.get("reproduced_failures"), "TEST_PASS contains reproduced failures", errors)
1498
- elif verdict == "TEST_BLOCKED":
1499
- require(bool(verdict_record.get("blockers")), "TEST_BLOCKED lacks blocker reasons", errors)
1500
- elif verdict == "TEST_FAIL":
1501
- require(bool(verdict_record.get("reproduced_failures")), "TEST_FAIL lacks reproduced failures", errors)
1502
- require(
1503
- verdict_record.get("repair_scope") in {
1504
- "test-infrastructure", "production", "runtime", "schema", "dependency", "public-interface",
1505
- },
1506
- "TEST_FAIL lacks a valid evidence-backed repair_scope",
1507
- errors,
1508
- )
1509
-
1510
- if verify_attestation:
1511
- validate_definition(validation, "validation", records_schema, "validation", errors)
1512
- payload = sorted(
1513
- ({"path": path, "sha256": entry["sha256"]} for path, entry in entry_map.items() if path not in FINAL_RECORDS),
1514
- key=lambda item: item["path"],
1515
- )
1516
- replayed_ids = [
1517
- item.get("execution_id") for item in executions
1518
- if isinstance(item.get("execution_id"), str) and isinstance(item.get("replay_of"), str)
1519
- ]
1520
- require(validation.get("validator") == "validate_test_evidence.py", "validation attestation names wrong validator", errors)
1521
- require(validation.get("validator_sha256") == file_sha256(Path(__file__).resolve()), "validation attestation validator hash mismatch", errors)
1522
- require(validation.get("payload_sha256") == canonical_sha256(payload), "validation attestation payload hash mismatch", errors)
1523
- require(validation.get("result") == "passed", "validation attestation is not passed", errors)
1524
- require(validation.get("verdict") == verdict, "validation attestation verdict mismatch", errors)
1525
- require(validation.get("semantics") == "integrity-and-protocol-validation-not-hostile-producer-proof", "validation attestation semantics exceed integrity/protocol validation", errors)
1526
- require(validation.get("replay_receipt_ids") == replayed_ids, "validation replay receipt IDs do not match actual replay executions", errors)
1527
-
1528
- if root.joinpath("lifecycle.json").is_file():
1529
- expected_report = render_report_text(root, manifest, verdict_record, validation)
1530
- require(report == expected_report, "derived report does not match canonical rendering", errors)
1531
-
1532
- fragments = [
1533
- f"Verdict: {verdict}", f"Evidence ID: {manifest.get('evidence_id')}",
1534
- "Derived from structured evidence", "sensitivity=project-controlled",
1535
- "project owns access control, retention, and upload policy",
1536
- "Mocked code-level evidence does not verify a real deployed environment",
1537
- "legacy test-report interface is not supported",
1538
- ]
1539
- for fragment in fragments:
1540
- require(fragment in report, f"derived report missing authoritative evidence fragment: {fragment}", errors)
1541
-
1542
-
1543
- def validate_evidence(
1544
- root: Path,
1545
- project_root: Path,
1546
- ignored_hash_paths: set[str] | None = None,
1547
- ) -> list[str]:
1548
- errors: list[str] = []
1549
- try:
1550
- manifest = load_json(root / "manifest.json")
1551
- classification = load_json(root / "change-classification.json")
1552
- scope = load_json(root / "scope.json")
1553
- inventory = load_json(root / "target-inventory.json")
1554
- verdict_record = load_json(root / "verdict.json")
1555
- validation = load_json(root / "validation.json")
1556
- report = (root / "test-report.md").read_text(encoding="utf-8")
1557
- manifest_schema = load_schema("evidence-manifest.schema.json")
1558
- records_schema = load_schema("authoritative-records.schema.json")
1559
- matrix_schema = load_schema("behavior-risk-matrix.schema.json")
1560
- except (EvidenceError, FileNotFoundError) as exc:
1561
- return [str(exc)]
1562
- if not all(isinstance(item, dict) for item in (manifest, classification, scope, inventory, verdict_record, validation)):
1563
- return ["structured evidence records must be JSON objects"]
1564
-
1565
- entry_map = validate_manifest(root, manifest, manifest_schema, records_schema, errors, ignored_hash_paths)
1566
- change_class = manifest.get("change_class")
1567
- final_verdict = manifest.get("final_verdict")
1568
- canonical_lifecycle = (root / "lifecycle.json").is_file()
1569
- required = COMMON_RECORDS
1570
- if canonical_lifecycle:
1571
- required = required | {"change-capture.json"}
1572
- if final_verdict != "TEST_BLOCKED":
1573
- required = required | (BEHAVIOR_RECORDS if change_class == "behavior" else LIGHTWEIGHT_RECORDS)
1574
- if canonical_lifecycle and change_class == "behavior":
1575
- required = required | {"test-preparation.json"}
1576
- for relative in required:
1577
- require(relative in entry_map, f"required evidence record not hashed: {relative}", errors)
1578
- if canonical_lifecycle:
1579
- capture_path = root / "change-capture.json"
1580
- if capture_path.is_file():
1581
- try:
1582
- capture = load_json(capture_path)
1583
- validate_definition(capture, "changeCapture", records_schema, "change-capture", errors)
1584
- validate_canonical_capture(root, project_root, capture, inventory, scope, errors)
1585
- except EvidenceError as exc:
1586
- errors.append(str(exc))
1587
- validate_definition(classification, "changeClassification", records_schema, "change-classification", errors)
1588
- validate_definition(scope, "scope", records_schema, "scope", errors)
1589
- validate_definition(inventory, "targetInventory", records_schema, "target-inventory", errors)
1590
- require(classification.get("classification") == change_class, "change classification does not match manifest", errors)
1591
- require(classification.get("full_protocol_required") is (change_class == "behavior"), "change classification protocol decision is inconsistent", errors)
1592
-
1593
- plan: dict[str, Any] | None = None
1594
- preparation: dict[str, Any] | None = None
1595
- if change_class == "behavior":
1596
- try:
1597
- plan = load_json(root / "test-plan.json")
1598
- if canonical_lifecycle:
1599
- preparation = load_json(root / "test-preparation.json")
1600
- except EvidenceError as exc:
1601
- errors.append(str(exc))
1602
- plan = preparation = None
1603
- if isinstance(plan, dict):
1604
- validate_definition(plan, "testPlan", records_schema, "test-plan", errors)
1605
- else:
1606
- errors.append("test-plan.json must be an object")
1607
- plan = None
1608
- if canonical_lifecycle:
1609
- if isinstance(preparation, dict):
1610
- pass
1611
- else:
1612
- errors.append("test-preparation.json must be an object")
1613
-
1614
- inventory_by_category, _ = validate_inventory(root, project_root, manifest, inventory, scope, plan, entry_map, errors)
1615
- if canonical_lifecycle and change_class == "behavior" and isinstance(preparation, dict):
1616
- validate_test_preparation(
1617
- root, project_root, preparation, plan or {}, inventory_by_category,
1618
- entry_map, records_schema, errors,
1619
- )
1620
- require(manifest.get("evidence_id_inputs", {}).get("scope_sha256") == canonical_sha256(scope), "scope identity hash mismatch", errors)
1621
- validate_scope(
1622
- scope, inventory, inventory_by_category, project_root, root,
1623
- get_captured_deleted_paths(root), errors,
1624
- )
1625
- execution_requests, executions = validate_requests_and_receipts(root, entry_map, records_schema, errors)
1626
-
1627
- if change_class == "behavior":
1628
- try:
1629
- matrix = load_json(root / "behavior-risk-matrix.json")
1630
- infrastructure = load_json(root / "infrastructure-changes.json")
1631
- except EvidenceError as exc:
1632
- errors.append(str(exc))
1633
- matrix = infrastructure = None
1634
- if all(isinstance(item, dict) for item in (matrix, infrastructure, plan)):
1635
- matrix_error_count = len(errors)
1636
- validate_schema(matrix, matrix_schema, matrix_schema, "behavior-risk-matrix", errors)
1637
- validate_definition(infrastructure, "infrastructureChanges", records_schema, "infrastructure-changes", errors)
1638
- if len(errors) > matrix_error_count:
1639
- errors.append("behavior semantic cross-link validation skipped because behavior-risk-matrix schema validation failed")
1640
- else:
1641
- stages = {item.get("name"): item for item in manifest.get("stages", []) if isinstance(item, dict)}
1642
- for stage_name in CORE_BEHAVIOR_STAGES:
1643
- require(stages.get(stage_name, {}).get("status") == "complete", f"behavior protocol stage cannot be N/A: {stage_name}", errors)
1644
- test_map = validate_plan_and_scope_bindings(
1645
- root, project_root, manifest, scope, matrix, plan, inventory_by_category, entry_map,
1646
- executions, records_schema, errors,
1647
- )
1648
- validate_matrix_risks(manifest, matrix, test_map, executions, records_schema, errors)
1649
- validate_infrastructure(manifest, infrastructure, executions, entry_map, root, errors)
1650
- else:
1651
- errors.append("behavior evidence records must be JSON objects")
1652
- elif change_class == "lightweight":
1653
- try:
1654
- lightweight = load_json(root / "lightweight-verification.json")
1655
- except EvidenceError as exc:
1656
- errors.append(str(exc))
1657
- lightweight = None
1658
- if isinstance(lightweight, dict):
1659
- validate_definition(lightweight, "lightweightVerification", records_schema, "lightweight-verification", errors)
1660
- validate_lightweight(manifest, lightweight, executions, errors)
1661
- else:
1662
- errors.append("lightweight-verification.json must be an object")
1663
- else:
1664
- errors.append("invalid change_class")
1665
-
1666
- validate_verdict_and_attestation(
1667
- root, manifest, verdict_record, validation, report, entry_map, executions, records_schema,
1668
- errors, verify_attestation=not bool(ignored_hash_paths),
1669
- )
1670
- return errors
1671
-
1672
-
1673
- def atomic_write_text(path: Path, value: str) -> None:
1674
- path.parent.mkdir(parents=True, exist_ok=True)
1675
- descriptor, temporary_name = tempfile.mkstemp(prefix=f".{path.name}.", suffix=".tmp", dir=path.parent)
1676
- try:
1677
- with os.fdopen(descriptor, "w", encoding="utf-8") as handle:
1678
- handle.write(value)
1679
- os.replace(temporary_name, path)
1680
- except BaseException:
1681
- try:
1682
- os.unlink(temporary_name)
1683
- except FileNotFoundError:
1684
- pass
1685
- raise
1686
-
1687
-
1688
- def render_report_text(
1689
- root: Path,
1690
- manifest: dict[str, Any],
1691
- verdict: dict[str, Any],
1692
- validation: dict[str, Any],
1693
- ) -> str:
1694
- lifecycle_path = root / "lifecycle.json"
1695
- lifecycle = load_json(lifecycle_path) if lifecycle_path.exists() else {}
1696
- executions_record = load_json(root / "executions.json")
1697
- executions = executions_record if isinstance(executions_record, list) else []
1698
- artifact_dir = lifecycle.get("artifact_dir") if isinstance(lifecycle, dict) else None
1699
- lines = [
1700
- "# Test Evidence Report",
1701
- "",
1702
- "Derived from structured evidence",
1703
- f"Evidence ID: {manifest.get('evidence_id')}",
1704
- f"Verdict: {verdict.get('verdict')}",
1705
- f"Validator result: {validation.get('result', 'pending')}",
1706
- "sensitivity=project-controlled",
1707
- "The project owns access control, retention, and upload policy.",
1708
- "Mocked code-level evidence does not verify a real deployed environment.",
1709
- "",
1710
- "## Scope",
1711
- f"- Artifact Dir: {artifact_dir or 'not provided'}",
1712
- "",
1713
- f"Execution receipts: {len(executions)}",
1714
- "The legacy test-report interface is not supported.",
1715
- "",
1716
- ]
1717
- return "\n".join(lines)
1718
-
1719
-
1720
- def refresh_final_record_hashes(root: Path, manifest: dict[str, Any]) -> None:
1721
- for entry in manifest.get("files", []):
1722
- if isinstance(entry, dict) and entry.get("path") in FINAL_RECORDS:
1723
- entry["sha256"] = file_sha256(root / entry["path"])
1724
- atomic_write_json(root / "manifest.json", manifest)
1725
-
1726
-
1727
- def main() -> int:
1728
- parser = argparse.ArgumentParser(description="Validate a PrizmKit test evidence package")
1729
- parser.add_argument("--evidence-dir", required=True, help="Path to .prizmkit/test/evidence/<evidence-id>")
1730
- parser.add_argument("--project-root", help="Target project root; defaults to four parents above evidence directory")
1731
- parser.add_argument("--attest", action="store_true", help="Validate payload, then atomically refresh the validator-owned attestation and final hashes")
1732
- args = parser.parse_args()
1733
- root = Path(args.evidence_dir).resolve()
1734
- if not root.is_dir():
1735
- print(f"Test evidence validation: FAIL\n- evidence directory not found: {root}")
1736
- return 1
1737
- try:
1738
- project_root = Path(args.project_root).resolve() if args.project_root else root.parents[3]
1739
- except IndexError:
1740
- print("Test evidence validation: FAIL\n- cannot infer project root from evidence directory")
1741
- return 1
1742
- errors = validate_evidence(root, project_root, ATTESTATION_MUTABLE if args.attest else None)
1743
- if errors:
1744
- print("Test evidence validation: FAIL")
1745
- for error in errors:
1746
- print(f"- {error}")
1747
- return 1
1748
-
1749
- manifest = load_json(root / "manifest.json")
1750
- verdict_record = load_json(root / "verdict.json")
1751
- if args.attest:
1752
- executions = load_json(root / "executions.json")
1753
- replayed_ids = [
1754
- item.get("execution_id") for item in executions
1755
- if isinstance(item, dict)
1756
- and isinstance(item.get("execution_id"), str)
1757
- and isinstance(item.get("replay_of"), str)
1758
- ] if isinstance(executions, list) else []
1759
- payload = sorted(
1760
- (
1761
- {"path": entry["path"], "sha256": entry["sha256"]}
1762
- for entry in manifest["files"] if entry["path"] not in FINAL_RECORDS
1763
- ),
1764
- key=lambda item: item["path"],
1765
- )
1766
- validation = {
1767
- "validator": "validate_test_evidence.py",
1768
- "validator_sha256": file_sha256(Path(__file__).resolve()),
1769
- "payload_sha256": canonical_sha256(payload),
1770
- "result": "passed",
1771
- "verdict": manifest["final_verdict"],
1772
- "semantics": "integrity-and-protocol-validation-not-hostile-producer-proof",
1773
- "replay_receipt_ids": replayed_ids,
1774
- }
1775
- atomic_write_json(root / "validation.json", validation)
1776
- report_text = render_report_text(root, manifest, verdict_record, validation)
1777
- atomic_write_text(root / "test-report.md", report_text)
1778
- refresh_final_record_hashes(root, manifest)
1779
- errors = validate_evidence(root, project_root)
1780
- if errors:
1781
- print("Test evidence validation: FAIL")
1782
- for error in errors:
1783
- print(f"- {error}")
1784
- return 1
1785
-
1786
- print("Evidence validation: PASS")
1787
- print(f"Testing verdict: {manifest['final_verdict']}")
1788
- print(f"Evidence ID: {manifest['evidence_id']}")
1789
- return 0
1790
-
1791
-
1792
- if __name__ == "__main__":
1793
- sys.exit(main())