prizmkit 1.1.119 → 1.1.121
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/create-prizmkit.js +1 -5
- package/bundled/VERSION.json +3 -3
- package/bundled/dev-pipeline/README.md +41 -38
- package/bundled/dev-pipeline/assets/skill-subagent-integration.md +76 -0
- package/bundled/dev-pipeline/prizmkit_runtime/config.py +2 -8
- package/bundled/dev-pipeline/prizmkit_runtime/interoperability.py +1 -1
- package/bundled/dev-pipeline/prizmkit_runtime/platform_detection.py +3 -3
- package/bundled/dev-pipeline/prizmkit_runtime/runner_classification.py +23 -25
- package/bundled/dev-pipeline/prizmkit_runtime/runner_models.py +8 -2
- package/bundled/dev-pipeline/prizmkit_runtime/runner_prompts.py +10 -1
- package/bundled/dev-pipeline/prizmkit_runtime/runners.py +88 -32
- package/bundled/dev-pipeline/prizmkit_runtime/sessions.py +28 -6
- package/bundled/dev-pipeline/scripts/continuation.py +51 -6
- package/bundled/dev-pipeline/scripts/generate-bootstrap-prompt.py +60 -102
- package/bundled/dev-pipeline/scripts/generate-bugfix-prompt.py +1 -19
- package/bundled/dev-pipeline/scripts/generate-refactor-prompt.py +1 -19
- package/bundled/dev-pipeline/scripts/{init-dev-team.py → init-change-artifact.py} +6 -16
- package/bundled/dev-pipeline/scripts/parse-stream-progress.py +364 -76
- package/bundled/dev-pipeline/scripts/update-bug-status.py +57 -14
- package/bundled/dev-pipeline/scripts/update-feature-status.py +54 -5
- package/bundled/dev-pipeline/scripts/update-refactor-status.py +57 -14
- package/bundled/dev-pipeline/scripts/utils.py +1 -1
- package/bundled/dev-pipeline/templates/bootstrap-prompt.md +2 -2
- package/bundled/dev-pipeline/templates/bootstrap-tier3.md +9 -11
- package/bundled/dev-pipeline/templates/bugfix-bootstrap-prompt.md +2 -1
- package/bundled/dev-pipeline/templates/refactor-bootstrap-prompt.md +2 -2
- package/bundled/dev-pipeline/templates/sections/bugfix-phase-review.md +3 -3
- package/bundled/dev-pipeline/templates/sections/context-budget-rules.md +1 -1
- package/bundled/dev-pipeline/templates/sections/critical-paths-agent.md +0 -1
- package/bundled/dev-pipeline/templates/sections/critical-paths-full.md +0 -2
- package/bundled/dev-pipeline/templates/sections/phase-commit-full.md +8 -1
- package/bundled/dev-pipeline/templates/sections/phase-implement-agent.md +6 -7
- package/bundled/dev-pipeline/templates/sections/phase-implement-full.md +6 -7
- package/bundled/dev-pipeline/templates/sections/phase-implement-lite.md +4 -2
- package/bundled/dev-pipeline/templates/sections/phase-review-agent.md +3 -3
- package/bundled/dev-pipeline/templates/sections/phase-review-full.md +5 -3
- package/bundled/dev-pipeline/templates/sections/phase-specify-plan-full.md +20 -0
- package/bundled/dev-pipeline/templates/sections/refactor-phase-review.md +3 -3
- package/bundled/dev-pipeline/tests/test_auto_skip.py +5 -5
- package/bundled/dev-pipeline/tests/test_generate_bootstrap_prompt.py +214 -144
- package/bundled/dev-pipeline/tests/test_generate_bugfix_prompt.py +25 -12
- package/bundled/dev-pipeline/tests/test_generate_refactor_prompt.py +11 -14
- package/bundled/dev-pipeline/tests/test_python_runner_parity.py +235 -32
- package/bundled/dev-pipeline/tests/test_unified_cli.py +219 -16
- package/bundled/skills/_metadata.json +4 -4
- package/bundled/skills/app-planner/SKILL.md +2 -2
- package/bundled/skills/app-planner/references/architecture-decisions.md +1 -3
- package/bundled/skills/prizmkit-code-review/SKILL.md +50 -47
- package/bundled/skills/prizmkit-code-review/references/review-report-template.md +13 -8
- package/bundled/skills/prizmkit-code-review/references/reviewer-agent-prompt.md +47 -33
- package/bundled/skills/prizmkit-code-review/references/reviewer-workspace-protocol.md +90 -0
- package/bundled/skills/prizmkit-implement/SKILL.md +21 -1
- package/bundled/skills/prizmkit-implement/references/implementation-subagent-procedure.md +67 -0
- package/bundled/skills/prizmkit-test/SKILL.md +159 -155
- package/bundled/skills/prizmkit-test/assets/authoritative-records.schema.json +420 -0
- package/bundled/skills/prizmkit-test/assets/behavior-risk-matrix.schema.json +116 -0
- package/bundled/skills/prizmkit-test/assets/evidence-manifest.schema.json +112 -0
- package/bundled/skills/prizmkit-test/assets/evidence-package-template.json +97 -0
- package/bundled/skills/prizmkit-test/references/boundary-coverage-protocol.md +76 -192
- package/bundled/skills/prizmkit-test/references/contract-mock-protocol.md +65 -0
- package/bundled/skills/prizmkit-test/references/evidence-protocol.md +194 -0
- package/bundled/skills/prizmkit-test/references/evidence-request-protocol.md +78 -0
- package/bundled/skills/prizmkit-test/references/examples.md +65 -108
- package/bundled/skills/prizmkit-test/references/service-boundary-test-catalog.md +10 -7
- package/bundled/skills/prizmkit-test/references/test-generation-steps.md +90 -82
- package/bundled/skills/prizmkit-test/references/test-report-template.md +77 -98
- package/bundled/skills/prizmkit-test/references/trusted-evidence-execution.md +97 -0
- package/bundled/skills/prizmkit-test/scripts/build_test_evidence.py +723 -0
- package/bundled/skills/prizmkit-test/scripts/validate_test_evidence.py +1426 -0
- package/package.json +1 -1
- package/src/clean.js +12 -8
- package/src/config.js +24 -42
- package/src/index.js +1 -10
- package/src/manifest.js +3 -9
- package/src/metadata.js +0 -26
- package/src/prompts.js +0 -13
- package/src/scaffold.js +76 -201
- package/src/upgrade.js +16 -33
- package/bundled/adapters/claude/agent-adapter.js +0 -96
- package/bundled/adapters/claude/team-adapter.js +0 -183
- package/bundled/adapters/codebuddy/agent-adapter.js +0 -42
- package/bundled/adapters/codebuddy/team-adapter.js +0 -46
- package/bundled/adapters/codex/agent-adapter.js +0 -38
- package/bundled/adapters/codex/team-adapter.js +0 -37
- package/bundled/agents/prizm-dev-team-dev.md +0 -123
- package/bundled/agents/prizm-dev-team-reviewer.md +0 -113
- package/bundled/dev-pipeline/assets/prizm-dev-team-integration.md +0 -65
- package/bundled/dev-pipeline/templates/bootstrap-tier1.md +0 -436
- package/bundled/dev-pipeline/templates/bootstrap-tier2.md +0 -510
- package/bundled/skills/prizmkit-test/scripts/validate_boundary_report.py +0 -347
- package/bundled/team/prizm-dev-team.json +0 -27
|
@@ -0,0 +1,1426 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Deterministically validate a PrizmKit structured test-evidence package.
|
|
3
|
+
|
|
4
|
+
This validator deliberately checks protocol mechanics and content-addressed evidence;
|
|
5
|
+
it does not treat a successful validation as a claim about a deployed environment or
|
|
6
|
+
as authorization to commit, release, or repair product behavior.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import argparse
|
|
12
|
+
import hashlib
|
|
13
|
+
import json
|
|
14
|
+
import math
|
|
15
|
+
import os
|
|
16
|
+
import re
|
|
17
|
+
import sys
|
|
18
|
+
import tempfile
|
|
19
|
+
from pathlib import Path
|
|
20
|
+
from typing import Any
|
|
21
|
+
|
|
22
|
+
STAGES = [
|
|
23
|
+
"CHANGE_CLASSIFY", "SCOPE_DISCOVER", "CONTRACT_MODEL", "TEST_PLAN",
|
|
24
|
+
"INFRA_READY", "TEST_BUILD", "EXECUTE_PROVE", "EVIDENCE_PACKAGE",
|
|
25
|
+
"EVIDENCE_VALIDATE",
|
|
26
|
+
]
|
|
27
|
+
CORE_BEHAVIOR_STAGES = set(STAGES)
|
|
28
|
+
VERDICTS = {"TEST_PASS", "TEST_FAIL", "TEST_BLOCKED"}
|
|
29
|
+
RISK_KEYS = {
|
|
30
|
+
"functional", "boundary", "permission", "concurrency",
|
|
31
|
+
"idempotency", "time", "dependency", "consumer",
|
|
32
|
+
}
|
|
33
|
+
LAYER_ORDER = {
|
|
34
|
+
"focused": 0,
|
|
35
|
+
"module-component": 1,
|
|
36
|
+
"contract-integration": 2,
|
|
37
|
+
"affected-module-regression": 3,
|
|
38
|
+
"regression-ring": 4,
|
|
39
|
+
}
|
|
40
|
+
FINAL_RECORDS = {"validation.json", "verdict.json", "test-report.md"}
|
|
41
|
+
COMMON_RECORDS = {
|
|
42
|
+
"change-classification.json", "scope.json", "target-inventory.json",
|
|
43
|
+
"environment.json", "executions.json", "verdict.json", "validation.json",
|
|
44
|
+
"source-change.patch", "test-report.md",
|
|
45
|
+
}
|
|
46
|
+
BEHAVIOR_RECORDS = {
|
|
47
|
+
"behavior-risk-matrix.json", "test-plan.json", "infrastructure-changes.json",
|
|
48
|
+
"differential-proof.json",
|
|
49
|
+
}
|
|
50
|
+
LIGHTWEIGHT_RECORDS = {"lightweight-verification.json"}
|
|
51
|
+
SHA256_RE = re.compile(r"^[a-f0-9]{64}$")
|
|
52
|
+
UUID_RE = re.compile(r"^[a-f0-9-]{36}$")
|
|
53
|
+
BLOCKED_EXTERNAL_CLASSES = {"production", "unknown"}
|
|
54
|
+
RISK_CONFLICT_PATTERNS = {
|
|
55
|
+
"permission": re.compile(r"auth|tenant|role|permission|access|acl|rbac", re.I),
|
|
56
|
+
"concurrency": re.compile(r"lock|shared[ _-]?state|worker|queue|concurr|parallel|race", re.I),
|
|
57
|
+
"idempotency": re.compile(r"retry|idempot|dedup|duplicate|replay", re.I),
|
|
58
|
+
"time": re.compile(r"timeout|ttl|expiry|expire|clock|date|time", re.I),
|
|
59
|
+
"dependency": re.compile(r"client|contract|network|https?|api|service|dependenc|external", re.I),
|
|
60
|
+
"consumer": re.compile(r"caller|consumer|shared[ _-]?contract|import", re.I),
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
class EvidenceError(Exception):
|
|
65
|
+
"""Raised for malformed or unreadable evidence input."""
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def canonical_sha256(value: Any) -> str:
|
|
69
|
+
payload = json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
|
|
70
|
+
return hashlib.sha256(payload.encode("utf-8")).hexdigest()
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def file_sha256(path: Path) -> str:
|
|
74
|
+
digest = hashlib.sha256()
|
|
75
|
+
with path.open("rb") as handle:
|
|
76
|
+
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
|
77
|
+
digest.update(chunk)
|
|
78
|
+
return digest.hexdigest()
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def _reject_duplicate_keys(pairs: list[tuple[str, Any]]) -> dict[str, Any]:
|
|
82
|
+
result: dict[str, Any] = {}
|
|
83
|
+
for key, value in pairs:
|
|
84
|
+
if key in result:
|
|
85
|
+
raise ValueError(f"duplicate object key: {key}")
|
|
86
|
+
result[key] = value
|
|
87
|
+
return result
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def _reject_non_json_number(value: str) -> Any:
|
|
91
|
+
raise ValueError(f"non-JSON numeric constant: {value}")
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def load_json(path: Path) -> Any:
|
|
95
|
+
try:
|
|
96
|
+
return json.loads(
|
|
97
|
+
path.read_text(encoding="utf-8"),
|
|
98
|
+
object_pairs_hook=_reject_duplicate_keys,
|
|
99
|
+
parse_constant=_reject_non_json_number,
|
|
100
|
+
)
|
|
101
|
+
except FileNotFoundError as exc:
|
|
102
|
+
raise EvidenceError(f"missing required file: {path.name}") from exc
|
|
103
|
+
except (json.JSONDecodeError, ValueError) as exc:
|
|
104
|
+
raise EvidenceError(f"invalid JSON in {path.name}: {exc}") from exc
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def atomic_write_json(path: Path, value: Any) -> None:
|
|
108
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
109
|
+
descriptor, temporary_name = tempfile.mkstemp(prefix=f".{path.name}.", suffix=".tmp", dir=path.parent)
|
|
110
|
+
try:
|
|
111
|
+
with os.fdopen(descriptor, "w", encoding="utf-8") as handle:
|
|
112
|
+
json.dump(value, handle, indent=2, ensure_ascii=False)
|
|
113
|
+
handle.write("\n")
|
|
114
|
+
os.replace(temporary_name, path)
|
|
115
|
+
except BaseException:
|
|
116
|
+
try:
|
|
117
|
+
os.unlink(temporary_name)
|
|
118
|
+
except FileNotFoundError:
|
|
119
|
+
pass
|
|
120
|
+
raise
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def require(condition: bool, message: str, errors: list[str]) -> None:
|
|
124
|
+
if not condition:
|
|
125
|
+
errors.append(message)
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def is_sha256(value: Any) -> bool:
|
|
129
|
+
return isinstance(value, str) and bool(SHA256_RE.fullmatch(value))
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def is_relative_path(value: Any) -> bool:
|
|
133
|
+
return isinstance(value, str) and bool(value) and not Path(value).is_absolute()
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def safe_path(root: Path, relative: Any, errors: list[str], label: str) -> Path | None:
|
|
137
|
+
if not is_relative_path(relative):
|
|
138
|
+
errors.append(f"{label} must be a non-empty relative path")
|
|
139
|
+
return None
|
|
140
|
+
candidate = (root / relative).resolve()
|
|
141
|
+
try:
|
|
142
|
+
candidate.relative_to(root.resolve())
|
|
143
|
+
except ValueError:
|
|
144
|
+
errors.append(f"{label} escapes its root: {relative}")
|
|
145
|
+
return None
|
|
146
|
+
return candidate
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
def path_under(relative: str, directory: str) -> bool:
|
|
150
|
+
try:
|
|
151
|
+
Path(relative).relative_to(directory)
|
|
152
|
+
return True
|
|
153
|
+
except ValueError:
|
|
154
|
+
return False
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
def path_below_or_equal(path: str, root: str) -> bool:
|
|
158
|
+
normalized_path = Path(path).as_posix()
|
|
159
|
+
normalized_root = Path(root).as_posix().rstrip("/")
|
|
160
|
+
return normalized_path == normalized_root or normalized_path.startswith(f"{normalized_root}/")
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def resolve_ref(schema: dict[str, Any], root_schema: dict[str, Any]) -> dict[str, Any]:
|
|
164
|
+
reference = schema.get("$ref")
|
|
165
|
+
if not reference:
|
|
166
|
+
return schema
|
|
167
|
+
if not isinstance(reference, str) or not reference.startswith("#/"):
|
|
168
|
+
raise EvidenceError(f"unsupported schema reference: {reference}")
|
|
169
|
+
current: Any = root_schema
|
|
170
|
+
for token in reference[2:].split("/"):
|
|
171
|
+
if not isinstance(current, dict):
|
|
172
|
+
raise EvidenceError(f"invalid schema reference: {reference}")
|
|
173
|
+
current = current[token.replace("~1", "/").replace("~0", "~")]
|
|
174
|
+
if not isinstance(current, dict):
|
|
175
|
+
raise EvidenceError(f"schema reference is not an object: {reference}")
|
|
176
|
+
return current
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
def json_type_matches(value: Any, expected: str) -> bool:
|
|
180
|
+
if expected == "object":
|
|
181
|
+
return isinstance(value, dict)
|
|
182
|
+
if expected == "array":
|
|
183
|
+
return isinstance(value, list)
|
|
184
|
+
if expected == "string":
|
|
185
|
+
return isinstance(value, str)
|
|
186
|
+
if expected == "boolean":
|
|
187
|
+
return isinstance(value, bool)
|
|
188
|
+
if expected == "integer":
|
|
189
|
+
return isinstance(value, int) and not isinstance(value, bool)
|
|
190
|
+
if expected == "number":
|
|
191
|
+
return isinstance(value, (int, float)) and not isinstance(value, bool) and math.isfinite(value)
|
|
192
|
+
if expected == "null":
|
|
193
|
+
return value is None
|
|
194
|
+
return False
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
def json_equals(left: Any, right: Any) -> bool:
|
|
198
|
+
"""JSON equality without Python's bool-is-an-int equivalence."""
|
|
199
|
+
if isinstance(left, bool) or isinstance(right, bool):
|
|
200
|
+
return type(left) is type(right) and left == right
|
|
201
|
+
if isinstance(left, (int, float)) and isinstance(right, (int, float)):
|
|
202
|
+
return type(left) is type(right) and left == right
|
|
203
|
+
return left == right
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
def validate_schema(
|
|
207
|
+
value: Any,
|
|
208
|
+
schema: dict[str, Any],
|
|
209
|
+
root_schema: dict[str, Any],
|
|
210
|
+
location: str,
|
|
211
|
+
errors: list[str],
|
|
212
|
+
) -> None:
|
|
213
|
+
"""Validate the strict JSON-Schema subset used by skill-owned schemas."""
|
|
214
|
+
schema = resolve_ref(schema, root_schema)
|
|
215
|
+
|
|
216
|
+
for child in schema.get("allOf", []):
|
|
217
|
+
if isinstance(child, dict):
|
|
218
|
+
validate_schema(value, child, root_schema, location, errors)
|
|
219
|
+
if "const" in schema:
|
|
220
|
+
require(json_equals(value, schema["const"]), f"{location}: value does not match const", errors)
|
|
221
|
+
if "enum" in schema:
|
|
222
|
+
allowed = schema["enum"]
|
|
223
|
+
require(
|
|
224
|
+
isinstance(allowed, list) and any(json_equals(value, item) for item in allowed),
|
|
225
|
+
f"{location}: value is outside enum",
|
|
226
|
+
errors,
|
|
227
|
+
)
|
|
228
|
+
|
|
229
|
+
expected = schema.get("type")
|
|
230
|
+
if expected is not None:
|
|
231
|
+
expected_types = expected if isinstance(expected, list) else [expected]
|
|
232
|
+
valid_types = [item for item in expected_types if isinstance(item, str)]
|
|
233
|
+
if not any(json_type_matches(value, item) for item in valid_types):
|
|
234
|
+
name = " or ".join(valid_types) if valid_types else repr(expected)
|
|
235
|
+
errors.append(f"{location}: expected {name}")
|
|
236
|
+
return
|
|
237
|
+
|
|
238
|
+
if isinstance(value, str):
|
|
239
|
+
if "minLength" in schema:
|
|
240
|
+
require(len(value) >= schema["minLength"], f"{location}: string is too short", errors)
|
|
241
|
+
if "maxLength" in schema:
|
|
242
|
+
require(len(value) <= schema["maxLength"], f"{location}: string is too long", errors)
|
|
243
|
+
if "pattern" in schema:
|
|
244
|
+
try:
|
|
245
|
+
matched = re.search(schema["pattern"], value) is not None
|
|
246
|
+
except re.error as exc:
|
|
247
|
+
raise EvidenceError(f"invalid schema pattern at {location}: {exc}") from exc
|
|
248
|
+
require(matched, f"{location}: string does not match pattern", errors)
|
|
249
|
+
|
|
250
|
+
if isinstance(value, (int, float)) and not isinstance(value, bool):
|
|
251
|
+
if "minimum" in schema:
|
|
252
|
+
require(value >= schema["minimum"], f"{location}: number is below minimum", errors)
|
|
253
|
+
if "maximum" in schema:
|
|
254
|
+
require(value <= schema["maximum"], f"{location}: number is above maximum", errors)
|
|
255
|
+
if "exclusiveMinimum" in schema:
|
|
256
|
+
require(value > schema["exclusiveMinimum"], f"{location}: number is not above exclusive minimum", errors)
|
|
257
|
+
if "exclusiveMaximum" in schema:
|
|
258
|
+
require(value < schema["exclusiveMaximum"], f"{location}: number is not below exclusive maximum", errors)
|
|
259
|
+
|
|
260
|
+
if isinstance(value, list):
|
|
261
|
+
if "minItems" in schema:
|
|
262
|
+
require(len(value) >= schema["minItems"], f"{location}: array has too few items", errors)
|
|
263
|
+
if "maxItems" in schema:
|
|
264
|
+
require(len(value) <= schema["maxItems"], f"{location}: array has too many items", errors)
|
|
265
|
+
if schema.get("uniqueItems") is True:
|
|
266
|
+
require(len({canonical_sha256(item) for item in value}) == len(value), f"{location}: array items are not unique", errors)
|
|
267
|
+
item_schema = schema.get("items")
|
|
268
|
+
if isinstance(item_schema, dict):
|
|
269
|
+
for index, item in enumerate(value):
|
|
270
|
+
validate_schema(item, item_schema, root_schema, f"{location}[{index}]", errors)
|
|
271
|
+
|
|
272
|
+
if isinstance(value, dict):
|
|
273
|
+
if "minProperties" in schema:
|
|
274
|
+
require(len(value) >= schema["minProperties"], f"{location}: object has too few properties", errors)
|
|
275
|
+
if "maxProperties" in schema:
|
|
276
|
+
require(len(value) <= schema["maxProperties"], f"{location}: object has too many properties", errors)
|
|
277
|
+
required = schema.get("required", [])
|
|
278
|
+
for key in required:
|
|
279
|
+
require(key in value, f"{location}: missing required property {key}", errors)
|
|
280
|
+
properties = schema.get("properties", {})
|
|
281
|
+
additional = schema.get("additionalProperties", True)
|
|
282
|
+
for key, item in value.items():
|
|
283
|
+
if key in properties:
|
|
284
|
+
continue
|
|
285
|
+
if additional is False:
|
|
286
|
+
errors.append(f"{location}: unexpected property {key}")
|
|
287
|
+
elif isinstance(additional, dict):
|
|
288
|
+
validate_schema(item, additional, root_schema, f"{location}.{key}", errors)
|
|
289
|
+
for key, child_schema in properties.items():
|
|
290
|
+
if key in value and isinstance(child_schema, dict):
|
|
291
|
+
validate_schema(value[key], child_schema, root_schema, f"{location}.{key}", errors)
|
|
292
|
+
|
|
293
|
+
|
|
294
|
+
def assets_dir() -> Path:
|
|
295
|
+
return Path(__file__).resolve().parent.parent / "assets"
|
|
296
|
+
|
|
297
|
+
|
|
298
|
+
def load_schema(name: str) -> dict[str, Any]:
|
|
299
|
+
schema = load_json(assets_dir() / name)
|
|
300
|
+
if not isinstance(schema, dict):
|
|
301
|
+
raise EvidenceError(f"schema is not an object: {name}")
|
|
302
|
+
return schema
|
|
303
|
+
|
|
304
|
+
|
|
305
|
+
def validate_definition(
|
|
306
|
+
value: Any,
|
|
307
|
+
definition: str,
|
|
308
|
+
records_schema: dict[str, Any],
|
|
309
|
+
location: str,
|
|
310
|
+
errors: list[str],
|
|
311
|
+
) -> None:
|
|
312
|
+
definitions = records_schema.get("$defs")
|
|
313
|
+
if not isinstance(definitions, dict) or not isinstance(definitions.get(definition), dict):
|
|
314
|
+
raise EvidenceError(f"missing authoritative schema definition: {definition}")
|
|
315
|
+
validate_schema(value, definitions[definition], records_schema, location, errors)
|
|
316
|
+
|
|
317
|
+
|
|
318
|
+
def require_manifest_entry(
|
|
319
|
+
relative: str,
|
|
320
|
+
entry_map: dict[str, dict[str, Any]],
|
|
321
|
+
root: Path,
|
|
322
|
+
errors: list[str],
|
|
323
|
+
label: str,
|
|
324
|
+
) -> Path | None:
|
|
325
|
+
entry = entry_map.get(relative)
|
|
326
|
+
require(entry is not None, f"{label} is not manifest-hashed: {relative}", errors)
|
|
327
|
+
path = safe_path(root, relative, errors, label)
|
|
328
|
+
if path is None:
|
|
329
|
+
return None
|
|
330
|
+
if not path.is_file():
|
|
331
|
+
errors.append(f"{label} is missing: {relative}")
|
|
332
|
+
return None
|
|
333
|
+
if entry is not None:
|
|
334
|
+
require(file_sha256(path) == entry.get("sha256"), f"{label} hash mismatch: {relative}", errors)
|
|
335
|
+
return path
|
|
336
|
+
|
|
337
|
+
|
|
338
|
+
def structured_na(
|
|
339
|
+
value: Any,
|
|
340
|
+
records_schema: dict[str, Any],
|
|
341
|
+
location: str,
|
|
342
|
+
errors: list[str],
|
|
343
|
+
risk: str | None = None,
|
|
344
|
+
) -> None:
|
|
345
|
+
validate_definition(value, "naDecision", records_schema, location, errors)
|
|
346
|
+
if not isinstance(value, dict):
|
|
347
|
+
return
|
|
348
|
+
signals = value.get("considered_signals")
|
|
349
|
+
require(isinstance(signals, list) and bool(signals), f"{location}: N/A needs considered signals", errors)
|
|
350
|
+
evidence = value.get("evidence")
|
|
351
|
+
require(isinstance(evidence, list) and bool(evidence), f"{location}: N/A needs discovery evidence", errors)
|
|
352
|
+
conflicts = value.get("conflicts")
|
|
353
|
+
require(isinstance(conflicts, list), f"{location}: N/A conflicts must be an array", errors)
|
|
354
|
+
if not isinstance(signals, list) or not isinstance(conflicts, list):
|
|
355
|
+
return
|
|
356
|
+
|
|
357
|
+
signal_text = "\n".join(
|
|
358
|
+
item for item in signals if isinstance(item, str)
|
|
359
|
+
)
|
|
360
|
+
if isinstance(evidence, list):
|
|
361
|
+
for item in evidence:
|
|
362
|
+
if isinstance(item, dict):
|
|
363
|
+
source = item.get("source")
|
|
364
|
+
observations = item.get("observations")
|
|
365
|
+
if isinstance(source, str):
|
|
366
|
+
signal_text += f"\n{source}"
|
|
367
|
+
if isinstance(observations, list):
|
|
368
|
+
signal_text += "\n" + "\n".join(value for value in observations if isinstance(value, str))
|
|
369
|
+
if risk and risk in RISK_CONFLICT_PATTERNS and RISK_CONFLICT_PATTERNS[risk].search(signal_text):
|
|
370
|
+
pattern = RISK_CONFLICT_PATTERNS[risk]
|
|
371
|
+
matching_signals = [
|
|
372
|
+
item for item in conflicts
|
|
373
|
+
if isinstance(item, dict)
|
|
374
|
+
and isinstance(item.get("signal"), str)
|
|
375
|
+
and pattern.search(item["signal"])
|
|
376
|
+
and isinstance(item.get("explanation"), str)
|
|
377
|
+
and len(item["explanation"].strip()) >= 8
|
|
378
|
+
]
|
|
379
|
+
require(
|
|
380
|
+
bool(matching_signals),
|
|
381
|
+
f"{location}: {risk} N/A conflicts with discovered signals but has no explained conflict",
|
|
382
|
+
errors,
|
|
383
|
+
)
|
|
384
|
+
|
|
385
|
+
|
|
386
|
+
def validate_external_targets(value: Any, location: str, errors: list[str]) -> None:
|
|
387
|
+
if not isinstance(value, list):
|
|
388
|
+
errors.append(f"{location}: external targets must be an array")
|
|
389
|
+
return
|
|
390
|
+
for index, target in enumerate(value):
|
|
391
|
+
target_location = f"{location}[{index}]"
|
|
392
|
+
if not isinstance(target, dict):
|
|
393
|
+
errors.append(f"{target_location}: external target must be an object")
|
|
394
|
+
continue
|
|
395
|
+
external = target.get("external")
|
|
396
|
+
require(type(external) is bool, f"{target_location}: external must be a boolean", errors)
|
|
397
|
+
if external is not True:
|
|
398
|
+
continue
|
|
399
|
+
classification = target.get("classification")
|
|
400
|
+
require(classification not in BLOCKED_EXTERNAL_CLASSES, f"{target_location}: external {classification} target is blocked", errors)
|
|
401
|
+
require(classification in {"isolated", "test", "staging"}, f"{target_location}: external target classification is unsafe", errors)
|
|
402
|
+
endpoint_evidence = target.get("endpoint_evidence")
|
|
403
|
+
allow_evidence = target.get("allow_evidence")
|
|
404
|
+
deny_evidence = target.get("deny_evidence")
|
|
405
|
+
require(isinstance(endpoint_evidence, list) and bool(endpoint_evidence), f"{target_location}: external target lacks endpoint evidence", errors)
|
|
406
|
+
require(isinstance(allow_evidence, list) and bool(allow_evidence), f"{target_location}: external target lacks allow evidence", errors)
|
|
407
|
+
require(isinstance(deny_evidence, list) and not deny_evidence, f"{target_location}: external target has deny evidence", errors)
|
|
408
|
+
|
|
409
|
+
|
|
410
|
+
def stage_input_sha(stage_name: str, target_hashes: dict[str, str], predecessor_files: list[dict[str, str]]) -> str:
|
|
411
|
+
return canonical_sha256({
|
|
412
|
+
"stage": stage_name,
|
|
413
|
+
"target_hashes": target_hashes,
|
|
414
|
+
"predecessor_outputs": sorted(predecessor_files, key=lambda item: item["path"]),
|
|
415
|
+
})
|
|
416
|
+
|
|
417
|
+
|
|
418
|
+
def validate_manifest(
|
|
419
|
+
root: Path,
|
|
420
|
+
manifest: dict[str, Any],
|
|
421
|
+
manifest_schema: dict[str, Any],
|
|
422
|
+
records_schema: dict[str, Any],
|
|
423
|
+
errors: list[str],
|
|
424
|
+
ignored_hash_paths: set[str] | None = None,
|
|
425
|
+
) -> dict[str, dict[str, Any]]:
|
|
426
|
+
validate_schema(manifest, manifest_schema, manifest_schema, "manifest", errors)
|
|
427
|
+
require(manifest.get("sensitivity") == "project-controlled", "sensitivity must be project-controlled", errors)
|
|
428
|
+
require(manifest.get("environment_claim") == "mocked-code-level-only", "environment claim must be mocked-code-level-only", errors)
|
|
429
|
+
require(manifest.get("compatibility") == "legacy-test-report-interface-not-supported", "legacy compatibility boundary missing", errors)
|
|
430
|
+
|
|
431
|
+
identity_inputs = manifest.get("evidence_id_inputs")
|
|
432
|
+
if isinstance(identity_inputs, dict):
|
|
433
|
+
require(manifest.get("evidence_id") == canonical_sha256(identity_inputs), "evidence_id does not match canonical identity inputs", errors)
|
|
434
|
+
require(root.name == manifest.get("evidence_id"), "evidence directory name does not match evidence_id", errors)
|
|
435
|
+
require(identity_inputs.get("baseline_commit") == manifest.get("baseline_commit"), "baseline commit identity mismatch", errors)
|
|
436
|
+
require(identity_inputs.get("working_diff_sha256") == manifest.get("working_diff_sha256"), "working diff identity mismatch", errors)
|
|
437
|
+
|
|
438
|
+
entries = manifest.get("files")
|
|
439
|
+
entry_map: dict[str, dict[str, Any]] = {}
|
|
440
|
+
if not isinstance(entries, list):
|
|
441
|
+
return entry_map
|
|
442
|
+
for index, entry in enumerate(entries):
|
|
443
|
+
entry_location = f"manifest.files[{index}]"
|
|
444
|
+
if not isinstance(entry, dict):
|
|
445
|
+
continue
|
|
446
|
+
relative = entry.get("path")
|
|
447
|
+
if not is_relative_path(relative):
|
|
448
|
+
continue
|
|
449
|
+
require(relative not in entry_map, f"duplicate manifest file entry: {relative}", errors)
|
|
450
|
+
entry_map[relative] = entry
|
|
451
|
+
candidate = safe_path(root, relative, errors, entry_location)
|
|
452
|
+
if candidate is None:
|
|
453
|
+
continue
|
|
454
|
+
if not candidate.is_file():
|
|
455
|
+
errors.append(f"manifest file missing: {relative}")
|
|
456
|
+
continue
|
|
457
|
+
if relative not in (ignored_hash_paths or set()):
|
|
458
|
+
require(file_sha256(candidate) == entry.get("sha256"), f"content hash mismatch: {relative}", errors)
|
|
459
|
+
|
|
460
|
+
actual_paths = {
|
|
461
|
+
candidate.relative_to(root).as_posix()
|
|
462
|
+
for candidate in root.rglob("*")
|
|
463
|
+
if candidate.is_file() and candidate.name != "manifest.json"
|
|
464
|
+
}
|
|
465
|
+
require(set(entry_map) == actual_paths, "manifest must hash every evidence file exactly once", errors)
|
|
466
|
+
|
|
467
|
+
stages = manifest.get("stages")
|
|
468
|
+
if not isinstance(stages, list):
|
|
469
|
+
return entry_map
|
|
470
|
+
names = [stage.get("name") for stage in stages if isinstance(stage, dict)]
|
|
471
|
+
require(names == STAGES, "stages are missing or out of order", errors)
|
|
472
|
+
predecessor_files: list[dict[str, str]] = []
|
|
473
|
+
for stage in stages:
|
|
474
|
+
if not isinstance(stage, dict):
|
|
475
|
+
continue
|
|
476
|
+
name = stage.get("name")
|
|
477
|
+
outputs = stage.get("outputs")
|
|
478
|
+
status = stage.get("status")
|
|
479
|
+
na = stage.get("not_applicable")
|
|
480
|
+
if status == "not_applicable":
|
|
481
|
+
structured_na(na, records_schema, f"stage {name} not_applicable", errors)
|
|
482
|
+
require(outputs == [], f"not-applicable stage must not claim outputs: {name}", errors)
|
|
483
|
+
elif status == "complete":
|
|
484
|
+
require(na is None, f"complete stage must have null not_applicable: {name}", errors)
|
|
485
|
+
for output in outputs if isinstance(outputs, list) else []:
|
|
486
|
+
entry = entry_map.get(output)
|
|
487
|
+
require(entry is not None, f"stage output is not a hashed evidence file: {name}/{output}", errors)
|
|
488
|
+
if entry is not None:
|
|
489
|
+
require(entry.get("produced_by") == name, f"stage output producer mismatch: {name}/{output}", errors)
|
|
490
|
+
if isinstance(name, str) and isinstance(manifest.get("target_hashes"), dict):
|
|
491
|
+
expected_input = stage_input_sha(name, manifest["target_hashes"], predecessor_files)
|
|
492
|
+
require(stage.get("input_sha256") == expected_input, f"stage input dependency hash mismatch: {name}", errors)
|
|
493
|
+
predecessor_files.extend(
|
|
494
|
+
{"path": output, "sha256": entry_map[output]["sha256"]}
|
|
495
|
+
for output in outputs if isinstance(outputs, list) and output in entry_map and output not in FINAL_RECORDS
|
|
496
|
+
)
|
|
497
|
+
return entry_map
|
|
498
|
+
|
|
499
|
+
|
|
500
|
+
def validate_inventory(
|
|
501
|
+
root: Path,
|
|
502
|
+
project_root: Path,
|
|
503
|
+
manifest: dict[str, Any],
|
|
504
|
+
inventory: dict[str, Any],
|
|
505
|
+
scope: dict[str, Any],
|
|
506
|
+
plan: dict[str, Any] | None,
|
|
507
|
+
entry_map: dict[str, dict[str, Any]],
|
|
508
|
+
errors: list[str],
|
|
509
|
+
) -> tuple[dict[str, dict[str, str]], dict[str, str]]:
|
|
510
|
+
categories = inventory.get("categories")
|
|
511
|
+
expected_categories = {"source", "tests", "contracts", "lockfiles"}
|
|
512
|
+
inventory_by_category: dict[str, dict[str, str]] = {}
|
|
513
|
+
aggregates: dict[str, str] = {}
|
|
514
|
+
if not isinstance(categories, dict) or set(categories) != expected_categories:
|
|
515
|
+
errors.append("target inventory categories are incomplete")
|
|
516
|
+
return inventory_by_category, aggregates
|
|
517
|
+
|
|
518
|
+
all_paths: dict[str, str] = {}
|
|
519
|
+
for category in sorted(expected_categories):
|
|
520
|
+
entries = categories.get(category)
|
|
521
|
+
category_entries: dict[str, str] = {}
|
|
522
|
+
if not isinstance(entries, list):
|
|
523
|
+
errors.append(f"target inventory category is not an array: {category}")
|
|
524
|
+
continue
|
|
525
|
+
for entry in entries:
|
|
526
|
+
if not isinstance(entry, dict):
|
|
527
|
+
errors.append(f"invalid target inventory entry: {category}")
|
|
528
|
+
continue
|
|
529
|
+
relative = entry.get("path")
|
|
530
|
+
expected_hash = entry.get("sha256")
|
|
531
|
+
if not is_relative_path(relative) or not is_sha256(expected_hash):
|
|
532
|
+
errors.append(f"invalid target inventory entry: {category}")
|
|
533
|
+
continue
|
|
534
|
+
require(relative not in category_entries, f"duplicate target inventory path: {relative}", errors)
|
|
535
|
+
require(relative not in all_paths, f"target inventory path appears in multiple categories: {relative}", errors)
|
|
536
|
+
candidate = safe_path(project_root, relative, errors, "target inventory path")
|
|
537
|
+
if candidate is None or not candidate.is_file():
|
|
538
|
+
errors.append(f"target project file missing: {relative}")
|
|
539
|
+
continue
|
|
540
|
+
actual = file_sha256(candidate)
|
|
541
|
+
require(actual == expected_hash, f"target project file drift: {relative}", errors)
|
|
542
|
+
category_entries[relative] = actual
|
|
543
|
+
all_paths[relative] = category
|
|
544
|
+
inventory_by_category[category] = category_entries
|
|
545
|
+
aggregates[category] = canonical_sha256([
|
|
546
|
+
{"path": path, "sha256": category_entries[path]}
|
|
547
|
+
for path in sorted(category_entries)
|
|
548
|
+
])
|
|
549
|
+
|
|
550
|
+
try:
|
|
551
|
+
environment = load_json(root / "environment.json")
|
|
552
|
+
except EvidenceError as exc:
|
|
553
|
+
errors.append(str(exc))
|
|
554
|
+
environment = None
|
|
555
|
+
aggregates["environment"] = canonical_sha256(environment)
|
|
556
|
+
plan_inputs = inventory.get("plan_inputs")
|
|
557
|
+
require(isinstance(plan_inputs, dict) and bool(plan_inputs), "target inventory plan_inputs missing", errors)
|
|
558
|
+
aggregates["plan"] = canonical_sha256(plan_inputs)
|
|
559
|
+
require(manifest.get("target_hashes") == aggregates, "target hashes do not match live project inventory", errors)
|
|
560
|
+
require(scope.get("target_hashes") == aggregates, "scope target hashes do not match live inventory", errors)
|
|
561
|
+
if plan is not None:
|
|
562
|
+
require(plan.get("target_hashes") == aggregates, "test plan target hashes do not match live inventory", errors)
|
|
563
|
+
require(plan.get("plan_inputs") == plan_inputs, "test plan inputs do not match target inventory", errors)
|
|
564
|
+
|
|
565
|
+
patch = require_manifest_entry("source-change.patch", entry_map, root, errors, "source-change patch")
|
|
566
|
+
if patch is not None:
|
|
567
|
+
require(manifest.get("working_diff_sha256") == file_sha256(patch), "working_diff_sha256 does not match source-change.patch", errors)
|
|
568
|
+
|
|
569
|
+
request_relative = inventory.get("inventory_request_path")
|
|
570
|
+
request_path = require_manifest_entry(request_relative, entry_map, root, errors, "inventory request") if isinstance(request_relative, str) else None
|
|
571
|
+
if request_path is not None:
|
|
572
|
+
require(file_sha256(request_path) == inventory.get("inventory_request_sha256"), "inventory request hash mismatch", errors)
|
|
573
|
+
request = load_json(request_path)
|
|
574
|
+
if isinstance(request, dict):
|
|
575
|
+
require(request.get("changed_files") == inventory.get("changed_files"), "inventory request changed_files mismatch", errors)
|
|
576
|
+
require(request.get("module_roots") == inventory.get("module_roots"), "inventory request module_roots mismatch", errors)
|
|
577
|
+
require(request.get("exclusions") == scope.get("exclusions"), "inventory request exclusions mismatch", errors)
|
|
578
|
+
require(request.get("plan_inputs") == plan_inputs, "inventory request plan_inputs mismatch", errors)
|
|
579
|
+
else:
|
|
580
|
+
errors.append("inventory request must be an object")
|
|
581
|
+
|
|
582
|
+
return inventory_by_category, aggregates
|
|
583
|
+
|
|
584
|
+
|
|
585
|
+
def validate_scope(
|
|
586
|
+
scope: dict[str, Any],
|
|
587
|
+
inventory: dict[str, Any],
|
|
588
|
+
inventory_by_category: dict[str, dict[str, str]],
|
|
589
|
+
project_root: Path,
|
|
590
|
+
evidence_root: Path,
|
|
591
|
+
errors: list[str],
|
|
592
|
+
) -> None:
|
|
593
|
+
changed_files = scope.get("changed_files")
|
|
594
|
+
require(isinstance(changed_files, list) and bool(changed_files), "scope changed_files is empty", errors)
|
|
595
|
+
if not isinstance(changed_files, list):
|
|
596
|
+
return
|
|
597
|
+
require(len(set(changed_files)) == len(changed_files), "scope changed_files contains duplicates", errors)
|
|
598
|
+
require(sorted(changed_files) == sorted(inventory.get("changed_files", [])), "scope and inventory changed_files do not agree", errors)
|
|
599
|
+
|
|
600
|
+
exclusions = scope.get("exclusions")
|
|
601
|
+
exclusion_paths = {
|
|
602
|
+
item.get("path") for item in exclusions if isinstance(item, dict) and isinstance(item.get("path"), str)
|
|
603
|
+
} if isinstance(exclusions, list) else set()
|
|
604
|
+
require(exclusion_paths == set(inventory.get("exclusions", [])), "scope and inventory exclusions do not agree", errors)
|
|
605
|
+
inventory_paths = {path for entries in inventory_by_category.values() for path in entries}
|
|
606
|
+
for relative in changed_files:
|
|
607
|
+
candidate = safe_path(project_root, relative, errors, "changed file")
|
|
608
|
+
if relative in exclusion_paths:
|
|
609
|
+
require(relative not in inventory_paths, f"excluded changed file is also inventoried: {relative}", errors)
|
|
610
|
+
else:
|
|
611
|
+
require(candidate is not None and candidate.is_file(), f"changed file is not live: {relative}", errors)
|
|
612
|
+
require(relative in inventory_paths, f"changed file is neither inventoried nor excluded: {relative}", errors)
|
|
613
|
+
|
|
614
|
+
roots = scope.get("module_roots")
|
|
615
|
+
require(isinstance(roots, list) and bool(roots), "scope module_roots is empty", errors)
|
|
616
|
+
if not isinstance(roots, list):
|
|
617
|
+
return
|
|
618
|
+
scope_root_paths = [item.get("path") for item in roots if isinstance(item, dict)]
|
|
619
|
+
require(len(set(scope_root_paths)) == len(scope_root_paths), "scope module_roots contains duplicates", errors)
|
|
620
|
+
require(sorted(scope_root_paths) == sorted(inventory.get("module_roots", [])), "scope and inventory module_roots do not agree", errors)
|
|
621
|
+
inventory_root_files = inventory.get("module_root_files")
|
|
622
|
+
require(isinstance(inventory_root_files, dict), "inventory module_root_files is missing", errors)
|
|
623
|
+
require(
|
|
624
|
+
isinstance(inventory_root_files, dict) and set(inventory_root_files) == set(scope_root_paths),
|
|
625
|
+
"inventory module_root_files does not match declared module roots",
|
|
626
|
+
errors,
|
|
627
|
+
)
|
|
628
|
+
evidence_resolved = evidence_root.resolve()
|
|
629
|
+
for root_info in roots:
|
|
630
|
+
if not isinstance(root_info, dict):
|
|
631
|
+
continue
|
|
632
|
+
root_relative = root_info.get("path")
|
|
633
|
+
root_path = safe_path(project_root, root_relative, errors, "module root")
|
|
634
|
+
require(root_path is not None and root_path.exists(), f"module root is not live: {root_relative}", errors)
|
|
635
|
+
files = root_info.get("files")
|
|
636
|
+
require(isinstance(files, list) and bool(files), f"module root has no files: {root_relative}", errors)
|
|
637
|
+
live_root_files: list[str] = []
|
|
638
|
+
if root_path is not None and root_path.exists():
|
|
639
|
+
candidates = [root_path] if root_path.is_file() else root_path.rglob("*")
|
|
640
|
+
for candidate in candidates:
|
|
641
|
+
if not candidate.is_file() or ".git" in candidate.parts or "__pycache__" in candidate.parts:
|
|
642
|
+
continue
|
|
643
|
+
resolved = candidate.resolve()
|
|
644
|
+
if resolved == evidence_resolved or evidence_resolved in resolved.parents:
|
|
645
|
+
continue
|
|
646
|
+
live_root_files.append(resolved.relative_to(project_root).as_posix())
|
|
647
|
+
recorded_root_files = inventory_root_files.get(root_relative) if isinstance(inventory_root_files, dict) else None
|
|
648
|
+
require(
|
|
649
|
+
isinstance(recorded_root_files, list) and sorted(recorded_root_files) == sorted(live_root_files),
|
|
650
|
+
f"module root live enumeration differs from inventory: {root_relative}",
|
|
651
|
+
errors,
|
|
652
|
+
)
|
|
653
|
+
require(
|
|
654
|
+
isinstance(files, list) and sorted(files) == sorted(live_root_files),
|
|
655
|
+
f"scope module root files are incomplete: {root_relative}",
|
|
656
|
+
errors,
|
|
657
|
+
)
|
|
658
|
+
for relative in files if isinstance(files, list) else []:
|
|
659
|
+
candidate = safe_path(project_root, relative, errors, "module root file")
|
|
660
|
+
require(candidate is not None and candidate.is_file(), f"module root file is not live: {relative}", errors)
|
|
661
|
+
require(path_below_or_equal(relative, root_relative), f"module root file lies outside its root: {relative}", errors)
|
|
662
|
+
require(relative in inventory_paths or relative in exclusion_paths, f"module root file is neither inventoried nor excluded: {relative}", errors)
|
|
663
|
+
|
|
664
|
+
require(
|
|
665
|
+
isinstance(scope.get("primary_scope"), list) and bool(scope.get("primary_scope")),
|
|
666
|
+
"Primary Scope must contain observable behavior descriptions",
|
|
667
|
+
errors,
|
|
668
|
+
)
|
|
669
|
+
|
|
670
|
+
|
|
671
|
+
def validate_requests_and_receipts(
|
|
672
|
+
root: Path,
|
|
673
|
+
entry_map: dict[str, dict[str, Any]],
|
|
674
|
+
records_schema: dict[str, Any],
|
|
675
|
+
errors: list[str],
|
|
676
|
+
) -> tuple[dict[str, dict[str, Any]], dict[str, dict[str, Any]], list[dict[str, Any]]]:
|
|
677
|
+
execution_requests: dict[str, dict[str, Any]] = {}
|
|
678
|
+
differential_requests: dict[str, dict[str, Any]] = {}
|
|
679
|
+
requests_root = root / "requests"
|
|
680
|
+
if requests_root.exists():
|
|
681
|
+
for path in sorted(requests_root.rglob("*.json")):
|
|
682
|
+
relative = path.relative_to(root).as_posix()
|
|
683
|
+
require_manifest_entry(relative, entry_map, root, errors, "request")
|
|
684
|
+
try:
|
|
685
|
+
request = load_json(path)
|
|
686
|
+
except EvidenceError as exc:
|
|
687
|
+
errors.append(str(exc))
|
|
688
|
+
continue
|
|
689
|
+
if not isinstance(request, dict):
|
|
690
|
+
errors.append(f"request must be an object: {relative}")
|
|
691
|
+
continue
|
|
692
|
+
if "execution_request" in request:
|
|
693
|
+
validate_definition(request, "differentialRequest", records_schema, relative, errors)
|
|
694
|
+
nested = request.get("execution_request")
|
|
695
|
+
if isinstance(nested, dict):
|
|
696
|
+
validate_definition(nested, "executionRequest", records_schema, f"{relative}.execution_request", errors)
|
|
697
|
+
validate_external_targets(nested.get("external_targets"), f"{relative}.execution_request.external_targets", errors)
|
|
698
|
+
differential_requests[relative] = request
|
|
699
|
+
elif "command" in request and "layer" in request:
|
|
700
|
+
validate_definition(request, "executionRequest", records_schema, relative, errors)
|
|
701
|
+
validate_execution_request_semantics(request, relative, errors)
|
|
702
|
+
validate_external_targets(request.get("external_targets"), f"{relative}.external_targets", errors)
|
|
703
|
+
execution_requests[relative] = request
|
|
704
|
+
elif set(request) == {
|
|
705
|
+
"request_version", "categories", "changed_files", "module_roots",
|
|
706
|
+
"exclusions", "discovery_evidence", "plan_inputs",
|
|
707
|
+
}:
|
|
708
|
+
require(request.get("request_version") == "1.0", f"unsupported inventory request version: {relative}", errors)
|
|
709
|
+
else:
|
|
710
|
+
errors.append(f"unrecognized request record: {relative}")
|
|
711
|
+
|
|
712
|
+
runner = Path(__file__).resolve().with_name("build_test_evidence.py")
|
|
713
|
+
if not runner.is_file():
|
|
714
|
+
errors.append("runner source is missing: build_test_evidence.py")
|
|
715
|
+
|
|
716
|
+
executions_path = root / "executions.json"
|
|
717
|
+
try:
|
|
718
|
+
executions = load_json(executions_path)
|
|
719
|
+
except EvidenceError as exc:
|
|
720
|
+
errors.append(str(exc))
|
|
721
|
+
return execution_requests, differential_requests, []
|
|
722
|
+
validate_definition(executions, "executionLog", records_schema, "executions", errors)
|
|
723
|
+
if not isinstance(executions, list):
|
|
724
|
+
return execution_requests, differential_requests, []
|
|
725
|
+
|
|
726
|
+
receipt_by_id: dict[str, dict[str, Any]] = {}
|
|
727
|
+
seen_receipt_paths: set[str] = set()
|
|
728
|
+
attempts_by_request: dict[str, int] = {}
|
|
729
|
+
previous_receipt: dict[str, Any] | None = None
|
|
730
|
+
for index, receipt in enumerate(executions):
|
|
731
|
+
location = f"executions[{index}]"
|
|
732
|
+
if not isinstance(receipt, dict):
|
|
733
|
+
continue
|
|
734
|
+
validate_definition(receipt, "executionReceipt", records_schema, location, errors)
|
|
735
|
+
receipt_id = receipt.get("execution_id")
|
|
736
|
+
require(isinstance(receipt_id, str) and bool(UUID_RE.fullmatch(receipt_id)), f"{location}: invalid execution_id", errors)
|
|
737
|
+
require(receipt_id not in receipt_by_id, f"duplicate execution id: {receipt_id}", errors)
|
|
738
|
+
if isinstance(receipt_id, str):
|
|
739
|
+
receipt_by_id[receipt_id] = receipt
|
|
740
|
+
|
|
741
|
+
require(receipt.get("receipt_format") == "prizmkit-runner-generated-v1", f"{location}: receipt is not runner-generated", errors)
|
|
742
|
+
runner_relative = receipt.get("runner_path")
|
|
743
|
+
runner_snapshot = require_manifest_entry(
|
|
744
|
+
runner_relative, entry_map, root, errors, "runner snapshot",
|
|
745
|
+
) if isinstance(runner_relative, str) else None
|
|
746
|
+
require(
|
|
747
|
+
isinstance(runner_relative, str) and path_under(runner_relative, "runner"),
|
|
748
|
+
f"{location}: runner snapshot must be under runner/",
|
|
749
|
+
errors,
|
|
750
|
+
)
|
|
751
|
+
if runner_snapshot is not None:
|
|
752
|
+
require(file_sha256(runner_snapshot) == receipt.get("runner_sha256"), f"{location}: runner snapshot hash mismatch", errors)
|
|
753
|
+
request_relative = receipt.get("request_path")
|
|
754
|
+
request = execution_requests.get(request_relative) if isinstance(request_relative, str) else None
|
|
755
|
+
require(request is not None, f"{location}: receipt references an unknown execution request", errors)
|
|
756
|
+
request_path = require_manifest_entry(request_relative, entry_map, root, errors, "receipt request") if isinstance(request_relative, str) else None
|
|
757
|
+
if request_path is not None:
|
|
758
|
+
require(file_sha256(request_path) == receipt.get("request_sha256"), f"{location}: request hash mismatch", errors)
|
|
759
|
+
if isinstance(request, dict):
|
|
760
|
+
for key in ("purpose", "command", "cwd", "layer", "test_ids", "external_targets"):
|
|
761
|
+
require(receipt.get(key) == request.get(key), f"{location}: receipt {key} does not match request", errors)
|
|
762
|
+
expected_environment = request.get("environment")
|
|
763
|
+
actual_environment = receipt.get("environment")
|
|
764
|
+
require(isinstance(actual_environment, dict) and isinstance(actual_environment.get("PATH"), str), f"{location}: receipt lacks complete PATH environment", errors)
|
|
765
|
+
if isinstance(expected_environment, dict) and isinstance(actual_environment, dict):
|
|
766
|
+
for key, value in expected_environment.items():
|
|
767
|
+
require(actual_environment.get(key) == value, f"{location}: receipt environment differs for {key}", errors)
|
|
768
|
+
expected_probes = request.get("tool_version_commands")
|
|
769
|
+
actual_probes = receipt.get("tool_versions")
|
|
770
|
+
require(isinstance(actual_probes, dict), f"{location}: receipt tool_versions must be an object", errors)
|
|
771
|
+
if isinstance(expected_probes, dict) and isinstance(actual_probes, dict):
|
|
772
|
+
require(set(actual_probes) == set(expected_probes), f"{location}: receipt tool probe names differ from request", errors)
|
|
773
|
+
for probe_name, command in expected_probes.items():
|
|
774
|
+
probe = actual_probes.get(probe_name)
|
|
775
|
+
require(isinstance(probe, dict), f"{location}: tool probe is not an object: {probe_name}", errors)
|
|
776
|
+
if isinstance(probe, dict):
|
|
777
|
+
require(probe.get("command") == command, f"{location}: tool probe command mismatch: {probe_name}", errors)
|
|
778
|
+
require("exit_code" in probe or "error" in probe, f"{location}: tool probe has no result: {probe_name}", errors)
|
|
779
|
+
|
|
780
|
+
validate_execution_request_semantics(receipt, location, errors, receipt=True)
|
|
781
|
+
validate_external_targets(receipt.get("external_targets"), f"{location}.external_targets", errors)
|
|
782
|
+
for raw_key, hash_key in (("stdout_path", "stdout_sha256"), ("stderr_path", "stderr_sha256")):
|
|
783
|
+
relative = receipt.get(raw_key)
|
|
784
|
+
raw_path = safe_path(root, relative, errors, f"{location}.{raw_key}")
|
|
785
|
+
require(isinstance(relative, str) and path_under(relative, "raw"), f"{location}: {raw_key} must be under raw/", errors)
|
|
786
|
+
if raw_path is not None:
|
|
787
|
+
require(raw_path.is_file(), f"{location}: raw output is missing: {relative}", errors)
|
|
788
|
+
if raw_path.is_file():
|
|
789
|
+
require(file_sha256(raw_path) == receipt.get(hash_key), f"{location}: raw output hash mismatch: {relative}", errors)
|
|
790
|
+
require_manifest_entry(relative, entry_map, root, errors, "raw output")
|
|
791
|
+
|
|
792
|
+
recorded_receipt_path = f"receipts/{receipt_id}.json" if isinstance(receipt_id, str) else ""
|
|
793
|
+
receipt_path = require_manifest_entry(recorded_receipt_path, entry_map, root, errors, "receipt") if recorded_receipt_path else None
|
|
794
|
+
if receipt_path is not None:
|
|
795
|
+
try:
|
|
796
|
+
require(load_json(receipt_path) == receipt, f"{location}: receipt file does not match execution log", errors)
|
|
797
|
+
except EvidenceError as exc:
|
|
798
|
+
errors.append(str(exc))
|
|
799
|
+
seen_receipt_paths.add(recorded_receipt_path)
|
|
800
|
+
|
|
801
|
+
expected_previous = canonical_sha256(previous_receipt) if previous_receipt is not None else None
|
|
802
|
+
require(receipt.get("previous_receipt_sha256") == expected_previous, f"{location}: receipt chain mismatch", errors)
|
|
803
|
+
request_hash = receipt.get("request_sha256")
|
|
804
|
+
if isinstance(request_hash, str):
|
|
805
|
+
attempts_by_request[request_hash] = attempts_by_request.get(request_hash, 0) + 1
|
|
806
|
+
require(receipt.get("attempt_index") == attempts_by_request[request_hash], f"{location}: attempt_index is not request-local sequence", errors)
|
|
807
|
+
replay_of = receipt.get("replay_of")
|
|
808
|
+
if replay_of is not None:
|
|
809
|
+
previous = receipt_by_id.get(replay_of)
|
|
810
|
+
require(previous is not None, f"{location}: replay source is missing", errors)
|
|
811
|
+
if previous is not None:
|
|
812
|
+
require(previous.get("request_sha256") == request_hash, f"{location}: replay request differs from source", errors)
|
|
813
|
+
previous_receipt = receipt
|
|
814
|
+
|
|
815
|
+
receipt_files = {
|
|
816
|
+
path.relative_to(root).as_posix()
|
|
817
|
+
for path in (root / "receipts").rglob("*.json")
|
|
818
|
+
} if (root / "receipts").is_dir() else set()
|
|
819
|
+
require(receipt_files == seen_receipt_paths, "receipt directory does not exactly match executions.json", errors)
|
|
820
|
+
return execution_requests, differential_requests, executions
|
|
821
|
+
|
|
822
|
+
|
|
823
|
+
def validate_execution_request_semantics(
|
|
824
|
+
value: dict[str, Any],
|
|
825
|
+
location: str,
|
|
826
|
+
errors: list[str],
|
|
827
|
+
receipt: bool = False,
|
|
828
|
+
) -> None:
|
|
829
|
+
command = value.get("command")
|
|
830
|
+
require(isinstance(command, list) and bool(command) and all(isinstance(item, str) and item for item in command), f"{location}: command must be a non-empty string argv", errors)
|
|
831
|
+
environment = value.get("environment")
|
|
832
|
+
require(isinstance(environment, dict) and all(isinstance(key, str) and isinstance(item, str) for key, item in environment.items()), f"{location}: environment must contain complete string values", errors)
|
|
833
|
+
test_ids = value.get("test_ids")
|
|
834
|
+
require(isinstance(test_ids, list) and all(isinstance(item, str) and item for item in test_ids), f"{location}: test_ids must be string IDs", errors)
|
|
835
|
+
if receipt:
|
|
836
|
+
require(type(value.get("selected_execution")) is bool, f"{location}: selected_execution must be boolean", errors)
|
|
837
|
+
require(type(value.get("reliable")) is bool, f"{location}: reliable must be boolean", errors)
|
|
838
|
+
require(type(value.get("exit_code")) is int and not isinstance(value.get("exit_code"), bool), f"{location}: exit_code must be an integer", errors)
|
|
839
|
+
|
|
840
|
+
|
|
841
|
+
def validate_plan_and_scope_bindings(
|
|
842
|
+
root: Path,
|
|
843
|
+
project_root: Path,
|
|
844
|
+
manifest: dict[str, Any],
|
|
845
|
+
scope: dict[str, Any],
|
|
846
|
+
matrix: dict[str, Any],
|
|
847
|
+
plan: dict[str, Any],
|
|
848
|
+
inventory_by_category: dict[str, dict[str, str]],
|
|
849
|
+
entry_map: dict[str, dict[str, Any]],
|
|
850
|
+
executions: list[dict[str, Any]],
|
|
851
|
+
records_schema: dict[str, Any],
|
|
852
|
+
errors: list[str],
|
|
853
|
+
) -> dict[str, dict[str, Any]]:
|
|
854
|
+
require(scope.get("affected_module") == matrix.get("affected_module", {}).get("name"), "matrix affected module does not match scope", errors)
|
|
855
|
+
require(scope.get("boundary_source") == matrix.get("affected_module", {}).get("boundary_source"), "matrix boundary source does not match scope", errors)
|
|
856
|
+
require(scope.get("primary_scope") == matrix.get("primary_scope"), "matrix primary scope does not match scope", errors)
|
|
857
|
+
scope_ring = sorted((item.get("name"), item.get("kind")) for item in scope.get("regression_ring", []) if isinstance(item, dict))
|
|
858
|
+
matrix_ring = sorted((item.get("name"), item.get("kind")) for item in matrix.get("regression_ring", []) if isinstance(item, dict))
|
|
859
|
+
require(scope_ring == matrix_ring, "scope and matrix regression rings do not agree", errors)
|
|
860
|
+
|
|
861
|
+
unresolved = scope.get("unresolved_edges")
|
|
862
|
+
if manifest.get("final_verdict") == "TEST_PASS" and isinstance(unresolved, list):
|
|
863
|
+
for edge in unresolved:
|
|
864
|
+
if isinstance(edge, dict) and edge.get("verdict_capable") is True and edge.get("resolved") is not True:
|
|
865
|
+
errors.append("TEST_PASS has a verdict-capable unresolved edge")
|
|
866
|
+
|
|
867
|
+
layers = plan.get("layers")
|
|
868
|
+
layer_map = {item.get("name"): item for item in layers if isinstance(item, dict)} if isinstance(layers, list) else {}
|
|
869
|
+
require(set(layer_map) == set(LAYER_ORDER), "test plan must assess every risk-adaptive layer exactly once", errors)
|
|
870
|
+
for name, layer in layer_map.items():
|
|
871
|
+
required = layer.get("required")
|
|
872
|
+
na = layer.get("not_applicable")
|
|
873
|
+
require(type(required) is bool, f"layer required flag invalid: {name}", errors)
|
|
874
|
+
if required is True:
|
|
875
|
+
require(na is None, f"required layer cannot be N/A: {name}", errors)
|
|
876
|
+
elif required is False:
|
|
877
|
+
structured_na(na, records_schema, f"layer {name} not_applicable", errors)
|
|
878
|
+
if matrix.get("regression_ring"):
|
|
879
|
+
require(layer_map.get("regression-ring", {}).get("required") is True, "nonempty regression ring must be required", errors)
|
|
880
|
+
if manifest.get("final_verdict") == "TEST_PASS":
|
|
881
|
+
for layer_name, layer in layer_map.items():
|
|
882
|
+
if layer.get("required") is True:
|
|
883
|
+
matches = [
|
|
884
|
+
item for item in executions
|
|
885
|
+
if item.get("selected_execution") is True
|
|
886
|
+
and item.get("layer") == layer_name
|
|
887
|
+
and item.get("reliable") is True
|
|
888
|
+
and item.get("exit_code") == 0
|
|
889
|
+
]
|
|
890
|
+
require(bool(matches), f"required layer lacks a successful reliable receipt: {layer_name}", errors)
|
|
891
|
+
|
|
892
|
+
tests = plan.get("tests")
|
|
893
|
+
test_map: dict[str, dict[str, Any]] = {}
|
|
894
|
+
if not isinstance(tests, list):
|
|
895
|
+
return test_map
|
|
896
|
+
selected_by_test: dict[str, list[dict[str, Any]]] = {}
|
|
897
|
+
for receipt in executions:
|
|
898
|
+
if receipt.get("selected_execution") is True:
|
|
899
|
+
for test_id in receipt.get("test_ids", []):
|
|
900
|
+
if isinstance(test_id, str):
|
|
901
|
+
selected_by_test.setdefault(test_id, []).append(receipt)
|
|
902
|
+
known_behavior_ids = {item.get("id") for item in matrix.get("behaviors", []) if isinstance(item, dict)}
|
|
903
|
+
for test in tests:
|
|
904
|
+
if not isinstance(test, dict):
|
|
905
|
+
continue
|
|
906
|
+
test_id = test.get("id")
|
|
907
|
+
require(isinstance(test_id, str) and bool(test_id), "planned test has no ID", errors)
|
|
908
|
+
require(test_id not in test_map, f"duplicate planned test ID: {test_id}", errors)
|
|
909
|
+
if not isinstance(test_id, str):
|
|
910
|
+
continue
|
|
911
|
+
test_map[test_id] = test
|
|
912
|
+
behavior_ids = test.get("behavior_ids")
|
|
913
|
+
require(isinstance(behavior_ids, list) and bool(behavior_ids), f"planned test has no behavior mapping: {test_id}", errors)
|
|
914
|
+
for behavior_id in behavior_ids if isinstance(behavior_ids, list) else []:
|
|
915
|
+
require(behavior_id in known_behavior_ids, f"planned test maps unknown behavior: {test_id}/{behavior_id}", errors)
|
|
916
|
+
project_relative = test.get("project_path")
|
|
917
|
+
inventory_relative = test.get("inventory_path")
|
|
918
|
+
snapshot_relative = test.get("snapshot_path")
|
|
919
|
+
project_path = safe_path(project_root, project_relative, errors, f"planned test {test_id} project_path")
|
|
920
|
+
require(project_path is not None and project_path.is_file(), f"planned test live path is missing: {test_id}", errors)
|
|
921
|
+
require(inventory_relative == project_relative, f"planned test inventory path differs from live project path: {test_id}", errors)
|
|
922
|
+
require(inventory_relative in inventory_by_category.get("tests", {}), f"planned test is not in the test inventory: {test_id}", errors)
|
|
923
|
+
snapshot_path = require_manifest_entry(snapshot_relative, entry_map, root, errors, f"planned test snapshot {test_id}") if isinstance(snapshot_relative, str) else None
|
|
924
|
+
require(isinstance(snapshot_relative, str) and path_under(snapshot_relative, "generated-tests"), f"planned test snapshot must be under generated-tests/: {test_id}", errors)
|
|
925
|
+
if project_path is not None and project_path.is_file() and snapshot_path is not None and snapshot_path.is_file():
|
|
926
|
+
require(file_sha256(project_path) == file_sha256(snapshot_path), f"generated test snapshot differs from live test: {test_id}", errors)
|
|
927
|
+
test_layers = test.get("layers")
|
|
928
|
+
require(isinstance(test_layers, list) and bool(test_layers), f"planned test has no layers: {test_id}", errors)
|
|
929
|
+
for layer in test_layers if isinstance(test_layers, list) else []:
|
|
930
|
+
require(layer in LAYER_ORDER, f"planned test uses unknown layer: {test_id}/{layer}", errors)
|
|
931
|
+
matching_receipts = selected_by_test.get(test_id, [])
|
|
932
|
+
require(bool(matching_receipts), f"planned test has no selected receipt: {test_id}", errors)
|
|
933
|
+
for receipt in matching_receipts:
|
|
934
|
+
require(receipt.get("layer") in test_layers, f"planned test receipt uses an unplanned layer: {test_id}", errors)
|
|
935
|
+
|
|
936
|
+
for ring in scope.get("regression_ring", []):
|
|
937
|
+
if not isinstance(ring, dict):
|
|
938
|
+
continue
|
|
939
|
+
ring_name = ring.get("name")
|
|
940
|
+
ring_files = ring.get("files")
|
|
941
|
+
require(isinstance(ring_files, list) and bool(ring_files), f"regression ring has no files: {ring_name}", errors)
|
|
942
|
+
for relative in ring_files if isinstance(ring_files, list) else []:
|
|
943
|
+
candidate = safe_path(project_root, relative, errors, "regression ring file")
|
|
944
|
+
require(candidate is not None and candidate.is_file(), f"regression ring file is not live: {relative}", errors)
|
|
945
|
+
require(relative in inventory_by_category.get("source", {}) or relative in inventory_by_category.get("tests", {}) or relative in inventory_by_category.get("contracts", {}) or relative in inventory_by_category.get("lockfiles", {}), f"regression ring file is not inventoried: {relative}", errors)
|
|
946
|
+
planned_ids = ring.get("planned_test_ids")
|
|
947
|
+
require(isinstance(planned_ids, list) and bool(planned_ids), f"regression ring has no planned tests: {ring_name}", errors)
|
|
948
|
+
for test_id in planned_ids if isinstance(planned_ids, list) else []:
|
|
949
|
+
test = test_map.get(test_id)
|
|
950
|
+
require(test is not None, f"regression ring maps unknown planned test: {test_id}", errors)
|
|
951
|
+
if test is not None:
|
|
952
|
+
require("regression-ring" in test.get("layers", []), f"regression ring test lacks regression-ring layer: {test_id}", errors)
|
|
953
|
+
return test_map
|
|
954
|
+
|
|
955
|
+
|
|
956
|
+
def validate_matrix_risks(
|
|
957
|
+
manifest: dict[str, Any],
|
|
958
|
+
matrix: dict[str, Any],
|
|
959
|
+
test_map: dict[str, dict[str, Any]],
|
|
960
|
+
executions: list[dict[str, Any]],
|
|
961
|
+
records_schema: dict[str, Any],
|
|
962
|
+
errors: list[str],
|
|
963
|
+
) -> None:
|
|
964
|
+
verdict = manifest.get("final_verdict")
|
|
965
|
+
execution_by_id = {item.get("execution_id"): item for item in executions if isinstance(item.get("execution_id"), str)}
|
|
966
|
+
selected_ranks = [
|
|
967
|
+
LAYER_ORDER[item["layer"]]
|
|
968
|
+
for item in executions
|
|
969
|
+
if item.get("selected_execution") is True and item.get("layer") in LAYER_ORDER
|
|
970
|
+
]
|
|
971
|
+
require(selected_ranks == sorted(selected_ranks), "selected executions are out of layer order", errors)
|
|
972
|
+
|
|
973
|
+
reliably_failed = any(
|
|
974
|
+
item.get("reliable") is True
|
|
975
|
+
and item.get("exit_code") != 0
|
|
976
|
+
and item.get("selected_execution") is True
|
|
977
|
+
for item in executions
|
|
978
|
+
)
|
|
979
|
+
if verdict == "TEST_FAIL":
|
|
980
|
+
require(reliably_failed, "TEST_FAIL lacks a reliable reproduced failing execution", errors)
|
|
981
|
+
|
|
982
|
+
for behavior in matrix.get("behaviors", []):
|
|
983
|
+
if not isinstance(behavior, dict):
|
|
984
|
+
continue
|
|
985
|
+
behavior_id = behavior.get("id")
|
|
986
|
+
risks = behavior.get("risks")
|
|
987
|
+
if not isinstance(risks, dict):
|
|
988
|
+
continue
|
|
989
|
+
require(risks.get("functional", {}).get("status") != "not_applicable", f"behavior functional risk cannot be N/A: {behavior_id}", errors)
|
|
990
|
+
for risk_name in RISK_KEYS:
|
|
991
|
+
cell = risks.get(risk_name)
|
|
992
|
+
if not isinstance(cell, dict):
|
|
993
|
+
continue
|
|
994
|
+
status = cell.get("status")
|
|
995
|
+
na = cell.get("not_applicable")
|
|
996
|
+
if status == "applicable":
|
|
997
|
+
require(na is None, f"applicable behavior risk cannot be N/A: {behavior_id}/{risk_name}", errors)
|
|
998
|
+
mapped_tests = cell.get("test_ids")
|
|
999
|
+
mapped_executions = cell.get("execution_ids")
|
|
1000
|
+
require(isinstance(mapped_tests, list) and bool(mapped_tests), f"behavior risk has no test mapping: {behavior_id}/{risk_name}", errors)
|
|
1001
|
+
require(isinstance(mapped_executions, list) and bool(mapped_executions), f"behavior risk has no receipt mapping: {behavior_id}/{risk_name}", errors)
|
|
1002
|
+
for test_id in mapped_tests if isinstance(mapped_tests, list) else []:
|
|
1003
|
+
require(test_id in test_map, f"behavior maps unknown planned test: {behavior_id}/{test_id}", errors)
|
|
1004
|
+
if test_id in test_map:
|
|
1005
|
+
require(behavior_id in test_map[test_id].get("behavior_ids", []), f"behavior test mapping is not bidirectional: {behavior_id}/{test_id}", errors)
|
|
1006
|
+
for execution_id in mapped_executions if isinstance(mapped_executions, list) else []:
|
|
1007
|
+
receipt = execution_by_id.get(execution_id)
|
|
1008
|
+
require(receipt is not None, f"behavior maps unknown execution: {behavior_id}/{execution_id}", errors)
|
|
1009
|
+
if receipt is not None:
|
|
1010
|
+
require(any(test_id in receipt.get("test_ids", []) for test_id in mapped_tests or []), f"behavior receipt did not run a mapped test: {behavior_id}/{risk_name}", errors)
|
|
1011
|
+
if verdict == "TEST_PASS":
|
|
1012
|
+
require(receipt.get("selected_execution") is True and receipt.get("reliable") is True and receipt.get("exit_code") == 0, f"TEST_PASS behavior maps a non-passing receipt: {behavior_id}/{risk_name}", errors)
|
|
1013
|
+
elif status == "not_applicable":
|
|
1014
|
+
require(cell.get("test_ids") == [] and cell.get("execution_ids") == [], f"N/A risk must not map tests or receipts: {behavior_id}/{risk_name}", errors)
|
|
1015
|
+
structured_na(na, records_schema, f"behavior {behavior_id} {risk_name} not_applicable", errors, risk_name)
|
|
1016
|
+
elif status == "unresolved":
|
|
1017
|
+
require(na is None, f"unresolved risk cannot also be N/A: {behavior_id}/{risk_name}", errors)
|
|
1018
|
+
if verdict == "TEST_PASS":
|
|
1019
|
+
errors.append(f"TEST_PASS has unresolved behavior risk: {behavior_id}/{risk_name}")
|
|
1020
|
+
|
|
1021
|
+
|
|
1022
|
+
def tree_sha256(root: Path, evidence_root: Path) -> str:
|
|
1023
|
+
excluded = evidence_root.resolve()
|
|
1024
|
+
entries: list[dict[str, str]] = []
|
|
1025
|
+
for candidate in sorted(root.rglob("*")):
|
|
1026
|
+
if not candidate.is_file() or ".git" in candidate.parts or "__pycache__" in candidate.parts:
|
|
1027
|
+
continue
|
|
1028
|
+
resolved = candidate.resolve()
|
|
1029
|
+
if resolved == excluded or excluded in resolved.parents:
|
|
1030
|
+
continue
|
|
1031
|
+
entries.append({"path": candidate.relative_to(root).as_posix(), "sha256": file_sha256(candidate)})
|
|
1032
|
+
return canonical_sha256(entries)
|
|
1033
|
+
|
|
1034
|
+
|
|
1035
|
+
def require_sha_or_null(value: Any, location: str, errors: list[str]) -> None:
|
|
1036
|
+
require(value is None or is_sha256(value), f"{location} must be a sha256 or null", errors)
|
|
1037
|
+
|
|
1038
|
+
|
|
1039
|
+
def validate_differential_proofs(
|
|
1040
|
+
root: Path,
|
|
1041
|
+
project_root: Path,
|
|
1042
|
+
manifest: dict[str, Any],
|
|
1043
|
+
matrix: dict[str, Any],
|
|
1044
|
+
proof: dict[str, Any],
|
|
1045
|
+
differential_requests: dict[str, dict[str, Any]],
|
|
1046
|
+
executions: list[dict[str, Any]],
|
|
1047
|
+
entry_map: dict[str, dict[str, Any]],
|
|
1048
|
+
records_schema: dict[str, Any],
|
|
1049
|
+
errors: list[str],
|
|
1050
|
+
) -> None:
|
|
1051
|
+
verdict = manifest.get("final_verdict")
|
|
1052
|
+
receipts = {item.get("execution_id"): item for item in executions if isinstance(item.get("execution_id"), str)}
|
|
1053
|
+
behavior_ids = {item.get("id") for item in matrix.get("behaviors", []) if isinstance(item, dict)}
|
|
1054
|
+
proof_by_behavior: dict[str, dict[str, Any]] = {}
|
|
1055
|
+
live_tree_hash = tree_sha256(project_root, root)
|
|
1056
|
+
proofs = proof.get("proofs")
|
|
1057
|
+
if not isinstance(proofs, list):
|
|
1058
|
+
return
|
|
1059
|
+
for item in proofs:
|
|
1060
|
+
if not isinstance(item, dict):
|
|
1061
|
+
continue
|
|
1062
|
+
behavior_id = item.get("behavior_id")
|
|
1063
|
+
require(behavior_id in behavior_ids, f"differential proof maps unknown behavior: {behavior_id}", errors)
|
|
1064
|
+
require(behavior_id not in proof_by_behavior, f"duplicate differential proof: {behavior_id}", errors)
|
|
1065
|
+
if isinstance(behavior_id, str):
|
|
1066
|
+
proof_by_behavior[behavior_id] = item
|
|
1067
|
+
classification = item.get("classification")
|
|
1068
|
+
if classification == "NOT_APPLICABLE":
|
|
1069
|
+
structured_na(item.get("not_applicable"), records_schema, f"differential proof {behavior_id} not_applicable", errors)
|
|
1070
|
+
for key in (
|
|
1071
|
+
"method", "differential_request_path", "differential_request_sha256", "baseline_commit",
|
|
1072
|
+
"current_tree_sha256", "baseline_execution_id", "mutation_execution_id", "current_execution_id",
|
|
1073
|
+
"project_tree_before_sha256", "project_tree_after_sha256", "isolation_tree_sha256",
|
|
1074
|
+
"mutation_apply_sha256", "mutation_restore_sha256",
|
|
1075
|
+
):
|
|
1076
|
+
require(item.get(key) is None, f"N/A differential proof must clear {key}: {behavior_id}", errors)
|
|
1077
|
+
require(item.get("failure_reason_matched") is False, f"N/A differential proof cannot claim a failure match: {behavior_id}", errors)
|
|
1078
|
+
continue
|
|
1079
|
+
|
|
1080
|
+
request_relative = item.get("differential_request_path")
|
|
1081
|
+
request = differential_requests.get(request_relative) if isinstance(request_relative, str) else None
|
|
1082
|
+
require(request is not None, f"differential proof has no matching request: {behavior_id}", errors)
|
|
1083
|
+
request_path = require_manifest_entry(request_relative, entry_map, root, errors, "differential request") if isinstance(request_relative, str) else None
|
|
1084
|
+
if request_path is not None:
|
|
1085
|
+
require(file_sha256(request_path) == item.get("differential_request_sha256"), f"differential request hash mismatch: {behavior_id}", errors)
|
|
1086
|
+
if not isinstance(request, dict):
|
|
1087
|
+
continue
|
|
1088
|
+
require(request.get("behavior_id") == behavior_id, f"differential request behavior mismatch: {behavior_id}", errors)
|
|
1089
|
+
method = item.get("method")
|
|
1090
|
+
require(method == request.get("method") and method in {"baseline", "controlled-mutation"}, f"differential method mismatch: {behavior_id}", errors)
|
|
1091
|
+
require(item.get("baseline_commit") == request.get("baseline_commit") == manifest.get("baseline_commit"), f"differential baseline commit mismatch: {behavior_id}", errors)
|
|
1092
|
+
require(item.get("current_tree_sha256") == request.get("current_tree_sha256") == live_tree_hash, f"differential current tree hash mismatch: {behavior_id}", errors)
|
|
1093
|
+
for key in (
|
|
1094
|
+
"differential_request_sha256", "current_tree_sha256", "project_tree_before_sha256",
|
|
1095
|
+
"project_tree_after_sha256", "isolation_tree_sha256", "mutation_apply_sha256", "mutation_restore_sha256",
|
|
1096
|
+
):
|
|
1097
|
+
require_sha_or_null(item.get(key), f"differential proof {behavior_id}.{key}", errors)
|
|
1098
|
+
require(item.get("project_tree_before_sha256") == live_tree_hash, f"differential project tree before mismatch: {behavior_id}", errors)
|
|
1099
|
+
require(item.get("project_tree_after_sha256") == live_tree_hash, f"differential project tree after mismatch: {behavior_id}", errors)
|
|
1100
|
+
require(item.get("cleanup_succeeded") is True, f"differential cleanup failed: {behavior_id}", errors)
|
|
1101
|
+
|
|
1102
|
+
current = receipts.get(item.get("current_execution_id"))
|
|
1103
|
+
require(current is not None, f"differential current receipt is missing: {behavior_id}", errors)
|
|
1104
|
+
failing_id = item.get("baseline_execution_id") if method == "baseline" else item.get("mutation_execution_id")
|
|
1105
|
+
failing = receipts.get(failing_id)
|
|
1106
|
+
require(failing is not None, f"differential failing receipt is missing: {behavior_id}", errors)
|
|
1107
|
+
if current is not None and failing is not None:
|
|
1108
|
+
require(current.get("request_sha256") == failing.get("request_sha256"), f"differential sides use different execution requests: {behavior_id}", errors)
|
|
1109
|
+
nested = request.get("execution_request")
|
|
1110
|
+
current_request_path = current.get("request_path")
|
|
1111
|
+
current_request = load_json(root / current_request_path) if isinstance(current_request_path, str) and safe_path(root, current_request_path, errors, "differential receipt request") else None
|
|
1112
|
+
require(current_request == nested, f"differential current receipt request does not match request: {behavior_id}", errors)
|
|
1113
|
+
require(current.get("selected_execution") is True and current.get("reliable") is True and current.get("exit_code") == 0, f"differential current side is not a reliable success: {behavior_id}", errors)
|
|
1114
|
+
require(failing.get("selected_execution") is False and failing.get("reliable") is True and failing.get("exit_code") != 0, f"differential failing side is not a reliable failure: {behavior_id}", errors)
|
|
1115
|
+
current_isolation = current.get("isolation")
|
|
1116
|
+
failing_isolation = failing.get("isolation")
|
|
1117
|
+
require(isinstance(current_isolation, dict) and current_isolation.get("kind") == "current", f"differential current receipt lacks current isolation: {behavior_id}", errors)
|
|
1118
|
+
require(isinstance(failing_isolation, dict) and failing_isolation.get("kind") == method, f"differential failing receipt isolation mismatch: {behavior_id}", errors)
|
|
1119
|
+
if isinstance(current_isolation, dict):
|
|
1120
|
+
require(current_isolation.get("tree_sha256") == item.get("isolation_tree_sha256") == live_tree_hash, f"differential current isolation hash mismatch: {behavior_id}", errors)
|
|
1121
|
+
if method == "baseline" and isinstance(failing_isolation, dict):
|
|
1122
|
+
require(failing_isolation.get("baseline_commit") == request.get("baseline_commit"), f"baseline receipt commit mismatch: {behavior_id}", errors)
|
|
1123
|
+
if method == "controlled-mutation" and isinstance(failing_isolation, dict):
|
|
1124
|
+
patch_relative = request.get("mutation_patch_path")
|
|
1125
|
+
patch_path = require_manifest_entry(patch_relative, entry_map, root, errors, "controlled mutation patch") if isinstance(patch_relative, str) else None
|
|
1126
|
+
require(patch_path is not None, f"controlled mutation patch is missing: {behavior_id}", errors)
|
|
1127
|
+
if patch_path is not None:
|
|
1128
|
+
patch_hash = file_sha256(patch_path)
|
|
1129
|
+
require(failing_isolation.get("patch_sha256") == patch_hash, f"controlled mutation receipt patch hash mismatch: {behavior_id}", errors)
|
|
1130
|
+
require(item.get("mutation_apply_sha256") == failing_isolation.get("tree_sha256"), f"controlled mutation apply hash mismatch: {behavior_id}", errors)
|
|
1131
|
+
require(item.get("mutation_restore_sha256") == item.get("isolation_tree_sha256"), f"controlled mutation restore hash mismatch: {behavior_id}", errors)
|
|
1132
|
+
if method == "baseline":
|
|
1133
|
+
require(item.get("mutation_execution_id") is None, f"baseline proof has mutation receipt: {behavior_id}", errors)
|
|
1134
|
+
require(request.get("mutation_patch_path") is None, f"baseline request has mutation patch: {behavior_id}", errors)
|
|
1135
|
+
require(item.get("mutation_apply_sha256") is None and item.get("mutation_restore_sha256") is None, f"baseline proof has mutation hashes: {behavior_id}", errors)
|
|
1136
|
+
elif method == "controlled-mutation":
|
|
1137
|
+
require(item.get("baseline_execution_id") is None, f"mutation proof has baseline receipt: {behavior_id}", errors)
|
|
1138
|
+
require(isinstance(request.get("mutation_patch_path"), str) and bool(request.get("mutation_patch_path")), f"mutation request lacks patch: {behavior_id}", errors)
|
|
1139
|
+
if classification == "PROVEN":
|
|
1140
|
+
require(item.get("failure_reason_matched") is True, f"differential failure reason was not matched: {behavior_id}", errors)
|
|
1141
|
+
if failing is not None:
|
|
1142
|
+
output = b""
|
|
1143
|
+
for key in ("stdout_path", "stderr_path"):
|
|
1144
|
+
relative = failing.get(key)
|
|
1145
|
+
path = safe_path(root, relative, errors, "differential raw output") if isinstance(relative, str) else None
|
|
1146
|
+
if path is not None and path.is_file():
|
|
1147
|
+
output += path.read_bytes()
|
|
1148
|
+
signals = request.get("expected_failure_signals")
|
|
1149
|
+
require(isinstance(signals, list) and any(isinstance(signal, str) and signal.encode() in output for signal in signals), f"differential expected failure signal is absent: {behavior_id}", errors)
|
|
1150
|
+
elif classification == "UNPROVEN" and verdict == "TEST_PASS":
|
|
1151
|
+
errors.append(f"TEST_PASS has unproven necessary behavior: {behavior_id}")
|
|
1152
|
+
|
|
1153
|
+
if verdict == "TEST_PASS":
|
|
1154
|
+
for behavior_id in behavior_ids:
|
|
1155
|
+
require(behavior_id in proof_by_behavior, f"behavior lacks differential proof: {behavior_id}", errors)
|
|
1156
|
+
|
|
1157
|
+
|
|
1158
|
+
def validate_infrastructure(
|
|
1159
|
+
manifest: dict[str, Any],
|
|
1160
|
+
infrastructure: dict[str, Any],
|
|
1161
|
+
executions: list[dict[str, Any]],
|
|
1162
|
+
entry_map: dict[str, dict[str, Any]],
|
|
1163
|
+
root: Path,
|
|
1164
|
+
errors: list[str],
|
|
1165
|
+
) -> None:
|
|
1166
|
+
verdict = manifest.get("final_verdict")
|
|
1167
|
+
require(infrastructure.get("cleanup_succeeded") is True or verdict == "TEST_BLOCKED", "infrastructure cleanup did not succeed", errors)
|
|
1168
|
+
external_targets = infrastructure.get("external_targets")
|
|
1169
|
+
validate_external_targets(external_targets, "infrastructure external_targets", errors)
|
|
1170
|
+
declared = {canonical_sha256(item) for item in external_targets if isinstance(item, dict)} if isinstance(external_targets, list) else set()
|
|
1171
|
+
for receipt in executions:
|
|
1172
|
+
for target in receipt.get("external_targets", []):
|
|
1173
|
+
if isinstance(target, dict):
|
|
1174
|
+
require(canonical_sha256(target) in declared, f"receipt external target is absent from infrastructure: {target.get('name')}", errors)
|
|
1175
|
+
snapshots = infrastructure.get("contract_snapshots")
|
|
1176
|
+
require(isinstance(snapshots, list), "contract snapshot provenance is missing", errors)
|
|
1177
|
+
for record in snapshots if isinstance(snapshots, list) else []:
|
|
1178
|
+
if not isinstance(record, dict):
|
|
1179
|
+
continue
|
|
1180
|
+
relative = record.get("path")
|
|
1181
|
+
require(isinstance(relative, str) and path_under(relative, "contracts"), "contract snapshot must be under contracts/", errors)
|
|
1182
|
+
path = require_manifest_entry(relative, entry_map, root, errors, "contract snapshot") if isinstance(relative, str) else None
|
|
1183
|
+
if path is not None:
|
|
1184
|
+
require(file_sha256(path) == record.get("sha256"), f"contract snapshot hash/provenance mismatch: {relative}", errors)
|
|
1185
|
+
|
|
1186
|
+
|
|
1187
|
+
def validate_lightweight(
|
|
1188
|
+
manifest: dict[str, Any],
|
|
1189
|
+
record: dict[str, Any],
|
|
1190
|
+
executions: list[dict[str, Any]],
|
|
1191
|
+
errors: list[str],
|
|
1192
|
+
) -> None:
|
|
1193
|
+
execution_by_id = {item.get("execution_id"): item for item in executions if isinstance(item.get("execution_id"), str)}
|
|
1194
|
+
checks = record.get("checks")
|
|
1195
|
+
require(isinstance(checks, list) and bool(checks), "lightweight deterministic checks are missing", errors)
|
|
1196
|
+
for check in checks if isinstance(checks, list) else []:
|
|
1197
|
+
if not isinstance(check, dict):
|
|
1198
|
+
continue
|
|
1199
|
+
receipt = execution_by_id.get(check.get("execution_id"))
|
|
1200
|
+
require(receipt is not None and receipt.get("selected_execution") is True and receipt.get("reliable") is True and receipt.get("exit_code") == 0, "lightweight check lacks a successful deterministic receipt", errors)
|
|
1201
|
+
if manifest.get("final_verdict") == "TEST_PASS":
|
|
1202
|
+
require(record.get("runtime_behavior_claimed") is False, "lightweight TEST_PASS must not claim runtime behavior", errors)
|
|
1203
|
+
|
|
1204
|
+
|
|
1205
|
+
def validate_verdict_and_attestation(
|
|
1206
|
+
root: Path,
|
|
1207
|
+
manifest: dict[str, Any],
|
|
1208
|
+
verdict_record: dict[str, Any],
|
|
1209
|
+
validation: dict[str, Any],
|
|
1210
|
+
report: str,
|
|
1211
|
+
entry_map: dict[str, dict[str, Any]],
|
|
1212
|
+
executions: list[dict[str, Any]],
|
|
1213
|
+
records_schema: dict[str, Any],
|
|
1214
|
+
errors: list[str],
|
|
1215
|
+
verify_attestation: bool,
|
|
1216
|
+
) -> None:
|
|
1217
|
+
validate_definition(verdict_record, "verdict", records_schema, "verdict", errors)
|
|
1218
|
+
verdict = manifest.get("final_verdict")
|
|
1219
|
+
require(verdict in VERDICTS and verdict_record.get("verdict") == verdict, "verdict record does not match manifest", errors)
|
|
1220
|
+
require(verdict_record.get("testing_domain_only") is True, "verdict exceeds testing-domain responsibility", errors)
|
|
1221
|
+
require(verdict_record.get("authorizes_commit_or_release") is False, "test verdict must not authorize commit or release", errors)
|
|
1222
|
+
require(verdict_record.get("real_environment_validated") is False, "code evidence must not claim real-environment validation", errors)
|
|
1223
|
+
require(verdict_record.get("repairs_business_defects") is False, "test skill must not repair business defects", errors)
|
|
1224
|
+
if verdict == "TEST_PASS":
|
|
1225
|
+
require(verdict_record.get("code_evidence_replayable") is True, "TEST_PASS must declare replayable code evidence", errors)
|
|
1226
|
+
require(not verdict_record.get("blockers"), "TEST_PASS contains blockers", errors)
|
|
1227
|
+
require(not verdict_record.get("reproduced_failures"), "TEST_PASS contains reproduced failures", errors)
|
|
1228
|
+
elif verdict == "TEST_BLOCKED":
|
|
1229
|
+
require(bool(verdict_record.get("blockers")), "TEST_BLOCKED lacks blocker reasons", errors)
|
|
1230
|
+
elif verdict == "TEST_FAIL":
|
|
1231
|
+
require(bool(verdict_record.get("reproduced_failures")), "TEST_FAIL lacks reproduced failures", errors)
|
|
1232
|
+
|
|
1233
|
+
if verify_attestation:
|
|
1234
|
+
validate_definition(validation, "validation", records_schema, "validation", errors)
|
|
1235
|
+
payload = sorted(
|
|
1236
|
+
({"path": path, "sha256": entry["sha256"]} for path, entry in entry_map.items() if path not in FINAL_RECORDS),
|
|
1237
|
+
key=lambda item: item["path"],
|
|
1238
|
+
)
|
|
1239
|
+
replayed_ids = [
|
|
1240
|
+
item.get("execution_id") for item in executions
|
|
1241
|
+
if isinstance(item.get("execution_id"), str) and isinstance(item.get("replay_of"), str)
|
|
1242
|
+
]
|
|
1243
|
+
require(validation.get("validator") == "validate_test_evidence.py", "validation attestation names wrong validator", errors)
|
|
1244
|
+
require(validation.get("validator_sha256") == file_sha256(Path(__file__).resolve()), "validation attestation validator hash mismatch", errors)
|
|
1245
|
+
require(validation.get("payload_sha256") == canonical_sha256(payload), "validation attestation payload hash mismatch", errors)
|
|
1246
|
+
require(validation.get("result") == "passed", "validation attestation is not passed", errors)
|
|
1247
|
+
require(validation.get("verdict") == verdict, "validation attestation verdict mismatch", errors)
|
|
1248
|
+
require(validation.get("semantics") == "integrity-and-protocol-validation-not-hostile-producer-proof", "validation attestation semantics exceed integrity/protocol validation", errors)
|
|
1249
|
+
require(validation.get("replay_receipt_ids") == replayed_ids, "validation replay receipt IDs do not match actual replay executions", errors)
|
|
1250
|
+
|
|
1251
|
+
fragments = [
|
|
1252
|
+
f"Verdict: {verdict}", f"Evidence ID: {manifest.get('evidence_id')}",
|
|
1253
|
+
"Derived from structured evidence", "sensitivity=project-controlled",
|
|
1254
|
+
"project owns access control, retention, and upload policy",
|
|
1255
|
+
"Mocked code-level evidence does not verify a real deployed environment",
|
|
1256
|
+
"legacy test-report interface is not supported",
|
|
1257
|
+
]
|
|
1258
|
+
for fragment in fragments:
|
|
1259
|
+
require(fragment in report, f"derived report missing authoritative evidence fragment: {fragment}", errors)
|
|
1260
|
+
|
|
1261
|
+
|
|
1262
|
+
def validate_evidence(
|
|
1263
|
+
root: Path,
|
|
1264
|
+
project_root: Path,
|
|
1265
|
+
ignored_hash_paths: set[str] | None = None,
|
|
1266
|
+
) -> list[str]:
|
|
1267
|
+
errors: list[str] = []
|
|
1268
|
+
try:
|
|
1269
|
+
manifest = load_json(root / "manifest.json")
|
|
1270
|
+
classification = load_json(root / "change-classification.json")
|
|
1271
|
+
scope = load_json(root / "scope.json")
|
|
1272
|
+
inventory = load_json(root / "target-inventory.json")
|
|
1273
|
+
verdict_record = load_json(root / "verdict.json")
|
|
1274
|
+
validation = load_json(root / "validation.json")
|
|
1275
|
+
report = (root / "test-report.md").read_text(encoding="utf-8")
|
|
1276
|
+
manifest_schema = load_schema("evidence-manifest.schema.json")
|
|
1277
|
+
records_schema = load_schema("authoritative-records.schema.json")
|
|
1278
|
+
matrix_schema = load_schema("behavior-risk-matrix.schema.json")
|
|
1279
|
+
except (EvidenceError, FileNotFoundError) as exc:
|
|
1280
|
+
return [str(exc)]
|
|
1281
|
+
if not all(isinstance(item, dict) for item in (manifest, classification, scope, inventory, verdict_record, validation)):
|
|
1282
|
+
return ["structured evidence records must be JSON objects"]
|
|
1283
|
+
|
|
1284
|
+
entry_map = validate_manifest(root, manifest, manifest_schema, records_schema, errors, ignored_hash_paths)
|
|
1285
|
+
change_class = manifest.get("change_class")
|
|
1286
|
+
required = COMMON_RECORDS | (BEHAVIOR_RECORDS if change_class == "behavior" else LIGHTWEIGHT_RECORDS)
|
|
1287
|
+
for relative in required:
|
|
1288
|
+
require(relative in entry_map, f"required evidence record not hashed: {relative}", errors)
|
|
1289
|
+
validate_definition(classification, "changeClassification", records_schema, "change-classification", errors)
|
|
1290
|
+
validate_definition(scope, "scope", records_schema, "scope", errors)
|
|
1291
|
+
validate_definition(inventory, "targetInventory", records_schema, "target-inventory", errors)
|
|
1292
|
+
require(classification.get("classification") == change_class, "change classification does not match manifest", errors)
|
|
1293
|
+
require(classification.get("full_protocol_required") is (change_class == "behavior"), "change classification protocol decision is inconsistent", errors)
|
|
1294
|
+
|
|
1295
|
+
plan: dict[str, Any] | None = None
|
|
1296
|
+
if change_class == "behavior":
|
|
1297
|
+
try:
|
|
1298
|
+
plan = load_json(root / "test-plan.json")
|
|
1299
|
+
except EvidenceError as exc:
|
|
1300
|
+
errors.append(str(exc))
|
|
1301
|
+
if isinstance(plan, dict):
|
|
1302
|
+
validate_definition(plan, "testPlan", records_schema, "test-plan", errors)
|
|
1303
|
+
else:
|
|
1304
|
+
errors.append("test-plan.json must be an object")
|
|
1305
|
+
plan = None
|
|
1306
|
+
|
|
1307
|
+
inventory_by_category, _ = validate_inventory(root, project_root, manifest, inventory, scope, plan, entry_map, errors)
|
|
1308
|
+
require(manifest.get("evidence_id_inputs", {}).get("scope_sha256") == canonical_sha256(scope), "scope identity hash mismatch", errors)
|
|
1309
|
+
validate_scope(scope, inventory, inventory_by_category, project_root, root, errors)
|
|
1310
|
+
execution_requests, differential_requests, executions = validate_requests_and_receipts(root, entry_map, records_schema, errors)
|
|
1311
|
+
|
|
1312
|
+
if change_class == "behavior":
|
|
1313
|
+
try:
|
|
1314
|
+
matrix = load_json(root / "behavior-risk-matrix.json")
|
|
1315
|
+
infrastructure = load_json(root / "infrastructure-changes.json")
|
|
1316
|
+
proof = load_json(root / "differential-proof.json")
|
|
1317
|
+
except EvidenceError as exc:
|
|
1318
|
+
errors.append(str(exc))
|
|
1319
|
+
matrix = infrastructure = proof = None
|
|
1320
|
+
if all(isinstance(item, dict) for item in (matrix, infrastructure, proof, plan)):
|
|
1321
|
+
validate_schema(matrix, matrix_schema, matrix_schema, "behavior-risk-matrix", errors)
|
|
1322
|
+
validate_definition(infrastructure, "infrastructureChanges", records_schema, "infrastructure-changes", errors)
|
|
1323
|
+
validate_definition(proof, "differentialProof", records_schema, "differential-proof", errors)
|
|
1324
|
+
stages = {item.get("name"): item for item in manifest.get("stages", []) if isinstance(item, dict)}
|
|
1325
|
+
for stage_name in CORE_BEHAVIOR_STAGES:
|
|
1326
|
+
require(stages.get(stage_name, {}).get("status") == "complete", f"behavior protocol stage cannot be N/A: {stage_name}", errors)
|
|
1327
|
+
test_map = validate_plan_and_scope_bindings(
|
|
1328
|
+
root, project_root, manifest, scope, matrix, plan, inventory_by_category, entry_map,
|
|
1329
|
+
executions, records_schema, errors,
|
|
1330
|
+
)
|
|
1331
|
+
validate_matrix_risks(manifest, matrix, test_map, executions, records_schema, errors)
|
|
1332
|
+
validate_differential_proofs(
|
|
1333
|
+
root, project_root, manifest, matrix, proof, differential_requests, executions,
|
|
1334
|
+
entry_map, records_schema, errors,
|
|
1335
|
+
)
|
|
1336
|
+
validate_infrastructure(manifest, infrastructure, executions, entry_map, root, errors)
|
|
1337
|
+
else:
|
|
1338
|
+
errors.append("behavior evidence records must be JSON objects")
|
|
1339
|
+
elif change_class == "lightweight":
|
|
1340
|
+
try:
|
|
1341
|
+
lightweight = load_json(root / "lightweight-verification.json")
|
|
1342
|
+
except EvidenceError as exc:
|
|
1343
|
+
errors.append(str(exc))
|
|
1344
|
+
lightweight = None
|
|
1345
|
+
if isinstance(lightweight, dict):
|
|
1346
|
+
validate_definition(lightweight, "lightweightVerification", records_schema, "lightweight-verification", errors)
|
|
1347
|
+
validate_lightweight(manifest, lightweight, executions, errors)
|
|
1348
|
+
else:
|
|
1349
|
+
errors.append("lightweight-verification.json must be an object")
|
|
1350
|
+
else:
|
|
1351
|
+
errors.append("invalid change_class")
|
|
1352
|
+
|
|
1353
|
+
validate_verdict_and_attestation(
|
|
1354
|
+
root, manifest, verdict_record, validation, report, entry_map, executions, records_schema,
|
|
1355
|
+
errors, verify_attestation=not bool(ignored_hash_paths),
|
|
1356
|
+
)
|
|
1357
|
+
return errors
|
|
1358
|
+
|
|
1359
|
+
|
|
1360
|
+
def main() -> int:
|
|
1361
|
+
parser = argparse.ArgumentParser(description="Validate a PrizmKit test evidence package")
|
|
1362
|
+
parser.add_argument("--evidence-dir", required=True, help="Path to .prizmkit/test/evidence/<evidence-id>")
|
|
1363
|
+
parser.add_argument("--project-root", help="Target project root; defaults to four parents above evidence directory")
|
|
1364
|
+
parser.add_argument("--attest", action="store_true", help="Validate payload, then atomically refresh the validator-owned attestation and final hashes")
|
|
1365
|
+
args = parser.parse_args()
|
|
1366
|
+
root = Path(args.evidence_dir).resolve()
|
|
1367
|
+
if not root.is_dir():
|
|
1368
|
+
print(f"Test evidence validation: FAIL\n- evidence directory not found: {root}")
|
|
1369
|
+
return 1
|
|
1370
|
+
try:
|
|
1371
|
+
project_root = Path(args.project_root).resolve() if args.project_root else root.parents[3]
|
|
1372
|
+
except IndexError:
|
|
1373
|
+
print("Test evidence validation: FAIL\n- cannot infer project root from evidence directory")
|
|
1374
|
+
return 1
|
|
1375
|
+
errors = validate_evidence(root, project_root, FINAL_RECORDS if args.attest else None)
|
|
1376
|
+
if errors:
|
|
1377
|
+
print("Test evidence validation: FAIL")
|
|
1378
|
+
for error in errors:
|
|
1379
|
+
print(f"- {error}")
|
|
1380
|
+
return 1
|
|
1381
|
+
|
|
1382
|
+
manifest = load_json(root / "manifest.json")
|
|
1383
|
+
if args.attest:
|
|
1384
|
+
executions = load_json(root / "executions.json")
|
|
1385
|
+
replayed_ids = [
|
|
1386
|
+
item.get("execution_id") for item in executions
|
|
1387
|
+
if isinstance(item, dict)
|
|
1388
|
+
and isinstance(item.get("execution_id"), str)
|
|
1389
|
+
and isinstance(item.get("replay_of"), str)
|
|
1390
|
+
] if isinstance(executions, list) else []
|
|
1391
|
+
payload = sorted(
|
|
1392
|
+
(
|
|
1393
|
+
{"path": entry["path"], "sha256": entry["sha256"]}
|
|
1394
|
+
for entry in manifest["files"] if entry["path"] not in FINAL_RECORDS
|
|
1395
|
+
),
|
|
1396
|
+
key=lambda item: item["path"],
|
|
1397
|
+
)
|
|
1398
|
+
validation = {
|
|
1399
|
+
"validator": "validate_test_evidence.py",
|
|
1400
|
+
"validator_sha256": file_sha256(Path(__file__).resolve()),
|
|
1401
|
+
"payload_sha256": canonical_sha256(payload),
|
|
1402
|
+
"result": "passed",
|
|
1403
|
+
"verdict": manifest["final_verdict"],
|
|
1404
|
+
"semantics": "integrity-and-protocol-validation-not-hostile-producer-proof",
|
|
1405
|
+
"replay_receipt_ids": replayed_ids,
|
|
1406
|
+
}
|
|
1407
|
+
atomic_write_json(root / "validation.json", validation)
|
|
1408
|
+
for entry in manifest["files"]:
|
|
1409
|
+
if entry["path"] in FINAL_RECORDS:
|
|
1410
|
+
entry["sha256"] = file_sha256(root / entry["path"])
|
|
1411
|
+
atomic_write_json(root / "manifest.json", manifest)
|
|
1412
|
+
errors = validate_evidence(root, project_root)
|
|
1413
|
+
if errors:
|
|
1414
|
+
print("Test evidence validation: FAIL")
|
|
1415
|
+
for error in errors:
|
|
1416
|
+
print(f"- {error}")
|
|
1417
|
+
return 1
|
|
1418
|
+
|
|
1419
|
+
print("Test evidence validation: PASS")
|
|
1420
|
+
print(f"Evidence ID: {manifest['evidence_id']}")
|
|
1421
|
+
print(f"Verdict: {manifest['final_verdict']}")
|
|
1422
|
+
return 0
|
|
1423
|
+
|
|
1424
|
+
|
|
1425
|
+
if __name__ == "__main__":
|
|
1426
|
+
sys.exit(main())
|