prizmkit 1.1.137 → 1.1.138
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bundled/VERSION.json +3 -3
- package/bundled/dev-pipeline/prizmkit_runtime/runner_models.py +0 -1
- package/bundled/dev-pipeline/prizmkit_runtime/runner_prompts.py +0 -1
- package/bundled/dev-pipeline/prizmkit_runtime/runners.py +3 -5
- package/bundled/dev-pipeline/scripts/generate-bootstrap-prompt.py +24 -39
- package/bundled/dev-pipeline/scripts/generate-bugfix-prompt.py +3 -5
- package/bundled/dev-pipeline/scripts/generate-recovery-prompt.py +4 -5
- package/bundled/dev-pipeline/scripts/generate-refactor-prompt.py +3 -5
- package/bundled/dev-pipeline/scripts/prompt_framework.py +0 -5
- package/bundled/dev-pipeline/templates/bootstrap-prompt.md +3 -3
- package/bundled/dev-pipeline/templates/bootstrap-tier3.md +24 -26
- package/bundled/dev-pipeline/templates/bugfix-bootstrap-prompt.md +5 -4
- package/bundled/dev-pipeline/templates/feature-list-schema.json +1 -1
- package/bundled/dev-pipeline/templates/refactor-bootstrap-prompt.md +3 -2
- package/bundled/dev-pipeline/templates/sections/bugfix-mission.md +1 -1
- package/bundled/dev-pipeline/templates/sections/bugfix-phase-review.md +2 -10
- package/bundled/dev-pipeline/templates/sections/bugfix-reminders.md +2 -1
- package/bundled/dev-pipeline/templates/sections/context-budget-rules.md +2 -2
- package/bundled/dev-pipeline/templates/sections/phase-implement-agent.md +5 -5
- package/bundled/dev-pipeline/templates/sections/phase-implement-full.md +5 -5
- package/bundled/dev-pipeline/templates/sections/phase-review-agent.md +2 -12
- package/bundled/dev-pipeline/templates/sections/phase-review-full.md +2 -10
- package/bundled/dev-pipeline/templates/sections/phase-specify-plan-full.md +1 -1
- package/bundled/dev-pipeline/templates/sections/refactor-mission.md +1 -1
- package/bundled/dev-pipeline/templates/sections/refactor-phase-implement.md +2 -2
- package/bundled/dev-pipeline/templates/sections/refactor-phase-review.md +2 -10
- package/bundled/dev-pipeline/templates/sections/refactor-reminders.md +2 -1
- package/bundled/dev-pipeline/templates/sections/subagent-timeout-recovery.md +5 -4
- package/bundled/dev-pipeline/tests/test_generate_bootstrap_prompt.py +10 -16
- package/bundled/dev-pipeline/tests/test_generate_bugfix_prompt.py +3 -1
- package/bundled/dev-pipeline/tests/test_python_runner_parity.py +1 -6
- package/bundled/dev-pipeline/tests/test_unified_cli.py +3 -5
- package/bundled/skills/_metadata.json +2 -2
- package/bundled/skills/prizmkit-test/SKILL.md +17 -20
- package/bundled/skills/prizmkit-test/assets/authoritative-records.schema.json +5 -73
- package/bundled/skills/prizmkit-test/assets/evidence-package-template.json +0 -2
- package/bundled/skills/prizmkit-test/references/boundary-coverage-protocol.md +2 -2
- package/bundled/skills/prizmkit-test/references/evidence-protocol.md +10 -10
- package/bundled/skills/prizmkit-test/references/evidence-request-protocol.md +8 -16
- package/bundled/skills/prizmkit-test/references/examples.md +3 -5
- package/bundled/skills/prizmkit-test/references/test-generation-steps.md +6 -8
- package/bundled/skills/prizmkit-test/references/test-report-template.md +4 -7
- package/bundled/skills/prizmkit-test/references/trusted-evidence-execution.md +5 -7
- package/bundled/skills/prizmkit-test/scripts/build_test_evidence.py +70 -196
- package/bundled/skills/prizmkit-test/scripts/validate_test_evidence.py +251 -167
- package/package.json +1 -1
|
@@ -14,6 +14,7 @@ import json
|
|
|
14
14
|
import math
|
|
15
15
|
import os
|
|
16
16
|
import re
|
|
17
|
+
import subprocess
|
|
17
18
|
import sys
|
|
18
19
|
import tempfile
|
|
19
20
|
from pathlib import Path
|
|
@@ -46,12 +47,22 @@ COMMON_RECORDS = {
|
|
|
46
47
|
}
|
|
47
48
|
BEHAVIOR_RECORDS = {
|
|
48
49
|
"behavior-risk-matrix.json", "test-plan.json",
|
|
49
|
-
"infrastructure-changes.json",
|
|
50
|
+
"infrastructure-changes.json",
|
|
50
51
|
}
|
|
51
52
|
LIGHTWEIGHT_RECORDS = {"lightweight-verification.json"}
|
|
52
53
|
SHA256_RE = re.compile(r"^[a-f0-9]{64}$")
|
|
53
54
|
UUID_RE = re.compile(r"^[a-f0-9-]{36}$")
|
|
54
55
|
BLOCKED_EXTERNAL_CLASSES = {"production", "unknown"}
|
|
56
|
+
INVENTORY_RESERVED_ROOTS = (
|
|
57
|
+
".prizmkit/state",
|
|
58
|
+
".prizmkit/test/evidence",
|
|
59
|
+
".prizmkit/" + "dev-" + "pipeline",
|
|
60
|
+
".claude/worktrees",
|
|
61
|
+
".agents/worktrees",
|
|
62
|
+
".codebuddy/worktrees",
|
|
63
|
+
".codex/worktrees",
|
|
64
|
+
)
|
|
65
|
+
_IGNORED_DIR_NAMES = {".git", "__pycache__", ".pytest_cache", ".mypy_cache"}
|
|
55
66
|
RISK_CONFLICT_PATTERNS = {
|
|
56
67
|
"permission": re.compile(r"auth|tenant|role|permission|access|acl|rbac", re.I),
|
|
57
68
|
"concurrency": re.compile(r"lock|shared[ _-]?state|worker|queue|concurr|parallel|race", re.I),
|
|
@@ -161,6 +172,152 @@ def path_below_or_equal(path: str, root: str) -> bool:
|
|
|
161
172
|
return normalized_path == normalized_root or normalized_path.startswith(f"{normalized_root}/")
|
|
162
173
|
|
|
163
174
|
|
|
175
|
+
def normalize_project_relative(value: str) -> str:
|
|
176
|
+
normalized = Path(value).as_posix()
|
|
177
|
+
if normalized == "." or normalized.startswith("../") or "/../" in normalized:
|
|
178
|
+
raise EvidenceError(f"Git returned a non-project-relative path: {value}")
|
|
179
|
+
return normalized
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
def is_reserved_inventory_path(relative: str) -> bool:
|
|
183
|
+
path = Path(relative)
|
|
184
|
+
if any(part in _IGNORED_DIR_NAMES for part in path.parts):
|
|
185
|
+
return True
|
|
186
|
+
return any(path == Path(prefix) or Path(prefix) in path.parents for prefix in INVENTORY_RESERVED_ROOTS)
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
def project_git(project_root: Path, *args: str) -> subprocess.CompletedProcess[bytes]:
|
|
190
|
+
result = subprocess.run(
|
|
191
|
+
["git", "-C", str(project_root), *args],
|
|
192
|
+
capture_output=True,
|
|
193
|
+
check=False,
|
|
194
|
+
)
|
|
195
|
+
if result.returncode != 0:
|
|
196
|
+
raise EvidenceError(f"git {' '.join(args)} failed: {result.stderr.decode(errors='replace')}")
|
|
197
|
+
return result
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
def git_status_entries(project_root: Path) -> list[dict[str, Any]]:
|
|
201
|
+
raw = project_git(
|
|
202
|
+
project_root, "status", "--porcelain=v1", "-z", "--untracked-files=all",
|
|
203
|
+
).stdout
|
|
204
|
+
entries: list[dict[str, Any]] = []
|
|
205
|
+
tokens = raw.split(b"\0")
|
|
206
|
+
index = 0
|
|
207
|
+
while index < len(tokens):
|
|
208
|
+
token = tokens[index]
|
|
209
|
+
index += 1
|
|
210
|
+
if not token:
|
|
211
|
+
continue
|
|
212
|
+
status = token[:2].decode(errors="replace")
|
|
213
|
+
relative = normalize_project_relative(token[3:].decode(errors="replace"))
|
|
214
|
+
old_path = None
|
|
215
|
+
if "R" in status or "C" in status:
|
|
216
|
+
if index >= len(tokens):
|
|
217
|
+
raise EvidenceError("git status returned an incomplete rename/copy record")
|
|
218
|
+
old_path = tokens[index].decode(errors="replace")
|
|
219
|
+
index += 1
|
|
220
|
+
if not is_reserved_inventory_path(relative):
|
|
221
|
+
entries.append({"path": relative, "status": status, "old_path": old_path})
|
|
222
|
+
if old_path is not None:
|
|
223
|
+
old_relative = normalize_project_relative(old_path)
|
|
224
|
+
if not is_reserved_inventory_path(old_relative):
|
|
225
|
+
entries.append({"path": old_relative, "status": status, "old_path": old_relative, "role": "source"})
|
|
226
|
+
return sorted(entries, key=lambda item: (item["path"], item.get("old_path") or ""))
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
def git_diff_entries(project_root: Path, baseline: str) -> list[dict[str, Any]]:
|
|
230
|
+
raw = project_git(
|
|
231
|
+
project_root, "diff", "--name-status", "-z", "--find-renames", "--find-copies",
|
|
232
|
+
baseline, "--", ".",
|
|
233
|
+
":!.prizmkit/state/**", ":!.prizmkit/test/evidence/**", ":!.prizmkit/" + "dev-" + "pipeline/**",
|
|
234
|
+
":!.claude/worktrees/**", ":!.agents/worktrees/**", ":!.codebuddy/worktrees/**", ":!.codex/worktrees/**",
|
|
235
|
+
).stdout
|
|
236
|
+
tokens = raw.split(b"\0")
|
|
237
|
+
entries: list[dict[str, Any]] = []
|
|
238
|
+
index = 0
|
|
239
|
+
while index < len(tokens):
|
|
240
|
+
status_token = tokens[index].decode(errors="replace")
|
|
241
|
+
index += 1
|
|
242
|
+
if not status_token:
|
|
243
|
+
continue
|
|
244
|
+
if index >= len(tokens):
|
|
245
|
+
raise EvidenceError("git diff returned an incomplete changed-file record")
|
|
246
|
+
first = normalize_project_relative(tokens[index].decode(errors="replace"))
|
|
247
|
+
index += 1
|
|
248
|
+
status = status_token[0]
|
|
249
|
+
old_path = None
|
|
250
|
+
if status in {"R", "C"}:
|
|
251
|
+
if index >= len(tokens):
|
|
252
|
+
raise EvidenceError("git diff returned an incomplete rename/copy record")
|
|
253
|
+
old_path = first
|
|
254
|
+
first = normalize_project_relative(tokens[index].decode(errors="replace"))
|
|
255
|
+
index += 1
|
|
256
|
+
if not is_reserved_inventory_path(first):
|
|
257
|
+
entries.append({"path": first, "status": status, "old_path": old_path})
|
|
258
|
+
if old_path and not is_reserved_inventory_path(old_path):
|
|
259
|
+
entries.append({"path": old_path, "status": status, "old_path": old_path, "role": "source"})
|
|
260
|
+
return entries
|
|
261
|
+
|
|
262
|
+
|
|
263
|
+
def merge_changed_file_entries(
|
|
264
|
+
baseline_entries: list[dict[str, Any]],
|
|
265
|
+
working_entries: list[dict[str, Any]],
|
|
266
|
+
) -> list[dict[str, Any]]:
|
|
267
|
+
merged: dict[tuple[str, str | None, str | None], dict[str, Any]] = {}
|
|
268
|
+
for entry in baseline_entries + working_entries:
|
|
269
|
+
key = (entry["path"], entry.get("old_path"), entry.get("role"))
|
|
270
|
+
existing = merged.get(key)
|
|
271
|
+
if existing is None:
|
|
272
|
+
merged[key] = dict(entry)
|
|
273
|
+
continue
|
|
274
|
+
statuses = {existing.get("status", "").strip(), entry.get("status", "").strip()}
|
|
275
|
+
if "D" in statuses:
|
|
276
|
+
existing["status"] = "D"
|
|
277
|
+
elif "R" in statuses:
|
|
278
|
+
existing["status"] = "R"
|
|
279
|
+
elif "C" in statuses:
|
|
280
|
+
existing["status"] = "C"
|
|
281
|
+
elif statuses:
|
|
282
|
+
existing["status"] = "M"
|
|
283
|
+
return sorted(merged.values(), key=lambda item: (item["path"], item.get("old_path") or "", item.get("role") or ""))
|
|
284
|
+
|
|
285
|
+
|
|
286
|
+
def canonical_change_patch(project_root: Path, baseline: str) -> tuple[bytes, list[dict[str, Any]]]:
|
|
287
|
+
status_entries = merge_changed_file_entries(
|
|
288
|
+
git_diff_entries(project_root, baseline),
|
|
289
|
+
git_status_entries(project_root),
|
|
290
|
+
)
|
|
291
|
+
tracked = project_git(
|
|
292
|
+
project_root, "diff", "--binary", "--full-index", "--find-renames", "--find-copies",
|
|
293
|
+
baseline, "--", ".",
|
|
294
|
+
":!.prizmkit/state/**", ":!.prizmkit/test/evidence/**", ":!.prizmkit/" + "dev-" + "pipeline/**",
|
|
295
|
+
":!.claude/worktrees/**", ":!.agents/worktrees/**", ":!.codebuddy/worktrees/**", ":!.codex/worktrees/**",
|
|
296
|
+
).stdout
|
|
297
|
+
chunks = [tracked]
|
|
298
|
+
untracked = [
|
|
299
|
+
item["path"] for item in status_entries
|
|
300
|
+
if item["status"].strip() == "??" and item.get("old_path") is None
|
|
301
|
+
]
|
|
302
|
+
for relative in sorted(untracked):
|
|
303
|
+
candidate = (project_root / relative).resolve()
|
|
304
|
+
try:
|
|
305
|
+
candidate.relative_to(project_root.resolve())
|
|
306
|
+
except ValueError as exc:
|
|
307
|
+
raise EvidenceError(f"untracked path escapes project root: {relative}") from exc
|
|
308
|
+
if candidate.is_file():
|
|
309
|
+
result = subprocess.run(
|
|
310
|
+
["git", "diff", "--no-index", "--binary", "/dev/null", "--", relative],
|
|
311
|
+
cwd=project_root,
|
|
312
|
+
capture_output=True,
|
|
313
|
+
check=False,
|
|
314
|
+
)
|
|
315
|
+
if result.returncode not in (0, 1):
|
|
316
|
+
raise EvidenceError(f"cannot capture untracked file: {relative}")
|
|
317
|
+
chunks.append(result.stdout)
|
|
318
|
+
return b"".join(chunk for chunk in chunks if chunk), status_entries
|
|
319
|
+
|
|
320
|
+
|
|
164
321
|
def resolve_ref(schema: dict[str, Any], root_schema: dict[str, Any]) -> dict[str, Any]:
|
|
165
322
|
reference = schema.get("$ref")
|
|
166
323
|
if not reference:
|
|
@@ -225,6 +382,11 @@ def validate_schema(
|
|
|
225
382
|
if not branch_errors:
|
|
226
383
|
matching += 1
|
|
227
384
|
require(matching == 1, f"{location}: value must match exactly one schema branch", errors)
|
|
385
|
+
prohibited = schema.get("not")
|
|
386
|
+
if isinstance(prohibited, dict):
|
|
387
|
+
prohibited_errors: list[str] = []
|
|
388
|
+
validate_schema(value, prohibited, root_schema, location, prohibited_errors)
|
|
389
|
+
require(bool(prohibited_errors), f"{location}: value matches a prohibited schema branch", errors)
|
|
228
390
|
|
|
229
391
|
for child in schema.get("allOf", []):
|
|
230
392
|
if isinstance(child, dict):
|
|
@@ -348,31 +510,6 @@ def require_manifest_entry(
|
|
|
348
510
|
return path
|
|
349
511
|
|
|
350
512
|
|
|
351
|
-
def validate_differential_na(
|
|
352
|
-
value: Any,
|
|
353
|
-
location: str,
|
|
354
|
-
errors: list[str],
|
|
355
|
-
) -> None:
|
|
356
|
-
"""Validate the differential-only N/A shape without weakening risk-cell N/A."""
|
|
357
|
-
if not isinstance(value, dict):
|
|
358
|
-
errors.append(f"{location}: differential N/A must be an object")
|
|
359
|
-
return
|
|
360
|
-
required = {"reason", "rationale", "evidence", "considered_signals", "conflicts"}
|
|
361
|
-
require(set(value) == required, f"{location}: differential N/A fields are incomplete", errors)
|
|
362
|
-
require(value.get("reason") in {"new-behavior", "textual-contract"}, f"{location}: invalid differential N/A reason", errors)
|
|
363
|
-
rationale = value.get("rationale")
|
|
364
|
-
require(isinstance(rationale, str) and len(rationale.strip()) >= 16, f"{location}: differential N/A rationale is too short", errors)
|
|
365
|
-
if value.get("reason") == "new-behavior" and isinstance(rationale, str):
|
|
366
|
-
require("no stable pre-change observable" in rationale.lower(), f"{location}: new behavior N/A lacks baseline rationale", errors)
|
|
367
|
-
if value.get("reason") == "textual-contract" and isinstance(rationale, str):
|
|
368
|
-
require("textual" in rationale.lower(), f"{location}: textual contract N/A lacks textual rationale", errors)
|
|
369
|
-
evidence = value.get("evidence")
|
|
370
|
-
require(isinstance(evidence, list) and bool(evidence) and all(isinstance(item, str) and item.strip() for item in evidence), f"{location}: differential N/A needs evidence strings", errors)
|
|
371
|
-
signals = value.get("considered_signals")
|
|
372
|
-
require(isinstance(signals, list) and bool(signals) and all(isinstance(item, str) and item.strip() for item in signals), f"{location}: differential N/A needs considered signals", errors)
|
|
373
|
-
require(isinstance(value.get("conflicts"), list), f"{location}: differential N/A conflicts must be an array", errors)
|
|
374
|
-
|
|
375
|
-
|
|
376
513
|
def structured_na(
|
|
377
514
|
value: Any,
|
|
378
515
|
records_schema: dict[str, Any],
|
|
@@ -734,9 +871,8 @@ def validate_requests_and_receipts(
|
|
|
734
871
|
entry_map: dict[str, dict[str, Any]],
|
|
735
872
|
records_schema: dict[str, Any],
|
|
736
873
|
errors: list[str],
|
|
737
|
-
) -> tuple[dict[str, dict[str, Any]],
|
|
874
|
+
) -> tuple[dict[str, dict[str, Any]], list[dict[str, Any]]]:
|
|
738
875
|
execution_requests: dict[str, dict[str, Any]] = {}
|
|
739
|
-
differential_requests: dict[str, dict[str, Any]] = {}
|
|
740
876
|
requests_root = root / "requests"
|
|
741
877
|
if requests_root.exists():
|
|
742
878
|
for path in sorted(requests_root.rglob("*.json")):
|
|
@@ -752,19 +888,6 @@ def validate_requests_and_receipts(
|
|
|
752
888
|
continue
|
|
753
889
|
if set(request) == {"request_version", "layers", "tests"}:
|
|
754
890
|
validate_definition(request, "testPreparationRequest", records_schema, relative, errors)
|
|
755
|
-
elif request.get("method") == "not-applicable":
|
|
756
|
-
validate_definition(request, "differentialRequest", records_schema, relative, errors)
|
|
757
|
-
not_applicable = request.get("not_applicable")
|
|
758
|
-
if isinstance(not_applicable, dict):
|
|
759
|
-
validate_differential_na(not_applicable, f"{relative}.not_applicable", errors)
|
|
760
|
-
differential_requests[relative] = request
|
|
761
|
-
elif "execution_request" in request:
|
|
762
|
-
validate_definition(request, "differentialRequest", records_schema, relative, errors)
|
|
763
|
-
nested = request.get("execution_request")
|
|
764
|
-
if isinstance(nested, dict):
|
|
765
|
-
validate_definition(nested, "executionRequest", records_schema, f"{relative}.execution_request", errors)
|
|
766
|
-
validate_external_targets(nested.get("external_targets"), f"{relative}.execution_request.external_targets", errors)
|
|
767
|
-
differential_requests[relative] = request
|
|
768
891
|
elif "command" in request and "layer" in request:
|
|
769
892
|
validate_definition(request, "executionRequest", records_schema, relative, errors)
|
|
770
893
|
validate_execution_request_semantics(request, relative, errors)
|
|
@@ -798,10 +921,10 @@ def validate_requests_and_receipts(
|
|
|
798
921
|
executions = load_json(executions_path)
|
|
799
922
|
except EvidenceError as exc:
|
|
800
923
|
errors.append(str(exc))
|
|
801
|
-
return execution_requests,
|
|
924
|
+
return execution_requests, []
|
|
802
925
|
validate_definition(executions, "executionLog", records_schema, "executions", errors)
|
|
803
926
|
if not isinstance(executions, list):
|
|
804
|
-
return execution_requests,
|
|
927
|
+
return execution_requests, []
|
|
805
928
|
|
|
806
929
|
receipt_by_id: dict[str, dict[str, Any]] = {}
|
|
807
930
|
seen_receipt_paths: set[str] = set()
|
|
@@ -857,6 +980,12 @@ def validate_requests_and_receipts(
|
|
|
857
980
|
require("exit_code" in probe or "error" in probe, f"{location}: tool probe has no result: {probe_name}", errors)
|
|
858
981
|
|
|
859
982
|
validate_execution_request_semantics(receipt, location, errors, receipt=True)
|
|
983
|
+
isolation = receipt.get("isolation")
|
|
984
|
+
require(
|
|
985
|
+
isinstance(isolation, dict) and isolation.get("kind") == "project-direct",
|
|
986
|
+
f"{location}: receipt did not execute in the current project workspace",
|
|
987
|
+
errors,
|
|
988
|
+
)
|
|
860
989
|
validate_external_targets(receipt.get("external_targets"), f"{location}.external_targets", errors)
|
|
861
990
|
for raw_key, hash_key in (("stdout_path", "stdout_sha256"), ("stderr_path", "stderr_sha256")):
|
|
862
991
|
relative = receipt.get(raw_key)
|
|
@@ -893,7 +1022,7 @@ def validate_requests_and_receipts(
|
|
|
893
1022
|
for path in (root / "receipts").rglob("*.json")
|
|
894
1023
|
} if (root / "receipts").is_dir() else set()
|
|
895
1024
|
require(receipt_files == seen_receipt_paths, "receipt directory does not exactly match executions.json", errors)
|
|
896
|
-
return execution_requests,
|
|
1025
|
+
return execution_requests, executions
|
|
897
1026
|
|
|
898
1027
|
|
|
899
1028
|
def validate_execution_request_semantics(
|
|
@@ -906,17 +1035,36 @@ def validate_execution_request_semantics(
|
|
|
906
1035
|
require(isinstance(command, list) and bool(command) and all(isinstance(item, str) and item for item in command), f"{location}: command must be a non-empty string argv", errors)
|
|
907
1036
|
environment = value.get("environment")
|
|
908
1037
|
require(isinstance(environment, dict) and all(isinstance(key, str) and isinstance(item, str) for key, item in environment.items()), f"{location}: environment must contain complete string values", errors)
|
|
909
|
-
|
|
910
|
-
|
|
1038
|
+
has_ids = "test_ids" in value
|
|
1039
|
+
has_paths = "test_paths" in value
|
|
1040
|
+
has_auto_bind = "auto_bind" in value
|
|
911
1041
|
auto_bind = value.get("auto_bind") is True
|
|
1042
|
+
binding_modes = int(has_ids) + int(has_paths) + int(auto_bind)
|
|
912
1043
|
require(
|
|
913
|
-
|
|
914
|
-
|
|
915
|
-
or auto_bind,
|
|
916
|
-
f"{location}: execution must provide test_ids, test_paths, or auto_bind",
|
|
1044
|
+
binding_modes == 1 and not (has_auto_bind and not auto_bind),
|
|
1045
|
+
f"{location}: binding modes test_ids, test_paths, and auto_bind=true must be mutually exclusive",
|
|
917
1046
|
errors,
|
|
918
1047
|
)
|
|
1048
|
+
if has_ids:
|
|
1049
|
+
test_ids = value.get("test_ids")
|
|
1050
|
+
require(
|
|
1051
|
+
isinstance(test_ids, list) and bool(test_ids)
|
|
1052
|
+
and all(isinstance(item, str) and item for item in test_ids)
|
|
1053
|
+
and len(set(test_ids)) == len(test_ids),
|
|
1054
|
+
f"{location}: test_ids must be a non-empty unique string array",
|
|
1055
|
+
errors,
|
|
1056
|
+
)
|
|
1057
|
+
if has_paths:
|
|
1058
|
+
test_paths = value.get("test_paths")
|
|
1059
|
+
require(
|
|
1060
|
+
isinstance(test_paths, list) and bool(test_paths)
|
|
1061
|
+
and all(isinstance(item, str) and item for item in test_paths)
|
|
1062
|
+
and len(set(test_paths)) == len(test_paths),
|
|
1063
|
+
f"{location}: test_paths must be a non-empty unique string array",
|
|
1064
|
+
errors,
|
|
1065
|
+
)
|
|
919
1066
|
if receipt:
|
|
1067
|
+
require(has_ids and not has_paths and not has_auto_bind, f"{location}: receipt binding must use canonical test_ids only", errors)
|
|
920
1068
|
require(type(value.get("selected_execution")) is bool, f"{location}: selected_execution must be boolean", errors)
|
|
921
1069
|
require(type(value.get("reliable")) is bool, f"{location}: reliable must be boolean", errors)
|
|
922
1070
|
require(type(value.get("exit_code")) is int and not isinstance(value.get("exit_code"), bool), f"{location}: exit_code must be an integer", errors)
|
|
@@ -964,6 +1112,25 @@ def validate_canonical_capture(
|
|
|
964
1112
|
scope_changed = set(scope.get("changed_files", [])) if isinstance(scope.get("changed_files"), list) else set()
|
|
965
1113
|
require(captured_paths == inventory_changed, "canonical capture and inventory changed_files do not match", errors)
|
|
966
1114
|
require(captured_paths == scope_changed, "canonical capture and scope changed_files do not match", errors)
|
|
1115
|
+
baseline = capture.get("baseline_commit")
|
|
1116
|
+
if not isinstance(baseline, str) or not baseline:
|
|
1117
|
+
errors.append("canonical capture baseline_commit is missing")
|
|
1118
|
+
return
|
|
1119
|
+
try:
|
|
1120
|
+
live_patch, live_entries = canonical_change_patch(project_root, baseline)
|
|
1121
|
+
except EvidenceError as exc:
|
|
1122
|
+
errors.append(f"cannot verify live canonical change: {exc}")
|
|
1123
|
+
return
|
|
1124
|
+
require(
|
|
1125
|
+
hashlib.sha256(live_patch).hexdigest() == capture.get("patch_sha256"),
|
|
1126
|
+
"canonical change capture is stale: live patch hash differs",
|
|
1127
|
+
errors,
|
|
1128
|
+
)
|
|
1129
|
+
require(
|
|
1130
|
+
live_entries == capture.get("changed_files"),
|
|
1131
|
+
"canonical change capture is stale: live changed-file entries differ",
|
|
1132
|
+
errors,
|
|
1133
|
+
)
|
|
967
1134
|
|
|
968
1135
|
|
|
969
1136
|
def validate_test_preparation(
|
|
@@ -1102,7 +1269,14 @@ def validate_plan_and_scope_bindings(
|
|
|
1102
1269
|
for layer in test_layers if isinstance(test_layers, list) else []:
|
|
1103
1270
|
require(layer in LAYER_ORDER, f"planned test uses unknown layer: {test_id}/{layer}", errors)
|
|
1104
1271
|
matching_receipts = selected_by_test.get(test_id, [])
|
|
1105
|
-
|
|
1272
|
+
if manifest.get("final_verdict") == "TEST_PASS":
|
|
1273
|
+
passing_receipts = [
|
|
1274
|
+
receipt for receipt in matching_receipts
|
|
1275
|
+
if receipt.get("reliable") is True and receipt.get("exit_code") == 0
|
|
1276
|
+
]
|
|
1277
|
+
require(bool(passing_receipts), f"planned test has no selected reliable zero-exit receipt: {test_id}", errors)
|
|
1278
|
+
else:
|
|
1279
|
+
require(bool(matching_receipts), f"planned test has no selected receipt: {test_id}", errors)
|
|
1106
1280
|
for receipt in matching_receipts:
|
|
1107
1281
|
require(receipt.get("layer") in test_layers, f"planned test receipt uses an unplanned layer: {test_id}", errors)
|
|
1108
1282
|
|
|
@@ -1164,6 +1338,12 @@ def validate_matrix_risks(
|
|
|
1164
1338
|
)
|
|
1165
1339
|
if verdict == "TEST_FAIL":
|
|
1166
1340
|
require(reliably_failed, "TEST_FAIL lacks a reliable reproduced failing execution", errors)
|
|
1341
|
+
elif verdict == "TEST_PASS":
|
|
1342
|
+
require(
|
|
1343
|
+
not reliably_failed,
|
|
1344
|
+
"TEST_PASS contains a reliable selected failing execution",
|
|
1345
|
+
errors,
|
|
1346
|
+
)
|
|
1167
1347
|
|
|
1168
1348
|
for behavior in matrix.get("behaviors", []):
|
|
1169
1349
|
if not isinstance(behavior, dict):
|
|
@@ -1205,109 +1385,6 @@ def validate_matrix_risks(
|
|
|
1205
1385
|
errors.append(f"TEST_PASS has unresolved behavior risk: {behavior_id}/{risk_name}")
|
|
1206
1386
|
|
|
1207
1387
|
|
|
1208
|
-
def validate_differential_proofs(
|
|
1209
|
-
root: Path,
|
|
1210
|
-
project_root: Path,
|
|
1211
|
-
manifest: dict[str, Any],
|
|
1212
|
-
matrix: dict[str, Any],
|
|
1213
|
-
proof: dict[str, Any],
|
|
1214
|
-
differential_requests: dict[str, dict[str, Any]],
|
|
1215
|
-
executions: list[dict[str, Any]],
|
|
1216
|
-
entry_map: dict[str, dict[str, Any]],
|
|
1217
|
-
records_schema: dict[str, Any],
|
|
1218
|
-
errors: list[str],
|
|
1219
|
-
) -> None:
|
|
1220
|
-
verdict = manifest.get("final_verdict")
|
|
1221
|
-
receipts = {item.get("execution_id"): item for item in executions if isinstance(item.get("execution_id"), str)}
|
|
1222
|
-
behavior_ids = {item.get("id") for item in matrix.get("behaviors", []) if isinstance(item, dict)}
|
|
1223
|
-
proof_by_behavior: dict[str, dict[str, Any]] = {}
|
|
1224
|
-
proofs = proof.get("proofs")
|
|
1225
|
-
if not isinstance(proofs, list):
|
|
1226
|
-
return
|
|
1227
|
-
for item in proofs:
|
|
1228
|
-
if not isinstance(item, dict):
|
|
1229
|
-
continue
|
|
1230
|
-
behavior_id = item.get("behavior_id")
|
|
1231
|
-
require(behavior_id in behavior_ids, f"differential proof maps unknown behavior: {behavior_id}", errors)
|
|
1232
|
-
require(behavior_id not in proof_by_behavior, f"duplicate differential proof: {behavior_id}", errors)
|
|
1233
|
-
if isinstance(behavior_id, str):
|
|
1234
|
-
proof_by_behavior[behavior_id] = item
|
|
1235
|
-
classification = item.get("classification")
|
|
1236
|
-
if classification == "NOT_APPLICABLE":
|
|
1237
|
-
validate_differential_na(item.get("not_applicable"), f"differential proof {behavior_id} not_applicable", errors)
|
|
1238
|
-
for key in (
|
|
1239
|
-
"method", "differential_request_path", "differential_request_sha256", "baseline_commit",
|
|
1240
|
-
"baseline_execution_id", "mutation_execution_id", "current_execution_id",
|
|
1241
|
-
):
|
|
1242
|
-
require(item.get(key) is None, f"N/A differential proof must clear {key}: {behavior_id}", errors)
|
|
1243
|
-
require(item.get("failure_reason_matched") is False, f"N/A differential proof cannot claim a failure match: {behavior_id}", errors)
|
|
1244
|
-
continue
|
|
1245
|
-
|
|
1246
|
-
request_relative = item.get("differential_request_path")
|
|
1247
|
-
request = differential_requests.get(request_relative) if isinstance(request_relative, str) else None
|
|
1248
|
-
require(request is not None, f"differential proof has no matching request: {behavior_id}", errors)
|
|
1249
|
-
request_path = require_manifest_entry(request_relative, entry_map, root, errors, "differential request") if isinstance(request_relative, str) else None
|
|
1250
|
-
if request_path is not None:
|
|
1251
|
-
require(file_sha256(request_path) == item.get("differential_request_sha256"), f"differential request hash mismatch: {behavior_id}", errors)
|
|
1252
|
-
if not isinstance(request, dict):
|
|
1253
|
-
continue
|
|
1254
|
-
require(request.get("behavior_id") == behavior_id, f"differential request behavior mismatch: {behavior_id}", errors)
|
|
1255
|
-
method = item.get("method")
|
|
1256
|
-
require(method == request.get("method") and method in {"baseline", "controlled-mutation"}, f"differential method mismatch: {behavior_id}", errors)
|
|
1257
|
-
require(item.get("baseline_commit") == request.get("baseline_commit") == manifest.get("baseline_commit"), f"differential baseline commit mismatch: {behavior_id}", errors)
|
|
1258
|
-
require(item.get("cleanup_succeeded") is True, f"differential cleanup failed: {behavior_id}", errors)
|
|
1259
|
-
|
|
1260
|
-
current = receipts.get(item.get("current_execution_id"))
|
|
1261
|
-
require(current is not None, f"differential current receipt is missing: {behavior_id}", errors)
|
|
1262
|
-
failing_id = item.get("baseline_execution_id") if method == "baseline" else item.get("mutation_execution_id")
|
|
1263
|
-
failing = receipts.get(failing_id)
|
|
1264
|
-
require(failing is not None, f"differential failing receipt is missing: {behavior_id}", errors)
|
|
1265
|
-
if current is not None and failing is not None:
|
|
1266
|
-
require(current.get("request_sha256") == failing.get("request_sha256"), f"differential sides use different execution requests: {behavior_id}", errors)
|
|
1267
|
-
nested = request.get("execution_request")
|
|
1268
|
-
current_request_path = current.get("request_path")
|
|
1269
|
-
current_request = load_json(root / current_request_path) if isinstance(current_request_path, str) and safe_path(root, current_request_path, errors, "differential receipt request") else None
|
|
1270
|
-
require(current_request == nested, f"differential current receipt request does not match request: {behavior_id}", errors)
|
|
1271
|
-
require(current.get("selected_execution") is True and current.get("reliable") is True and current.get("exit_code") == 0, f"differential current side is not a reliable success: {behavior_id}", errors)
|
|
1272
|
-
require(failing.get("selected_execution") is False and failing.get("reliable") is True and failing.get("exit_code") != 0, f"differential failing side is not a reliable failure: {behavior_id}", errors)
|
|
1273
|
-
current_isolation = current.get("isolation")
|
|
1274
|
-
failing_isolation = failing.get("isolation")
|
|
1275
|
-
require(isinstance(current_isolation, dict) and current_isolation.get("kind") == "project-direct", f"differential current receipt lacks project-direct isolation: {behavior_id}", errors)
|
|
1276
|
-
require(isinstance(failing_isolation, dict) and failing_isolation.get("kind") == method, f"differential failing receipt isolation mismatch: {behavior_id}", errors)
|
|
1277
|
-
if method == "baseline" and isinstance(failing_isolation, dict):
|
|
1278
|
-
require(failing_isolation.get("baseline_commit") == request.get("baseline_commit"), f"baseline receipt commit mismatch: {behavior_id}", errors)
|
|
1279
|
-
if method == "controlled-mutation" and isinstance(failing_isolation, dict):
|
|
1280
|
-
patch_relative = request.get("mutation_patch_path")
|
|
1281
|
-
patch_path = require_manifest_entry(patch_relative, entry_map, root, errors, "controlled mutation patch") if isinstance(patch_relative, str) else None
|
|
1282
|
-
require(patch_path is not None, f"controlled mutation patch is missing: {behavior_id}", errors)
|
|
1283
|
-
if patch_path is not None:
|
|
1284
|
-
patch_hash = file_sha256(patch_path)
|
|
1285
|
-
require(failing_isolation.get("patch_sha256") == patch_hash, f"controlled mutation receipt patch hash mismatch: {behavior_id}", errors)
|
|
1286
|
-
if method == "baseline":
|
|
1287
|
-
require(item.get("mutation_execution_id") is None, f"baseline proof has mutation receipt: {behavior_id}", errors)
|
|
1288
|
-
require(request.get("mutation_patch_path") is None, f"baseline request has mutation patch: {behavior_id}", errors)
|
|
1289
|
-
elif method == "controlled-mutation":
|
|
1290
|
-
require(item.get("baseline_execution_id") is None, f"mutation proof has baseline receipt: {behavior_id}", errors)
|
|
1291
|
-
require(isinstance(request.get("mutation_patch_path"), str) and bool(request.get("mutation_patch_path")), f"mutation request lacks patch: {behavior_id}", errors)
|
|
1292
|
-
if classification == "PROVEN":
|
|
1293
|
-
require(item.get("failure_reason_matched") is True, f"differential failure reason was not matched: {behavior_id}", errors)
|
|
1294
|
-
if failing is not None:
|
|
1295
|
-
output = b""
|
|
1296
|
-
for key in ("stdout_path", "stderr_path"):
|
|
1297
|
-
relative = failing.get(key)
|
|
1298
|
-
path = safe_path(root, relative, errors, "differential raw output") if isinstance(relative, str) else None
|
|
1299
|
-
if path is not None and path.is_file():
|
|
1300
|
-
output += path.read_bytes()
|
|
1301
|
-
signals = request.get("expected_failure_signals")
|
|
1302
|
-
require(isinstance(signals, list) and any(isinstance(signal, str) and signal.encode() in output for signal in signals), f"differential expected failure signal is absent: {behavior_id}", errors)
|
|
1303
|
-
elif classification == "UNPROVEN" and verdict == "TEST_PASS":
|
|
1304
|
-
errors.append(f"TEST_PASS has unproven necessary behavior: {behavior_id}")
|
|
1305
|
-
|
|
1306
|
-
if verdict == "TEST_PASS":
|
|
1307
|
-
for behavior_id in behavior_ids:
|
|
1308
|
-
require(behavior_id in proof_by_behavior, f"behavior lacks differential proof: {behavior_id}", errors)
|
|
1309
|
-
|
|
1310
|
-
|
|
1311
1388
|
def validate_infrastructure(
|
|
1312
1389
|
manifest: dict[str, Any],
|
|
1313
1390
|
infrastructure: dict[str, Any],
|
|
@@ -1375,6 +1452,23 @@ def validate_verdict_and_attestation(
|
|
|
1375
1452
|
require(verdict_record.get("real_environment_validated") is False, "code evidence must not claim real-environment validation", errors)
|
|
1376
1453
|
require(verdict_record.get("repairs_business_defects") is False, "test skill must not repair business defects", errors)
|
|
1377
1454
|
if verdict == "TEST_PASS":
|
|
1455
|
+
selected = [
|
|
1456
|
+
item for item in executions
|
|
1457
|
+
if isinstance(item, dict) and item.get("selected_execution") is True
|
|
1458
|
+
]
|
|
1459
|
+
require(bool(selected), "TEST_PASS has no selected executions", errors)
|
|
1460
|
+
for item in selected:
|
|
1461
|
+
execution_id = item.get("execution_id", "unknown-execution")
|
|
1462
|
+
require(
|
|
1463
|
+
item.get("reliable") is True,
|
|
1464
|
+
f"TEST_PASS contains a selected unreliable execution: {execution_id}",
|
|
1465
|
+
errors,
|
|
1466
|
+
)
|
|
1467
|
+
require(
|
|
1468
|
+
item.get("exit_code") == 0,
|
|
1469
|
+
f"TEST_PASS contains a selected failing execution: {execution_id}",
|
|
1470
|
+
errors,
|
|
1471
|
+
)
|
|
1378
1472
|
require(verdict_record.get("code_evidence_replayable") is True, "TEST_PASS must declare replayable code evidence", errors)
|
|
1379
1473
|
require(not verdict_record.get("blockers"), "TEST_PASS contains blockers", errors)
|
|
1380
1474
|
require(not verdict_record.get("reproduced_failures"), "TEST_PASS contains reproduced failures", errors)
|
|
@@ -1505,20 +1599,18 @@ def validate_evidence(
|
|
|
1505
1599
|
scope, inventory, inventory_by_category, project_root, root,
|
|
1506
1600
|
get_captured_deleted_paths(root), errors,
|
|
1507
1601
|
)
|
|
1508
|
-
execution_requests,
|
|
1602
|
+
execution_requests, executions = validate_requests_and_receipts(root, entry_map, records_schema, errors)
|
|
1509
1603
|
|
|
1510
1604
|
if change_class == "behavior":
|
|
1511
1605
|
try:
|
|
1512
1606
|
matrix = load_json(root / "behavior-risk-matrix.json")
|
|
1513
1607
|
infrastructure = load_json(root / "infrastructure-changes.json")
|
|
1514
|
-
proof = load_json(root / "differential-proof.json")
|
|
1515
1608
|
except EvidenceError as exc:
|
|
1516
1609
|
errors.append(str(exc))
|
|
1517
|
-
matrix = infrastructure =
|
|
1518
|
-
if all(isinstance(item, dict) for item in (matrix, infrastructure,
|
|
1610
|
+
matrix = infrastructure = None
|
|
1611
|
+
if all(isinstance(item, dict) for item in (matrix, infrastructure, plan)):
|
|
1519
1612
|
validate_schema(matrix, matrix_schema, matrix_schema, "behavior-risk-matrix", errors)
|
|
1520
1613
|
validate_definition(infrastructure, "infrastructureChanges", records_schema, "infrastructure-changes", errors)
|
|
1521
|
-
validate_definition(proof, "differentialProof", records_schema, "differential-proof", errors)
|
|
1522
1614
|
stages = {item.get("name"): item for item in manifest.get("stages", []) if isinstance(item, dict)}
|
|
1523
1615
|
for stage_name in CORE_BEHAVIOR_STAGES:
|
|
1524
1616
|
require(stages.get(stage_name, {}).get("status") == "complete", f"behavior protocol stage cannot be N/A: {stage_name}", errors)
|
|
@@ -1527,10 +1619,6 @@ def validate_evidence(
|
|
|
1527
1619
|
executions, records_schema, errors,
|
|
1528
1620
|
)
|
|
1529
1621
|
validate_matrix_risks(manifest, matrix, test_map, executions, records_schema, errors)
|
|
1530
|
-
validate_differential_proofs(
|
|
1531
|
-
root, project_root, manifest, matrix, proof, differential_requests, executions,
|
|
1532
|
-
entry_map, records_schema, errors,
|
|
1533
|
-
)
|
|
1534
1622
|
validate_infrastructure(manifest, infrastructure, executions, entry_map, root, errors)
|
|
1535
1623
|
else:
|
|
1536
1624
|
errors.append("behavior evidence records must be JSON objects")
|
|
@@ -1578,10 +1666,7 @@ def render_report_text(
|
|
|
1578
1666
|
) -> str:
|
|
1579
1667
|
lifecycle_path = root / "lifecycle.json"
|
|
1580
1668
|
lifecycle = load_json(lifecycle_path) if lifecycle_path.exists() else {}
|
|
1581
|
-
proof_path = root / "differential-proof.json"
|
|
1582
|
-
proof_record = load_json(proof_path) if proof_path.exists() else {}
|
|
1583
1669
|
executions_record = load_json(root / "executions.json")
|
|
1584
|
-
proofs = proof_record.get("proofs", []) if isinstance(proof_record, dict) else []
|
|
1585
1670
|
executions = executions_record if isinstance(executions_record, list) else []
|
|
1586
1671
|
artifact_dir = lifecycle.get("artifact_dir") if isinstance(lifecycle, dict) else None
|
|
1587
1672
|
lines = [
|
|
@@ -1599,7 +1684,6 @@ def render_report_text(
|
|
|
1599
1684
|
f"- Artifact Dir: {artifact_dir or 'not provided'}",
|
|
1600
1685
|
"",
|
|
1601
1686
|
f"Execution receipts: {len(executions)}",
|
|
1602
|
-
f"Differential proofs: {len(proofs) if isinstance(proofs, list) else 0}",
|
|
1603
1687
|
"The legacy test-report interface is not supported.",
|
|
1604
1688
|
"",
|
|
1605
1689
|
]
|