prizmkit 1.1.138 → 1.1.139
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bundled/VERSION.json +3 -3
- package/bundled/dev-pipeline/scripts/prompt_framework.py +9 -5
- package/bundled/dev-pipeline/scripts/update-checkpoint.py +51 -0
- package/bundled/dev-pipeline/templates/sections/phase-prizmkit-test.md +2 -1
- package/bundled/dev-pipeline/tests/test_checkpoint_state.py +47 -0
- package/bundled/skills/_metadata.json +1 -1
- package/bundled/skills/prizmkit-test/SKILL.md +6 -5
- package/bundled/skills/prizmkit-test/assets/evidence-package-template.json +1 -0
- package/bundled/skills/prizmkit-test/references/evidence-protocol.md +4 -1
- package/bundled/skills/prizmkit-test/references/evidence-request-protocol.md +4 -1
- package/bundled/skills/prizmkit-test/scripts/build_test_evidence.py +182 -9
- package/bundled/skills/prizmkit-test/scripts/validate_test_evidence.py +41 -14
- package/package.json +1 -1
package/bundled/VERSION.json
CHANGED
|
@@ -117,6 +117,14 @@ def pipeline_checkpoint_metadata(artifact_dir, evidence=None, stage="plan",
|
|
|
117
117
|
blocked_reason=None, terminal_status=None):
|
|
118
118
|
"""Build family-neutral semantic L4 metadata without replacing L1 state."""
|
|
119
119
|
evidence = evidence or {}
|
|
120
|
+
if evidence:
|
|
121
|
+
evidence_paths = {
|
|
122
|
+
"manifest": evidence.get("manifest_path"),
|
|
123
|
+
"verdict": evidence.get("verdict_path"),
|
|
124
|
+
"validation": evidence.get("validation_path"),
|
|
125
|
+
}
|
|
126
|
+
else:
|
|
127
|
+
evidence_paths = {"manifest": None, "verdict": None, "validation": None}
|
|
120
128
|
current_stage = None if terminal_status else stage
|
|
121
129
|
semantic_next_stage = None if terminal_status else next_stage
|
|
122
130
|
return {
|
|
@@ -129,11 +137,7 @@ def pipeline_checkpoint_metadata(artifact_dir, evidence=None, stage="plan",
|
|
|
129
137
|
"stage_result": stage_result,
|
|
130
138
|
"repair_scope": repair_scope,
|
|
131
139
|
"repair_round": repair_round,
|
|
132
|
-
"authoritative_evidence_paths":
|
|
133
|
-
"manifest": evidence.get("manifest_path"),
|
|
134
|
-
"verdict": evidence.get("verdict_path"),
|
|
135
|
-
"validation": evidence.get("validation_path"),
|
|
136
|
-
},
|
|
140
|
+
"authoritative_evidence_paths": evidence_paths,
|
|
137
141
|
"next_stage": semantic_next_stage,
|
|
138
142
|
"resume_from": resume_from,
|
|
139
143
|
"blocked_reason": blocked_reason,
|
|
@@ -213,6 +213,43 @@ def _checkpoint_project_root(checkpoint_path):
|
|
|
213
213
|
return str(parent.parent)
|
|
214
214
|
return str(path.parent)
|
|
215
215
|
|
|
216
|
+
def _discover_test_evidence_paths(checkpoint_path, data):
|
|
217
|
+
"""Discover the latest finalized test package from the artifact handoff pointer."""
|
|
218
|
+
project_root = Path(_checkpoint_project_root(checkpoint_path))
|
|
219
|
+
candidates = []
|
|
220
|
+
artifact_dir = data.get("artifact_dir")
|
|
221
|
+
if isinstance(artifact_dir, str) and artifact_dir:
|
|
222
|
+
candidates.append(project_root / artifact_dir)
|
|
223
|
+
for state_key in ("feature_state", "bugfix_state", "refactor_state"):
|
|
224
|
+
state = data.get(state_key)
|
|
225
|
+
if isinstance(state, dict) and isinstance(state.get("artifact_dir"), str):
|
|
226
|
+
candidates.append(project_root / state["artifact_dir"])
|
|
227
|
+
item_slug = data.get("item_slug")
|
|
228
|
+
if isinstance(item_slug, str) and item_slug:
|
|
229
|
+
candidates.append(project_root / ".prizmkit" / "specs" / item_slug)
|
|
230
|
+
candidates.append(project_root / ".prizmkit" / "bugfix" / item_slug)
|
|
231
|
+
candidates.append(project_root / ".prizmkit" / "refactor" / item_slug)
|
|
232
|
+
|
|
233
|
+
for artifact in candidates:
|
|
234
|
+
pointer = artifact / "test-report-path.txt"
|
|
235
|
+
try:
|
|
236
|
+
report = Path(pointer.read_text(encoding="utf-8").strip()).resolve()
|
|
237
|
+
except (OSError, ValueError):
|
|
238
|
+
continue
|
|
239
|
+
evidence_dir = report.parent
|
|
240
|
+
paths = {
|
|
241
|
+
"manifest": str(evidence_dir / "manifest.json"),
|
|
242
|
+
"verdict": str(evidence_dir / "verdict.json"),
|
|
243
|
+
"validation": str(evidence_dir / "validation.json"),
|
|
244
|
+
}
|
|
245
|
+
if all(Path(value).is_file() for value in paths.values()):
|
|
246
|
+
return {
|
|
247
|
+
key: str(Path(value).relative_to(project_root))
|
|
248
|
+
for key, value in paths.items()
|
|
249
|
+
}
|
|
250
|
+
return None
|
|
251
|
+
|
|
252
|
+
|
|
216
253
|
def _finalize_semantics(checkpoint_path, data, state_key):
|
|
217
254
|
"""Derive a deterministic terminal status after an atomic mutation."""
|
|
218
255
|
candidate = validate_checkpoint_data(
|
|
@@ -286,11 +323,25 @@ def update_checkpoint(
|
|
|
286
323
|
return {"ok": False, "error": verdict_error}
|
|
287
324
|
|
|
288
325
|
semantic_update = _normalize_semantic_update(semantic_update)
|
|
326
|
+
if (
|
|
327
|
+
semantic_update.get("stage_result") == "TEST_BLOCKED"
|
|
328
|
+
and not semantic_update.get("authoritative_evidence_paths")
|
|
329
|
+
):
|
|
330
|
+
discovered = _discover_test_evidence_paths(checkpoint_path, data)
|
|
331
|
+
if discovered:
|
|
332
|
+
semantic_update["authoritative_evidence_paths"] = discovered
|
|
289
333
|
semantic_error = _validate_semantic_update(step, semantic_update)
|
|
290
334
|
if semantic_error:
|
|
291
335
|
return {"ok": False, "error": semantic_error}
|
|
292
336
|
|
|
293
337
|
stage_result = semantic_update.get("stage_result")
|
|
338
|
+
if step.get("skill") == "prizmkit-test" and stage_result in TEST_RESULTS:
|
|
339
|
+
evidence = semantic_update.get("authoritative_evidence_paths") or step.get("authoritative_evidence_paths")
|
|
340
|
+
if not isinstance(evidence, dict) or set(evidence) != {"manifest", "verdict", "validation"}:
|
|
341
|
+
return {
|
|
342
|
+
"ok": False,
|
|
343
|
+
"error": "Every terminal test verdict requires manifest, verdict, and validation evidence paths",
|
|
344
|
+
}
|
|
294
345
|
if new_status == "completed" and step.get("skill") == "prizmkit-code-review":
|
|
295
346
|
stage_result = stage_result or step.get("stage_result")
|
|
296
347
|
if stage_result != "REVIEW_PASS":
|
|
@@ -33,7 +33,8 @@ Preserve the same artifact directory and update the L1 workflow state plus this
|
|
|
33
33
|
- `TEST_FAIL` with `repair_scope=test-infrastructure` → `prizmkit-implement` → `prizmkit-test`; do not make speculative production edits.
|
|
34
34
|
- `TEST_FAIL` with `repair_scope=production|runtime|schema|dependency|public-interface` → `prizmkit-implement` → `prizmkit-code-review` → `prizmkit-test`.
|
|
35
35
|
- `TEST_FAIL` with unknown or unsafe scope → do not guess a production edit; record `WORKFLOW_BLOCKED` and a safe resume entry.
|
|
36
|
-
- `TEST_BLOCKED` → preserve all evidence and artifacts, perform only bounded environment recovery, then finish terminal `WORKFLOW_BLOCKED` with `blocked_reason=test_blocked`; never edit production speculatively.
|
|
36
|
+
- `TEST_BLOCKED` → preserve all evidence and artifacts, perform only bounded environment recovery, then finish terminal `WORKFLOW_BLOCKED` with `blocked_reason=test_blocked`; never edit production speculatively. When a finalized blocked package exists, pass its validated `manifest`, `verdict`, and `validation` paths to the checkpoint; do not leave these paths null.
|
|
37
|
+
- The builder's evidence-protocol recovery budget is separate from the outer `repair_round`: after bounded preflight/package attempts, stop creating evidence versions and return `TEST_BLOCKED`.
|
|
37
38
|
- The outer `repair_round` is shared across review/test failures and may not exceed three. The Code Review internal loop limit remains separate and is at most ten completed rounds.
|
|
38
39
|
|
|
39
40
|
A missing, stale, incomplete, mismatched, tampered, or wrong-scope package is not `TEST_PASS`: preserve the evidence, write `failure-log.md`, and leave the checkpoint resumable or blocked according to headless policy.
|
|
@@ -440,6 +440,53 @@ def test_declared_final_artifact_must_exist_for_semantic_completion(tmp_path, mo
|
|
|
440
440
|
|
|
441
441
|
|
|
442
442
|
|
|
443
|
+
def test_blocked_test_handoff_discovers_finalized_evidence_paths(tmp_path):
|
|
444
|
+
artifact = tmp_path / ".prizmkit" / "specs" / "feature"
|
|
445
|
+
artifact.mkdir(parents=True)
|
|
446
|
+
evidence = tmp_path / ".prizmkit" / "test" / "evidence" / "evidence-1"
|
|
447
|
+
evidence.mkdir(parents=True)
|
|
448
|
+
for name in ("manifest.json", "verdict.json", "validation.json"):
|
|
449
|
+
(evidence / name).write_text("{}", encoding="utf-8")
|
|
450
|
+
report = evidence / "test-report.md"
|
|
451
|
+
report.write_text("blocked", encoding="utf-8")
|
|
452
|
+
(artifact / "test-report-path.txt").write_text(str(report), encoding="utf-8")
|
|
453
|
+
|
|
454
|
+
payload = _family_checkpoint(
|
|
455
|
+
"feature-pipeline",
|
|
456
|
+
[
|
|
457
|
+
"prizmkit-implement", "prizmkit-code-review", "prizmkit-test",
|
|
458
|
+
"prizmkit-retrospective", "prizmkit-committer", "completion-summary",
|
|
459
|
+
],
|
|
460
|
+
semantic={"artifact_dir": ".prizmkit/specs/feature"},
|
|
461
|
+
)
|
|
462
|
+
payload["feature_state"]["artifact_dir"] = ".prizmkit/specs/feature"
|
|
463
|
+
payload["steps"][1]["stage_result"] = "REVIEW_PASS"
|
|
464
|
+
payload["steps"][2]["status"] = "in_progress"
|
|
465
|
+
payload["steps"][2].pop("stage_result")
|
|
466
|
+
for step in payload["steps"][3:]:
|
|
467
|
+
step["status"] = "pending"
|
|
468
|
+
step.pop("stage_result", None)
|
|
469
|
+
checkpoint = _write(tmp_path / ".prizmkit" / "specs" / "feature" / "workflow-checkpoint.json", payload)
|
|
470
|
+
|
|
471
|
+
result = update_checkpoint(
|
|
472
|
+
str(checkpoint),
|
|
473
|
+
"prizmkit-test",
|
|
474
|
+
"failed",
|
|
475
|
+
semantic_update={
|
|
476
|
+
"stage_result": "TEST_BLOCKED",
|
|
477
|
+
"blocked_reason": "test_blocked",
|
|
478
|
+
"terminal_status": "WORKFLOW_BLOCKED",
|
|
479
|
+
},
|
|
480
|
+
)
|
|
481
|
+
|
|
482
|
+
assert result["ok"] is True
|
|
483
|
+
paths = json.loads(checkpoint.read_text(encoding="utf-8"))["steps"][2]["authoritative_evidence_paths"]
|
|
484
|
+
assert paths["manifest"] == ".prizmkit/test/evidence/evidence-1/manifest.json"
|
|
485
|
+
assert paths["verdict"] == ".prizmkit/test/evidence/evidence-1/verdict.json"
|
|
486
|
+
assert paths["validation"] == ".prizmkit/test/evidence/evidence-1/validation.json"
|
|
487
|
+
|
|
488
|
+
|
|
489
|
+
|
|
443
490
|
def test_update_checkpoint_rejects_test_pass_when_validator_replay_fails(tmp_path, monkeypatch):
|
|
444
491
|
paths = _write_test_evidence(tmp_path)
|
|
445
492
|
payload = _family_checkpoint(
|
|
@@ -30,13 +30,13 @@ Use `${SKILL_DIR}/scripts/build_test_evidence.py` for every lifecycle command:
|
|
|
30
30
|
- `inventory`: hashes current source/tests/contracts/lockfiles, rejects category overlap, excludes runtime-only paths, and validates directory roots plus the complete capture-derived changed-file set.
|
|
31
31
|
- `prepare-tests`: discovers repository-native test files from the canonical capture and inventory, generates stable builder-owned test IDs, derives existing/added/modified status, and snapshots only added/modified live test sources.
|
|
32
32
|
- `execute`: runs native project commands and owns `executions.json`, `receipts/`, and `raw/`. Each request uses exactly one binding mode: `test_ids`, `test_paths`, or `auto_bind=true`. The builder infers recognized package-wide or explicit file/directory/glob command scope, blocks opaque commands or mismatched declarations, and persists verified canonical `test_ids` only.
|
|
33
|
-
- `finalize`: rechecks canonical capture freshness, blocks every selected unreliable execution, and owns manifest, stage hashes, verdict, generated-test snapshots, and evidence identity.
|
|
33
|
+
- `finalize`: runs the pre-finalize semantic-record and required-layer checks, rechecks canonical capture freshness, blocks every selected unreliable execution, and owns manifest, stage hashes, verdict, generated-test snapshots, and evidence identity. If semantic records are incomplete or malformed, finalize fails before creating an immutable package; do not hand-edit a finalized package or start another package after a protocol budget is exhausted.
|
|
34
34
|
- `render-report`: derives `test-report.md`; it is never authoritative.
|
|
35
35
|
- `resume`: preserves prior receipts and refuses to rebind an identity-changing package; create a new package after scope/source identity changes.
|
|
36
36
|
|
|
37
37
|
`test-plan.json` is the only source of builder-generated test IDs. An execution request provides exactly one of native `test_paths`, `auto_bind=true`, or canonical `test_ids`; the actual command must expose the same deterministic scope, including all prepared tests for recognized package-wide commands. The builder blocks opaque or mismatched command scope and rewrites verified requests to canonical `test_ids` only before execution. Assess all five layers explicitly, but set `required=true` only when risk requires it; every omitted layer needs structured N/A evidence. The same binding applies test ↔ behavior ↔ execution; current-workspace receipts are the execution evidence.
|
|
38
38
|
|
|
39
|
-
The builder/validator, not the Main Agent, produces or updates `manifest.json`, `executions.json`, `receipts/*`, `validation.json`, `verdict.json`, `test-report.md`, and `source-change.patch`. `validation.json` is produced only by `validate_test_evidence.py --attest`; a package is not a `TEST_PASS` until the second strict validation succeeds.
|
|
39
|
+
The builder/validator, not the Main Agent, produces or updates `manifest.json`, `executions.json`, `receipts/*`, `validation.json`, `verdict.json`, `test-report.md`, and `source-change.patch`. Semantic records are authored through schema-shaped requests and must pass the builder preflight before finalization. `validation.json` is produced only by `validate_test_evidence.py --attest`; a package is not a `TEST_PASS` until the second strict validation succeeds.
|
|
40
40
|
|
|
41
41
|
|
|
42
42
|
For a formal requirement, `/prizmkit-test` runs after the Main-Agent code-review stage reaches `PASS` and before retrospective:
|
|
@@ -68,7 +68,8 @@ Documentation, comment, and formatting-only scopes use deterministic lightweight
|
|
|
68
68
|
3. Preserve complete command/environment/output values as requested. Mark the package `sensitivity=project-controlled`; the project owns access control, retention, and upload policy.
|
|
69
69
|
4. Treat code-level dependencies as mock-first. Real deployed test-environment validation is a separate authorized activity.
|
|
70
70
|
5. Never delete existing tests.
|
|
71
|
-
6.
|
|
71
|
+
6. Evidence-protocol recovery is separately bounded: allow at most two package initializations and two semantic preflight failures for one canonical change identity. After the budget is exhausted, return `TEST_BLOCKED`; do not create another versioned package or hand-edit finalized evidence.
|
|
72
|
+
7. For `TEST_FAIL`, classify and persist an evidence-backed `repair_scope` of `test-infrastructure`, `production`, `runtime`, `schema`, `dependency`, `public-interface`, or `unknown`. Test-infrastructure scope routes to implement then test; production-family scope routes to implement, code-review, then test. If evidence cannot support a safe classification, return `TEST_BLOCKED`.
|
|
72
73
|
|
|
73
74
|
If test infrastructure is missing, the infrastructure preparation state establishes only the necessary project-native infrastructure autonomously. Do not ask the user to install routine test dependencies in a headless run; verify dependency versions before editing manifests.
|
|
74
75
|
|
|
@@ -216,7 +217,7 @@ python3 ${SKILL_DIR}/scripts/validate_test_evidence.py \
|
|
|
216
217
|
--project-root <target-project-root>
|
|
217
218
|
```
|
|
218
219
|
|
|
219
|
-
If this pre-check finds errors, fix them
|
|
220
|
+
If this pre-check finds errors, fix them only while the package remains mutable and before `finalize` relocates it to its content-addressed immutable directory. Never edit `manifest.json`, `verdict.json`, `validation.json`, or any semantic record after finalization. The validator checks every authoritative record against its shipped schema, live target inventories, patch/diff binding, changed-file/module-root/exclusion/Regression Ring cross-links, generated-test snapshots, planned-test execution links, runner-generated receipts/request/raw-output hashes, successful behavior-risk mappings, production/unknown external-target safety, and package integrity.
|
|
220
221
|
|
|
221
222
|
**Second pass (attestation, with `--attest`):**
|
|
222
223
|
|
|
@@ -255,7 +256,7 @@ No conditional pass exists.
|
|
|
255
256
|
|
|
256
257
|
Before writing state, read `${SKILL_DIR}/references/workflow-state-protocol.md`.
|
|
257
258
|
|
|
258
|
-
On `TEST_PASS`, update the active workflow state with `status=TEST_PASS`, `stage_result=TEST_PASS`, and `next_stage=retrospective`. On `TEST_FAIL`, persist the evidence-backed `repair_scope`, set `status=TEST_FAIL` and `stage_result=TEST_FAIL`, and return to implementation; test-infrastructure scope selects `next_stage=implement` with a direct test resume, while production-family scope selects `next_stage=implement` and requires code review before testing again. On `TEST_BLOCKED`, preserve the blocker, do not modify production code speculatively, and return a blocked result. The authoritative verdict is the validated evidence record; the derived Markdown report is only a compatibility view. If an active orchestrator exists, return the terminal result and `next_stage` to it; do not invoke the next skill independently.
|
|
259
|
+
On `TEST_PASS`, update the active workflow state with `status=TEST_PASS`, `stage_result=TEST_PASS`, and `next_stage=retrospective`. On `TEST_FAIL`, persist the evidence-backed `repair_scope`, set `status=TEST_FAIL` and `stage_result=TEST_FAIL`, and return to implementation; test-infrastructure scope selects `next_stage=implement` with a direct test resume, while production-family scope selects `next_stage=implement` and requires code review before testing again. On `TEST_BLOCKED`, preserve the blocker and any validated finalized evidence paths, do not modify production code speculatively, and return a blocked result. A validator `PASS` means only that the evidence package is structurally valid; it does not mean the testing verdict passed. The authoritative verdict is the validated evidence record; the derived Markdown report is only a compatibility view. If an active orchestrator exists, return the terminal result and `next_stage` to it; do not invoke the next skill independently.
|
|
259
260
|
|
|
260
261
|
## Derived Report and Handoff
|
|
261
262
|
|
|
@@ -22,7 +22,10 @@ Hashes and validator receipts provide byte-level binding and drift detection, no
|
|
|
22
22
|
|
|
23
23
|
`source-change.patch` preserves the complete target diff. Snapshot generated/changed tests under `generated-tests/`, contracts under `contracts/`, and raw stdout/stderr under `raw/`. Never replace old execution/output records.
|
|
24
24
|
|
|
25
|
-
##
|
|
25
|
+
## Bounded Protocol Recovery
|
|
26
|
+
|
|
27
|
+
Evidence-protocol repair is separate from production/test repair. The builder permits only a bounded recovery budget for semantic-record preflight and package initialization. After the budget is exhausted, return `TEST_BLOCKED`; do not create another versioned evidence package, copy requests from a finalized package, or hand-edit `manifest.json`, `verdict.json`, `validation.json`, receipts, hashes, or evidence IDs. A finalized package is immutable. `Evidence validation: PASS` means only that the package is structurally valid; the testing verdict must be read separately from `Testing verdict: TEST_PASS | TEST_FAIL | TEST_BLOCKED`.
|
|
28
|
+
|
|
26
29
|
|
|
27
30
|
The evidence package is an audit of repository behavior, not a request to invent a parallel test suite. Discover and execute the smallest relevant native tests before adding anything. Add, modify, or delete tests only in the target repository. The builder's `prepare-tests` command derives stable test IDs and change status from canonical Git capture, and snapshots only added or modified repository-native tests. `generated-tests/` is an evidence archive, never an execution workspace; it may be empty when existing tests cover the behavior.
|
|
28
31
|
|
|
@@ -15,7 +15,10 @@ The model writes project-semantic requests after inspecting manifests, runner co
|
|
|
15
15
|
|
|
16
16
|
Never hand-author `executions.json` receipt fields. Never copy a command's claimed output into `raw/`. Invoke the builder so provenance is linked to the actual process.
|
|
17
17
|
|
|
18
|
-
##
|
|
18
|
+
## Producer Boundary and Recovery
|
|
19
|
+
|
|
20
|
+
The model supplies schema-shaped semantic requests and mutable semantic records only before finalization. The builder performs preflight validation before creating an immutable package. Once `finalize` relocates a package to its content-addressed evidence ID, the returned `evidence_dir`, `manifest_path`, `verdict_path`, and `validation_path` are the only valid handoff paths; do not continue using the old staging directory. A validator pass is not a testing pass: read the testing verdict separately.
|
|
21
|
+
|
|
19
22
|
|
|
20
23
|
Place a request under `requests/` with:
|
|
21
24
|
|
|
@@ -59,6 +59,8 @@ INVENTORY_RESERVED_ROOTS = (
|
|
|
59
59
|
|
|
60
60
|
_IGNORED_DIR_NAMES = {".git", "__pycache__", ".pytest_cache", ".mypy_cache"}
|
|
61
61
|
FINAL_RECORDS = {"manifest.json", "validation.json", "verdict.json", "test-report.md"}
|
|
62
|
+
MAX_PROTOCOL_PACKAGE_ATTEMPTS = 2
|
|
63
|
+
PROTOCOL_STATE_NAME = ".protocol-recovery.json"
|
|
62
64
|
|
|
63
65
|
|
|
64
66
|
class RequestError(Exception):
|
|
@@ -978,11 +980,14 @@ def run_capture_change(
|
|
|
978
980
|
|
|
979
981
|
|
|
980
982
|
def run_init(project_root: Path, evidence_dir: Path, baseline: str | None, output_name: str) -> Path:
|
|
983
|
+
align_protocol_state(evidence_dir, protocol_run_key(project_root, baseline))
|
|
984
|
+
require_protocol_budget(evidence_dir, package_initialization=True)
|
|
981
985
|
if evidence_dir.exists() and any(evidence_dir.iterdir()):
|
|
982
986
|
if (evidence_dir / "manifest.json").exists():
|
|
983
987
|
raise RequestError("evidence package is finalized; initialize a new evidence directory")
|
|
984
988
|
raise RequestError("evidence directory is not empty; initialize a new evidence directory")
|
|
985
989
|
evidence_dir.mkdir(parents=True, exist_ok=True)
|
|
990
|
+
update_protocol_state(evidence_dir, event="package-initialized", increment="package_initializations")
|
|
986
991
|
for directory in ("requests", "receipts", "raw", "generated-tests", "contracts", "runner"):
|
|
987
992
|
(evidence_dir / directory).mkdir(parents=True, exist_ok=True)
|
|
988
993
|
write_json_atomic(evidence_dir / "executions.json", [])
|
|
@@ -1004,6 +1009,140 @@ def run_init(project_root: Path, evidence_dir: Path, baseline: str | None, outpu
|
|
|
1004
1009
|
return output
|
|
1005
1010
|
|
|
1006
1011
|
|
|
1012
|
+
def protocol_state_path(evidence_dir: Path) -> Path:
|
|
1013
|
+
# Keep recovery accounting outside each package so reinitializing v2/v3
|
|
1014
|
+
# cannot reset the budget by choosing another evidence directory.
|
|
1015
|
+
return evidence_dir.parent / PROTOCOL_STATE_NAME
|
|
1016
|
+
|
|
1017
|
+
|
|
1018
|
+
def protocol_run_key(project_root: Path, baseline: str | None) -> str:
|
|
1019
|
+
resolved_baseline = git_baseline(project_root, baseline)
|
|
1020
|
+
patch, _ = canonical_change_patch(project_root, resolved_baseline)
|
|
1021
|
+
return canonical_sha256({
|
|
1022
|
+
"baseline_commit": resolved_baseline,
|
|
1023
|
+
"working_diff_sha256": hashlib.sha256(patch).hexdigest(),
|
|
1024
|
+
})
|
|
1025
|
+
|
|
1026
|
+
|
|
1027
|
+
def align_protocol_state(evidence_dir: Path, run_key: str) -> None:
|
|
1028
|
+
state_path = protocol_state_path(evidence_dir)
|
|
1029
|
+
if state_path.exists():
|
|
1030
|
+
state = ensure_object(load_json(state_path), "protocol recovery state")
|
|
1031
|
+
if state.get("run_key") == run_key:
|
|
1032
|
+
return
|
|
1033
|
+
write_json_atomic(state_path, {"run_key": run_key})
|
|
1034
|
+
|
|
1035
|
+
|
|
1036
|
+
def update_protocol_state(
|
|
1037
|
+
evidence_dir: Path,
|
|
1038
|
+
*,
|
|
1039
|
+
event: str,
|
|
1040
|
+
error: str | None = None,
|
|
1041
|
+
increment: str | None = None,
|
|
1042
|
+
) -> dict[str, Any]:
|
|
1043
|
+
state_path = protocol_state_path(evidence_dir)
|
|
1044
|
+
state: dict[str, Any] = {}
|
|
1045
|
+
if state_path.exists():
|
|
1046
|
+
state = ensure_object(load_json(state_path), "protocol recovery state")
|
|
1047
|
+
if increment is not None:
|
|
1048
|
+
count = state.get(increment, 0)
|
|
1049
|
+
if not isinstance(count, int) or isinstance(count, bool):
|
|
1050
|
+
count = 0
|
|
1051
|
+
state[increment] = count + 1
|
|
1052
|
+
state["last_event"] = event
|
|
1053
|
+
state["last_error"] = error
|
|
1054
|
+
state["max_package_initializations"] = MAX_PROTOCOL_PACKAGE_ATTEMPTS
|
|
1055
|
+
state["max_preflight_failures"] = MAX_PROTOCOL_PACKAGE_ATTEMPTS
|
|
1056
|
+
state["blocked"] = (
|
|
1057
|
+
state.get("package_initializations", 0) >= MAX_PROTOCOL_PACKAGE_ATTEMPTS
|
|
1058
|
+
or state.get("preflight_failures", 0) >= MAX_PROTOCOL_PACKAGE_ATTEMPTS
|
|
1059
|
+
)
|
|
1060
|
+
write_json_atomic(state_path, state)
|
|
1061
|
+
return state
|
|
1062
|
+
|
|
1063
|
+
|
|
1064
|
+
def require_protocol_budget(evidence_dir: Path, *, package_initialization: bool = False) -> None:
|
|
1065
|
+
state_path = protocol_state_path(evidence_dir)
|
|
1066
|
+
if not state_path.exists():
|
|
1067
|
+
return
|
|
1068
|
+
state = ensure_object(load_json(state_path), "protocol recovery state")
|
|
1069
|
+
if package_initialization and state.get("package_initializations", 0) >= MAX_PROTOCOL_PACKAGE_ATTEMPTS:
|
|
1070
|
+
raise RequestError(
|
|
1071
|
+
"evidence protocol package budget exhausted; stop and return TEST_BLOCKED "
|
|
1072
|
+
"instead of creating another evidence package"
|
|
1073
|
+
)
|
|
1074
|
+
if state.get("preflight_failures", 0) >= MAX_PROTOCOL_PACKAGE_ATTEMPTS:
|
|
1075
|
+
raise RequestError(
|
|
1076
|
+
"evidence protocol preflight budget exhausted; return TEST_BLOCKED "
|
|
1077
|
+
"instead of continuing semantic record repair"
|
|
1078
|
+
)
|
|
1079
|
+
|
|
1080
|
+
|
|
1081
|
+
def semantic_record_paths(change_class: str) -> set[str]:
|
|
1082
|
+
common = {"change-classification.json", "scope.json"}
|
|
1083
|
+
if change_class == "behavior":
|
|
1084
|
+
return common | {"behavior-risk-matrix.json", "infrastructure-changes.json"}
|
|
1085
|
+
return common | {"lightweight-verification.json"}
|
|
1086
|
+
|
|
1087
|
+
|
|
1088
|
+
def preflight_semantic_records(evidence_dir: Path, change_class: str) -> None:
|
|
1089
|
+
missing = sorted(
|
|
1090
|
+
relative for relative in semantic_record_paths(change_class)
|
|
1091
|
+
if not (evidence_dir / relative).is_file()
|
|
1092
|
+
)
|
|
1093
|
+
if missing:
|
|
1094
|
+
raise RequestError(
|
|
1095
|
+
"pre-finalize semantic records are incomplete: " + ", ".join(missing)
|
|
1096
|
+
+ "; keep this package mutable and repair it before finalize"
|
|
1097
|
+
)
|
|
1098
|
+
|
|
1099
|
+
validator_path = Path(__file__).resolve().with_name("validate_test_evidence.py")
|
|
1100
|
+
try:
|
|
1101
|
+
import importlib.util
|
|
1102
|
+
spec = importlib.util.spec_from_file_location("prizmkit_test_validator_preflight", validator_path)
|
|
1103
|
+
if spec is None or spec.loader is None:
|
|
1104
|
+
raise RequestError("cannot load semantic preflight validator")
|
|
1105
|
+
module = importlib.util.module_from_spec(spec)
|
|
1106
|
+
spec.loader.exec_module(module)
|
|
1107
|
+
records_schema = module.load_schema("authoritative-records.schema.json")
|
|
1108
|
+
matrix_schema = module.load_schema("behavior-risk-matrix.schema.json")
|
|
1109
|
+
errors: list[str] = []
|
|
1110
|
+
definitions = {
|
|
1111
|
+
"change-classification.json": "changeClassification",
|
|
1112
|
+
"scope.json": "scope",
|
|
1113
|
+
"infrastructure-changes.json": "infrastructureChanges",
|
|
1114
|
+
"lightweight-verification.json": "lightweightVerification",
|
|
1115
|
+
}
|
|
1116
|
+
for relative in sorted(semantic_record_paths(change_class)):
|
|
1117
|
+
value = load_json(evidence_dir / relative)
|
|
1118
|
+
if relative == "behavior-risk-matrix.json":
|
|
1119
|
+
module.validate_schema(value, matrix_schema, matrix_schema, "behavior-risk-matrix", errors)
|
|
1120
|
+
else:
|
|
1121
|
+
module.validate_definition(value, definitions[relative], records_schema, relative, errors)
|
|
1122
|
+
if errors:
|
|
1123
|
+
raise RequestError("semantic preflight failed: " + "; ".join(errors[:20]))
|
|
1124
|
+
except ImportError as exc:
|
|
1125
|
+
raise RequestError(f"cannot run semantic preflight: {exc}") from exc
|
|
1126
|
+
|
|
1127
|
+
|
|
1128
|
+
def selected_layer_blockers(plan: dict[str, Any] | None, executions: list[Any]) -> list[str]:
|
|
1129
|
+
if not isinstance(plan, dict):
|
|
1130
|
+
return []
|
|
1131
|
+
selected_layers = {
|
|
1132
|
+
item.get("layer")
|
|
1133
|
+
for item in executions
|
|
1134
|
+
if isinstance(item, dict) and item.get("selected_execution") is True
|
|
1135
|
+
}
|
|
1136
|
+
blockers: list[str] = []
|
|
1137
|
+
for layer in plan.get("layers", []):
|
|
1138
|
+
if not isinstance(layer, dict) or layer.get("required") is not True:
|
|
1139
|
+
continue
|
|
1140
|
+
name = layer.get("name")
|
|
1141
|
+
if name not in selected_layers:
|
|
1142
|
+
blockers.append(f"required layer has no selected execution: {name}")
|
|
1143
|
+
return blockers
|
|
1144
|
+
|
|
1145
|
+
|
|
1007
1146
|
def load_lifecycle(evidence_dir: Path, *, required: bool = False) -> dict[str, Any]:
|
|
1008
1147
|
path = evidence_dir / "lifecycle.json"
|
|
1009
1148
|
if not path.exists():
|
|
@@ -1337,6 +1476,7 @@ def write_handoff_pointer(evidence_dir: Path, artifact_dir: str | None, project_
|
|
|
1337
1476
|
|
|
1338
1477
|
|
|
1339
1478
|
def run_finalize(project_root: Path, evidence_dir: Path, request: dict[str, Any], output_name: str) -> Path:
|
|
1479
|
+
require_protocol_budget(evidence_dir)
|
|
1340
1480
|
allowed = {"request_version", "change_class", "baseline_commit", "scope_path", "environment_path", "artifact_dir", "blockers", "repair_scope"}
|
|
1341
1481
|
if set(request) - allowed:
|
|
1342
1482
|
raise RequestError(f"finalize request contains unsupported fields: {sorted(set(request) - allowed)}")
|
|
@@ -1358,6 +1498,21 @@ def run_finalize(project_root: Path, evidence_dir: Path, request: dict[str, Any]
|
|
|
1358
1498
|
change_class = request.get("change_class", "behavior")
|
|
1359
1499
|
if change_class not in {"behavior", "lightweight"}:
|
|
1360
1500
|
raise RequestError("change_class must be behavior or lightweight")
|
|
1501
|
+
try:
|
|
1502
|
+
preflight_semantic_records(evidence_dir, change_class)
|
|
1503
|
+
except RequestError as exc:
|
|
1504
|
+
state = update_protocol_state(
|
|
1505
|
+
evidence_dir,
|
|
1506
|
+
event="preflight-failed",
|
|
1507
|
+
error=str(exc),
|
|
1508
|
+
increment="preflight_failures",
|
|
1509
|
+
)
|
|
1510
|
+
if state.get("blocked"):
|
|
1511
|
+
raise RequestError(
|
|
1512
|
+
str(exc) + "; evidence protocol recovery budget exhausted; return TEST_BLOCKED "
|
|
1513
|
+
"and do not initialize another package"
|
|
1514
|
+
) from exc
|
|
1515
|
+
raise
|
|
1361
1516
|
artifact_dir = request.get("artifact_dir")
|
|
1362
1517
|
if artifact_dir is not None and not isinstance(artifact_dir, str):
|
|
1363
1518
|
raise RequestError("artifact_dir must be a project-relative directory")
|
|
@@ -1391,8 +1546,10 @@ def run_finalize(project_root: Path, evidence_dir: Path, request: dict[str, Any]
|
|
|
1391
1546
|
"scope_sha256": canonical_sha256(scope),
|
|
1392
1547
|
}
|
|
1393
1548
|
evidence_id = canonical_sha256(identity)
|
|
1549
|
+
destination = evidence_dir.parent / evidence_id
|
|
1394
1550
|
executions = load_json(evidence_dir / "executions.json") if (evidence_dir / "executions.json").exists() else []
|
|
1395
1551
|
blockers = list(request.get("blockers", [])) if isinstance(request.get("blockers", []), list) else []
|
|
1552
|
+
blockers.extend(selected_layer_blockers(plan, executions if isinstance(executions, list) else []))
|
|
1396
1553
|
required_records = {
|
|
1397
1554
|
"change-classification.json", "scope.json", "target-inventory.json", "environment.json",
|
|
1398
1555
|
"source-change.patch",
|
|
@@ -1409,8 +1566,17 @@ def run_finalize(project_root: Path, evidence_dir: Path, request: dict[str, Any]
|
|
|
1409
1566
|
if not (evidence_dir / relative).is_file()
|
|
1410
1567
|
)
|
|
1411
1568
|
if missing_records:
|
|
1412
|
-
|
|
1413
|
-
|
|
1569
|
+
state = update_protocol_state(
|
|
1570
|
+
evidence_dir,
|
|
1571
|
+
event="finalize-precondition-failed",
|
|
1572
|
+
error="required evidence records are missing: " + ", ".join(missing_records),
|
|
1573
|
+
increment="preflight_failures",
|
|
1574
|
+
)
|
|
1575
|
+
suffix = ""
|
|
1576
|
+
if state.get("blocked"):
|
|
1577
|
+
suffix = "; evidence protocol recovery budget exhausted; return TEST_BLOCKED"
|
|
1578
|
+
raise RequestError(
|
|
1579
|
+
"pre-finalize package records are incomplete: " + ", ".join(missing_records) + suffix
|
|
1414
1580
|
)
|
|
1415
1581
|
selected_unreliable = [
|
|
1416
1582
|
item for item in executions
|
|
@@ -1472,13 +1638,15 @@ def run_finalize(project_root: Path, evidence_dir: Path, request: dict[str, Any]
|
|
|
1472
1638
|
"immutable_identity": identity,
|
|
1473
1639
|
})
|
|
1474
1640
|
output = confined(evidence_dir, output_name)
|
|
1641
|
+
final_evidence_relative = destination.relative_to(project_root).as_posix()
|
|
1475
1642
|
write_json_atomic(output, {
|
|
1476
1643
|
"command": "finalize",
|
|
1477
1644
|
"evidence_id": evidence_id,
|
|
1478
|
-
"
|
|
1479
|
-
"
|
|
1480
|
-
"
|
|
1481
|
-
"
|
|
1645
|
+
"evidence_dir": final_evidence_relative,
|
|
1646
|
+
"manifest_path": f"{final_evidence_relative}/manifest.json",
|
|
1647
|
+
"verdict_path": f"{final_evidence_relative}/verdict.json",
|
|
1648
|
+
"report_path": f"{final_evidence_relative}/test-report.md",
|
|
1649
|
+
"validation_path": f"{final_evidence_relative}/validation.json",
|
|
1482
1650
|
"next": "validate_test_evidence.py --attest",
|
|
1483
1651
|
})
|
|
1484
1652
|
# Second manifest: re-hashes after render-report produces test-report.md.
|
|
@@ -1488,7 +1656,6 @@ def run_finalize(project_root: Path, evidence_dir: Path, request: dict[str, Any]
|
|
|
1488
1656
|
baseline_commit=baseline, final_verdict=verdict,
|
|
1489
1657
|
)
|
|
1490
1658
|
write_json_atomic(evidence_dir / "manifest.json", manifest)
|
|
1491
|
-
destination = evidence_dir.parent / evidence_id
|
|
1492
1659
|
if destination != evidence_dir:
|
|
1493
1660
|
if destination.exists():
|
|
1494
1661
|
try:
|
|
@@ -1502,8 +1669,14 @@ def run_finalize(project_root: Path, evidence_dir: Path, request: dict[str, Any]
|
|
|
1502
1669
|
f"evidence identity already exists at {destination}; re-init a new package"
|
|
1503
1670
|
)
|
|
1504
1671
|
os.replace(evidence_dir, destination)
|
|
1505
|
-
|
|
1506
|
-
|
|
1672
|
+
final_evidence_dir = destination if destination != evidence_dir else evidence_dir
|
|
1673
|
+
write_handoff_pointer(final_evidence_dir, lifecycle.get("artifact_dir"), project_root)
|
|
1674
|
+
update_protocol_state(
|
|
1675
|
+
final_evidence_dir,
|
|
1676
|
+
event="finalized",
|
|
1677
|
+
error="; ".join(verdict_record.get("blockers", [])) if verdict == "TEST_BLOCKED" else None,
|
|
1678
|
+
)
|
|
1679
|
+
return final_evidence_dir / output_name
|
|
1507
1680
|
|
|
1508
1681
|
|
|
1509
1682
|
def run_resume(
|
|
@@ -1179,9 +1179,32 @@ def validate_plan_and_scope_bindings(
|
|
|
1179
1179
|
records_schema: dict[str, Any],
|
|
1180
1180
|
errors: list[str],
|
|
1181
1181
|
) -> dict[str, dict[str, Any]]:
|
|
1182
|
-
|
|
1183
|
-
|
|
1184
|
-
|
|
1182
|
+
if not isinstance(scope, dict):
|
|
1183
|
+
errors.append("scope must be an object")
|
|
1184
|
+
if not isinstance(matrix, dict):
|
|
1185
|
+
errors.append("behavior-risk-matrix must be an object")
|
|
1186
|
+
scope_module = scope.get("affected_module") if isinstance(scope, dict) else None
|
|
1187
|
+
matrix_module = matrix.get("affected_module") if isinstance(matrix, dict) else None
|
|
1188
|
+
matrix_module_name = matrix_module.get("name") if isinstance(matrix_module, dict) else None
|
|
1189
|
+
matrix_boundary_source = matrix_module.get("boundary_source") if isinstance(matrix_module, dict) else None
|
|
1190
|
+
require(
|
|
1191
|
+
isinstance(scope_module, str) and scope_module == matrix_module_name,
|
|
1192
|
+
"matrix affected module does not match scope",
|
|
1193
|
+
errors,
|
|
1194
|
+
)
|
|
1195
|
+
require(
|
|
1196
|
+
isinstance(scope.get("boundary_source") if isinstance(scope, dict) else None, str)
|
|
1197
|
+
and scope.get("boundary_source") == matrix_boundary_source,
|
|
1198
|
+
"matrix boundary source does not match scope",
|
|
1199
|
+
errors,
|
|
1200
|
+
)
|
|
1201
|
+
require(
|
|
1202
|
+
scope.get("primary_scope") == matrix.get("primary_scope") if isinstance(scope, dict) and isinstance(matrix, dict) else False,
|
|
1203
|
+
"matrix primary scope does not match scope",
|
|
1204
|
+
errors,
|
|
1205
|
+
)
|
|
1206
|
+
if not isinstance(scope, dict) or not isinstance(matrix, dict) or not isinstance(matrix_module, dict):
|
|
1207
|
+
return {}
|
|
1185
1208
|
scope_ring = sorted((item.get("name"), item.get("kind")) for item in scope.get("regression_ring", []) if isinstance(item, dict))
|
|
1186
1209
|
matrix_ring = sorted((item.get("name"), item.get("kind")) for item in matrix.get("regression_ring", []) if isinstance(item, dict))
|
|
1187
1210
|
require(scope_ring == matrix_ring, "scope and matrix regression rings do not agree", errors)
|
|
@@ -1609,17 +1632,21 @@ def validate_evidence(
|
|
|
1609
1632
|
errors.append(str(exc))
|
|
1610
1633
|
matrix = infrastructure = None
|
|
1611
1634
|
if all(isinstance(item, dict) for item in (matrix, infrastructure, plan)):
|
|
1635
|
+
matrix_error_count = len(errors)
|
|
1612
1636
|
validate_schema(matrix, matrix_schema, matrix_schema, "behavior-risk-matrix", errors)
|
|
1613
1637
|
validate_definition(infrastructure, "infrastructureChanges", records_schema, "infrastructure-changes", errors)
|
|
1614
|
-
|
|
1615
|
-
|
|
1616
|
-
|
|
1617
|
-
|
|
1618
|
-
|
|
1619
|
-
|
|
1620
|
-
|
|
1621
|
-
|
|
1622
|
-
|
|
1638
|
+
if len(errors) > matrix_error_count:
|
|
1639
|
+
errors.append("behavior semantic cross-link validation skipped because behavior-risk-matrix schema validation failed")
|
|
1640
|
+
else:
|
|
1641
|
+
stages = {item.get("name"): item for item in manifest.get("stages", []) if isinstance(item, dict)}
|
|
1642
|
+
for stage_name in CORE_BEHAVIOR_STAGES:
|
|
1643
|
+
require(stages.get(stage_name, {}).get("status") == "complete", f"behavior protocol stage cannot be N/A: {stage_name}", errors)
|
|
1644
|
+
test_map = validate_plan_and_scope_bindings(
|
|
1645
|
+
root, project_root, manifest, scope, matrix, plan, inventory_by_category, entry_map,
|
|
1646
|
+
executions, records_schema, errors,
|
|
1647
|
+
)
|
|
1648
|
+
validate_matrix_risks(manifest, matrix, test_map, executions, records_schema, errors)
|
|
1649
|
+
validate_infrastructure(manifest, infrastructure, executions, entry_map, root, errors)
|
|
1623
1650
|
else:
|
|
1624
1651
|
errors.append("behavior evidence records must be JSON objects")
|
|
1625
1652
|
elif change_class == "lightweight":
|
|
@@ -1756,9 +1783,9 @@ def main() -> int:
|
|
|
1756
1783
|
print(f"- {error}")
|
|
1757
1784
|
return 1
|
|
1758
1785
|
|
|
1759
|
-
print("
|
|
1786
|
+
print("Evidence validation: PASS")
|
|
1787
|
+
print(f"Testing verdict: {manifest['final_verdict']}")
|
|
1760
1788
|
print(f"Evidence ID: {manifest['evidence_id']}")
|
|
1761
|
-
print(f"Verdict: {manifest['final_verdict']}")
|
|
1762
1789
|
return 0
|
|
1763
1790
|
|
|
1764
1791
|
|