prizmkit 1.1.138 → 1.1.140
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bundled/VERSION.json +3 -3
- package/bundled/dev-pipeline/prizmkit_runtime/checkpoint_state.py +1 -1
- package/bundled/dev-pipeline/scripts/prompt_framework.py +9 -5
- package/bundled/dev-pipeline/scripts/update-checkpoint.py +77 -0
- package/bundled/dev-pipeline/templates/sections/phase-prizmkit-test.md +2 -1
- package/bundled/dev-pipeline/tests/test_checkpoint_state.py +100 -0
- package/bundled/skills/_metadata.json +1 -1
- package/bundled/skills/prizmkit-test/SKILL.md +13 -10
- package/bundled/skills/prizmkit-test/assets/authoritative-records.schema.json +2 -1
- package/bundled/skills/prizmkit-test/assets/evidence-manifest.schema.json +1 -1
- package/bundled/skills/prizmkit-test/assets/evidence-package-template.json +1 -0
- package/bundled/skills/prizmkit-test/references/evidence-protocol.md +8 -5
- package/bundled/skills/prizmkit-test/references/evidence-request-protocol.md +5 -2
- package/bundled/skills/prizmkit-test/references/trusted-evidence-execution.md +2 -5
- package/bundled/skills/prizmkit-test/scripts/build_test_evidence.py +279 -27
- package/bundled/skills/prizmkit-test/scripts/validate_test_evidence.py +98 -17
- package/package.json +1 -1
package/bundled/VERSION.json
CHANGED
|
@@ -668,10 +668,10 @@ def _evidence_validator_passes(
|
|
|
668
668
|
|
|
669
669
|
def _resolve_evidence_validator(project_root: Path) -> Path | None:
|
|
670
670
|
candidates = (
|
|
671
|
+
project_root / "core" / "skills" / "prizmkit-skill" / "prizmkit-test" / "scripts" / "validate_test_evidence.py",
|
|
671
672
|
project_root / ".claude" / "command-assets" / "prizmkit-test" / "scripts" / "validate_test_evidence.py",
|
|
672
673
|
project_root / ".agents" / "skills" / "prizmkit-test" / "scripts" / "validate_test_evidence.py",
|
|
673
674
|
project_root / ".codebuddy" / "skills" / "prizmkit-test" / "scripts" / "validate_test_evidence.py",
|
|
674
|
-
project_root / "core" / "skills" / "prizmkit-skill" / "prizmkit-test" / "scripts" / "validate_test_evidence.py",
|
|
675
675
|
)
|
|
676
676
|
return next((path for path in candidates if path.is_file()), None)
|
|
677
677
|
|
|
@@ -117,6 +117,14 @@ def pipeline_checkpoint_metadata(artifact_dir, evidence=None, stage="plan",
|
|
|
117
117
|
blocked_reason=None, terminal_status=None):
|
|
118
118
|
"""Build family-neutral semantic L4 metadata without replacing L1 state."""
|
|
119
119
|
evidence = evidence or {}
|
|
120
|
+
if evidence:
|
|
121
|
+
evidence_paths = {
|
|
122
|
+
"manifest": evidence.get("manifest_path"),
|
|
123
|
+
"verdict": evidence.get("verdict_path"),
|
|
124
|
+
"validation": evidence.get("validation_path"),
|
|
125
|
+
}
|
|
126
|
+
else:
|
|
127
|
+
evidence_paths = {"manifest": None, "verdict": None, "validation": None}
|
|
120
128
|
current_stage = None if terminal_status else stage
|
|
121
129
|
semantic_next_stage = None if terminal_status else next_stage
|
|
122
130
|
return {
|
|
@@ -129,11 +137,7 @@ def pipeline_checkpoint_metadata(artifact_dir, evidence=None, stage="plan",
|
|
|
129
137
|
"stage_result": stage_result,
|
|
130
138
|
"repair_scope": repair_scope,
|
|
131
139
|
"repair_round": repair_round,
|
|
132
|
-
"authoritative_evidence_paths":
|
|
133
|
-
"manifest": evidence.get("manifest_path"),
|
|
134
|
-
"verdict": evidence.get("verdict_path"),
|
|
135
|
-
"validation": evidence.get("validation_path"),
|
|
136
|
-
},
|
|
140
|
+
"authoritative_evidence_paths": evidence_paths,
|
|
137
141
|
"next_stage": semantic_next_stage,
|
|
138
142
|
"resume_from": resume_from,
|
|
139
143
|
"blocked_reason": blocked_reason,
|
|
@@ -213,6 +213,70 @@ def _checkpoint_project_root(checkpoint_path):
|
|
|
213
213
|
return str(parent.parent)
|
|
214
214
|
return str(path.parent)
|
|
215
215
|
|
|
216
|
+
def _discover_test_evidence_paths(checkpoint_path, data):
|
|
217
|
+
"""Discover the latest finalized test package from the artifact handoff pointer."""
|
|
218
|
+
project_root = Path(_checkpoint_project_root(checkpoint_path))
|
|
219
|
+
candidates = []
|
|
220
|
+
artifact_dir = data.get("artifact_dir")
|
|
221
|
+
if isinstance(artifact_dir, str) and artifact_dir:
|
|
222
|
+
candidates.append(project_root / artifact_dir)
|
|
223
|
+
for state_key in ("feature_state", "bugfix_state", "refactor_state"):
|
|
224
|
+
state = data.get(state_key)
|
|
225
|
+
if isinstance(state, dict) and isinstance(state.get("artifact_dir"), str):
|
|
226
|
+
candidates.append(project_root / state["artifact_dir"])
|
|
227
|
+
item_slug = data.get("item_slug")
|
|
228
|
+
if isinstance(item_slug, str) and item_slug:
|
|
229
|
+
candidates.append(project_root / ".prizmkit" / "specs" / item_slug)
|
|
230
|
+
candidates.append(project_root / ".prizmkit" / "bugfix" / item_slug)
|
|
231
|
+
candidates.append(project_root / ".prizmkit" / "refactor" / item_slug)
|
|
232
|
+
|
|
233
|
+
for artifact in candidates:
|
|
234
|
+
pointer = artifact / "test-report-path.txt"
|
|
235
|
+
try:
|
|
236
|
+
report = Path(pointer.read_text(encoding="utf-8").strip()).resolve()
|
|
237
|
+
except (OSError, ValueError):
|
|
238
|
+
continue
|
|
239
|
+
evidence_dir = report.parent
|
|
240
|
+
evidence_root = (project_root / ".prizmkit" / "test" / "evidence").resolve()
|
|
241
|
+
try:
|
|
242
|
+
relative_evidence = evidence_dir.resolve().relative_to(evidence_root)
|
|
243
|
+
except (OSError, ValueError):
|
|
244
|
+
continue
|
|
245
|
+
if len(relative_evidence.parts) != 1:
|
|
246
|
+
continue
|
|
247
|
+
paths = {
|
|
248
|
+
"manifest": evidence_dir / "manifest.json",
|
|
249
|
+
"verdict": evidence_dir / "verdict.json",
|
|
250
|
+
"validation": evidence_dir / "validation.json",
|
|
251
|
+
}
|
|
252
|
+
if not all(path.is_file() for path in paths.values()):
|
|
253
|
+
continue
|
|
254
|
+
try:
|
|
255
|
+
manifest = json.loads(paths["manifest"].read_text(encoding="utf-8"))
|
|
256
|
+
verdict = json.loads(paths["verdict"].read_text(encoding="utf-8"))
|
|
257
|
+
validation = json.loads(paths["validation"].read_text(encoding="utf-8"))
|
|
258
|
+
except (OSError, UnicodeDecodeError, json.JSONDecodeError):
|
|
259
|
+
continue
|
|
260
|
+
if not all(isinstance(record, dict) for record in (manifest, verdict, validation)):
|
|
261
|
+
continue
|
|
262
|
+
if (
|
|
263
|
+
manifest.get("evidence_id") != evidence_dir.name
|
|
264
|
+
or manifest.get("final_verdict") != "TEST_BLOCKED"
|
|
265
|
+
or verdict.get("verdict") != "TEST_BLOCKED"
|
|
266
|
+
or validation.get("result") != "passed"
|
|
267
|
+
or validation.get("verdict") != "TEST_BLOCKED"
|
|
268
|
+
):
|
|
269
|
+
continue
|
|
270
|
+
try:
|
|
271
|
+
return {
|
|
272
|
+
key: str(path.resolve().relative_to(project_root.resolve()))
|
|
273
|
+
for key, path in paths.items()
|
|
274
|
+
}
|
|
275
|
+
except (OSError, ValueError):
|
|
276
|
+
continue
|
|
277
|
+
return None
|
|
278
|
+
|
|
279
|
+
|
|
216
280
|
def _finalize_semantics(checkpoint_path, data, state_key):
|
|
217
281
|
"""Derive a deterministic terminal status after an atomic mutation."""
|
|
218
282
|
candidate = validate_checkpoint_data(
|
|
@@ -286,11 +350,24 @@ def update_checkpoint(
|
|
|
286
350
|
return {"ok": False, "error": verdict_error}
|
|
287
351
|
|
|
288
352
|
semantic_update = _normalize_semantic_update(semantic_update)
|
|
353
|
+
if semantic_update.get("stage_result") == "TEST_BLOCKED" and not semantic_update.get("authoritative_evidence_paths"):
|
|
354
|
+
discovered = _discover_test_evidence_paths(checkpoint_path, data)
|
|
355
|
+
if discovered:
|
|
356
|
+
semantic_update["authoritative_evidence_paths"] = discovered
|
|
289
357
|
semantic_error = _validate_semantic_update(step, semantic_update)
|
|
290
358
|
if semantic_error:
|
|
291
359
|
return {"ok": False, "error": semantic_error}
|
|
292
360
|
|
|
293
361
|
stage_result = semantic_update.get("stage_result")
|
|
362
|
+
if step.get("skill") == "prizmkit-test" and stage_result in TEST_RESULTS:
|
|
363
|
+
evidence = semantic_update.get("authoritative_evidence_paths") or step.get("authoritative_evidence_paths")
|
|
364
|
+
if stage_result != "TEST_BLOCKED" and (
|
|
365
|
+
not isinstance(evidence, dict) or set(evidence) != {"manifest", "verdict", "validation"}
|
|
366
|
+
):
|
|
367
|
+
return {
|
|
368
|
+
"ok": False,
|
|
369
|
+
"error": "Every terminal test verdict requires manifest, verdict, and validation evidence paths",
|
|
370
|
+
}
|
|
294
371
|
if new_status == "completed" and step.get("skill") == "prizmkit-code-review":
|
|
295
372
|
stage_result = stage_result or step.get("stage_result")
|
|
296
373
|
if stage_result != "REVIEW_PASS":
|
|
@@ -33,7 +33,8 @@ Preserve the same artifact directory and update the L1 workflow state plus this
|
|
|
33
33
|
- `TEST_FAIL` with `repair_scope=test-infrastructure` → `prizmkit-implement` → `prizmkit-test`; do not make speculative production edits.
|
|
34
34
|
- `TEST_FAIL` with `repair_scope=production|runtime|schema|dependency|public-interface` → `prizmkit-implement` → `prizmkit-code-review` → `prizmkit-test`.
|
|
35
35
|
- `TEST_FAIL` with unknown or unsafe scope → do not guess a production edit; record `WORKFLOW_BLOCKED` and a safe resume entry.
|
|
36
|
-
- `TEST_BLOCKED` → preserve all evidence and artifacts, perform only bounded environment recovery, then finish terminal `WORKFLOW_BLOCKED` with `blocked_reason=test_blocked`; never edit production speculatively.
|
|
36
|
+
- `TEST_BLOCKED` → preserve all evidence and artifacts, perform only bounded environment recovery, then finish terminal `WORKFLOW_BLOCKED` with `blocked_reason=test_blocked`; never edit production speculatively. When a finalized blocked package exists, pass its validated `manifest`, `verdict`, and `validation` paths to the checkpoint; do not leave these paths null.
|
|
37
|
+
- The builder's evidence-protocol recovery budget is separate from the outer `repair_round`: after bounded preflight/package attempts, stop creating evidence versions and return `TEST_BLOCKED`.
|
|
37
38
|
- The outer `repair_round` is shared across review/test failures and may not exceed three. The Code Review internal loop limit remains separate and is at most ten completed rounds.
|
|
38
39
|
|
|
39
40
|
A missing, stale, incomplete, mismatched, tampered, or wrong-scope package is not `TEST_PASS`: preserve the evidence, write `failure-log.md`, and leave the checkpoint resumable or blocked according to headless policy.
|
|
@@ -440,6 +440,106 @@ def test_declared_final_artifact_must_exist_for_semantic_completion(tmp_path, mo
|
|
|
440
440
|
|
|
441
441
|
|
|
442
442
|
|
|
443
|
+
def test_blocked_test_handoff_discovers_finalized_evidence_paths(tmp_path):
|
|
444
|
+
artifact = tmp_path / ".prizmkit" / "specs" / "feature"
|
|
445
|
+
artifact.mkdir(parents=True)
|
|
446
|
+
evidence = tmp_path / ".prizmkit" / "test" / "evidence" / "evidence-1"
|
|
447
|
+
evidence.mkdir(parents=True)
|
|
448
|
+
(evidence / "manifest.json").write_text(
|
|
449
|
+
json.dumps({"evidence_id": "evidence-1", "final_verdict": "TEST_BLOCKED"}),
|
|
450
|
+
encoding="utf-8",
|
|
451
|
+
)
|
|
452
|
+
(evidence / "verdict.json").write_text(
|
|
453
|
+
json.dumps({"verdict": "TEST_BLOCKED"}),
|
|
454
|
+
encoding="utf-8",
|
|
455
|
+
)
|
|
456
|
+
(evidence / "validation.json").write_text(
|
|
457
|
+
json.dumps({"result": "passed", "verdict": "TEST_BLOCKED"}),
|
|
458
|
+
encoding="utf-8",
|
|
459
|
+
)
|
|
460
|
+
(evidence / "lifecycle.json").write_text(
|
|
461
|
+
json.dumps({"status": "sealed", "finalized": True}),
|
|
462
|
+
encoding="utf-8",
|
|
463
|
+
)
|
|
464
|
+
report = evidence / "test-report.md"
|
|
465
|
+
report.write_text("blocked", encoding="utf-8")
|
|
466
|
+
(artifact / "test-report-path.txt").write_text(str(report), encoding="utf-8")
|
|
467
|
+
|
|
468
|
+
payload = _family_checkpoint(
|
|
469
|
+
"feature-pipeline",
|
|
470
|
+
[
|
|
471
|
+
"prizmkit-implement", "prizmkit-code-review", "prizmkit-test",
|
|
472
|
+
"prizmkit-retrospective", "prizmkit-committer", "completion-summary",
|
|
473
|
+
],
|
|
474
|
+
semantic={"artifact_dir": ".prizmkit/specs/feature"},
|
|
475
|
+
)
|
|
476
|
+
payload["feature_state"]["artifact_dir"] = ".prizmkit/specs/feature"
|
|
477
|
+
payload["steps"][1]["stage_result"] = "REVIEW_PASS"
|
|
478
|
+
payload["steps"][2]["status"] = "in_progress"
|
|
479
|
+
payload["steps"][2].pop("stage_result")
|
|
480
|
+
for step in payload["steps"][3:]:
|
|
481
|
+
step["status"] = "pending"
|
|
482
|
+
step.pop("stage_result", None)
|
|
483
|
+
checkpoint = _write(tmp_path / ".prizmkit" / "specs" / "feature" / "workflow-checkpoint.json", payload)
|
|
484
|
+
|
|
485
|
+
result = update_checkpoint(
|
|
486
|
+
str(checkpoint),
|
|
487
|
+
"prizmkit-test",
|
|
488
|
+
"failed",
|
|
489
|
+
semantic_update={
|
|
490
|
+
"stage_result": "TEST_BLOCKED",
|
|
491
|
+
"blocked_reason": "test_blocked",
|
|
492
|
+
"terminal_status": "WORKFLOW_BLOCKED",
|
|
493
|
+
},
|
|
494
|
+
)
|
|
495
|
+
|
|
496
|
+
assert result["ok"] is True
|
|
497
|
+
paths = json.loads(checkpoint.read_text(encoding="utf-8"))["steps"][2]["authoritative_evidence_paths"]
|
|
498
|
+
assert paths["manifest"] == ".prizmkit/test/evidence/evidence-1/manifest.json"
|
|
499
|
+
assert paths["verdict"] == ".prizmkit/test/evidence/evidence-1/verdict.json"
|
|
500
|
+
assert paths["validation"] == ".prizmkit/test/evidence/evidence-1/validation.json"
|
|
501
|
+
|
|
502
|
+
|
|
503
|
+
|
|
504
|
+
def test_blocked_test_handoff_without_finalized_evidence_preserves_test_result(tmp_path):
|
|
505
|
+
artifact = tmp_path / ".prizmkit" / "specs" / "feature"
|
|
506
|
+
artifact.mkdir(parents=True)
|
|
507
|
+
payload = _family_checkpoint(
|
|
508
|
+
"feature-pipeline",
|
|
509
|
+
[
|
|
510
|
+
"prizmkit-implement", "prizmkit-code-review", "prizmkit-test",
|
|
511
|
+
"prizmkit-retrospective", "prizmkit-committer", "completion-summary",
|
|
512
|
+
],
|
|
513
|
+
semantic={"artifact_dir": ".prizmkit/specs/feature"},
|
|
514
|
+
)
|
|
515
|
+
payload["feature_state"]["artifact_dir"] = ".prizmkit/specs/feature"
|
|
516
|
+
payload["steps"][1]["stage_result"] = "REVIEW_PASS"
|
|
517
|
+
payload["steps"][2]["status"] = "in_progress"
|
|
518
|
+
payload["steps"][2].pop("stage_result")
|
|
519
|
+
for step in payload["steps"][3:]:
|
|
520
|
+
step["status"] = "pending"
|
|
521
|
+
step.pop("stage_result", None)
|
|
522
|
+
checkpoint = _write(artifact / "workflow-checkpoint.json", payload)
|
|
523
|
+
|
|
524
|
+
result = update_checkpoint(
|
|
525
|
+
str(checkpoint),
|
|
526
|
+
"prizmkit-test",
|
|
527
|
+
"failed",
|
|
528
|
+
semantic_update={
|
|
529
|
+
"stage_result": "TEST_BLOCKED",
|
|
530
|
+
"blocked_reason": "test_blocked",
|
|
531
|
+
"terminal_status": "WORKFLOW_BLOCKED",
|
|
532
|
+
},
|
|
533
|
+
)
|
|
534
|
+
|
|
535
|
+
persisted = json.loads(checkpoint.read_text(encoding="utf-8"))
|
|
536
|
+
assert result["ok"] is True
|
|
537
|
+
assert result["semantic"]["stage_result"] == "TEST_BLOCKED"
|
|
538
|
+
assert persisted["steps"][2]["stage_result"] == "TEST_BLOCKED"
|
|
539
|
+
assert persisted["feature_state"]["stage_result"] == "TEST_BLOCKED"
|
|
540
|
+
assert "authoritative_evidence_paths" not in persisted["steps"][2]
|
|
541
|
+
|
|
542
|
+
|
|
443
543
|
def test_update_checkpoint_rejects_test_pass_when_validator_replay_fails(tmp_path, monkeypatch):
|
|
444
544
|
paths = _write_test_evidence(tmp_path)
|
|
445
545
|
payload = _family_checkpoint(
|
|
@@ -20,23 +20,23 @@ Fix only the protocol mechanisms needed for safe, replayable evidence: project/e
|
|
|
20
20
|
The Main Agent supplies semantic requests only; it does not author authoritative package records. Initialize and drive one evidence directory through the builder in this order:
|
|
21
21
|
|
|
22
22
|
```text
|
|
23
|
-
init → capture-change → inventory → prepare-tests → execute → finalize →
|
|
23
|
+
init → capture-change → inventory → prepare-tests → execute → finalize → validate --attest → resume
|
|
24
24
|
```
|
|
25
25
|
|
|
26
26
|
Use `${SKILL_DIR}/scripts/build_test_evidence.py` for every lifecycle command:
|
|
27
27
|
|
|
28
|
-
- `init`: creates the package directories and
|
|
28
|
+
- `init`: creates the package directories and pins the baseline identity.
|
|
29
29
|
- `capture-change`: captures the complete canonical change, including untracked, deleted, renamed, and copied files; never substitute plain `git diff`. `inventory` then requires its changed-file set to match the capture exactly and rejects stale captures or unaccounted live paths.
|
|
30
30
|
- `inventory`: hashes current source/tests/contracts/lockfiles, rejects category overlap, excludes runtime-only paths, and validates directory roots plus the complete capture-derived changed-file set.
|
|
31
31
|
- `prepare-tests`: discovers repository-native test files from the canonical capture and inventory, generates stable builder-owned test IDs, derives existing/added/modified status, and snapshots only added/modified live test sources.
|
|
32
32
|
- `execute`: runs native project commands and owns `executions.json`, `receipts/`, and `raw/`. Each request uses exactly one binding mode: `test_ids`, `test_paths`, or `auto_bind=true`. The builder infers recognized package-wide or explicit file/directory/glob command scope, blocks opaque commands or mismatched declarations, and persists verified canonical `test_ids` only.
|
|
33
|
-
- `finalize`: rechecks canonical capture freshness, blocks every selected unreliable execution, and owns
|
|
34
|
-
- `render-report`: derives `test-report.md`; it is never authoritative.
|
|
33
|
+
- `finalize`: derives builder-owned target hashes, runs semantic-record and required-layer checks, rechecks canonical capture freshness, blocks every selected unreliable execution, and owns the payload manifest, verdict, generated-test snapshots, and evidence identity. It leaves the package in a mutable `payload-finalized` state until validator attestation; malformed semantic records fail before package relocation.
|
|
34
|
+
- `render-report`: recovery/debugging command that derives `test-report.md`; normal finalization and attestation render it automatically, and it is never authoritative.
|
|
35
35
|
- `resume`: preserves prior receipts and refuses to rebind an identity-changing package; create a new package after scope/source identity changes.
|
|
36
36
|
|
|
37
37
|
`test-plan.json` is the only source of builder-generated test IDs. An execution request provides exactly one of native `test_paths`, `auto_bind=true`, or canonical `test_ids`; the actual command must expose the same deterministic scope, including all prepared tests for recognized package-wide commands. The builder blocks opaque or mismatched command scope and rewrites verified requests to canonical `test_ids` only before execution. Assess all five layers explicitly, but set `required=true` only when risk requires it; every omitted layer needs structured N/A evidence. The same binding applies test ↔ behavior ↔ execution; current-workspace receipts are the execution evidence.
|
|
38
38
|
|
|
39
|
-
The builder/validator, not the Main Agent, produces or updates `manifest.json`, `executions.json`, `receipts/*`, `validation.json`, `verdict.json`, `test-report.md`, and `source-change.patch`. `validation.json` is produced only by `validate_test_evidence.py --attest`; a package is not a `TEST_PASS` until the second strict validation succeeds.
|
|
39
|
+
The builder/validator, not the Main Agent, produces or updates `manifest.json`, `executions.json`, `receipts/*`, `validation.json`, `verdict.json`, `test-report.md`, and `source-change.patch`. Semantic records are authored through schema-shaped requests and must pass the builder preflight before finalization. `validation.json` is produced only by `validate_test_evidence.py --attest`; a package is not a `TEST_PASS` until the second strict validation succeeds.
|
|
40
40
|
|
|
41
41
|
|
|
42
42
|
For a formal requirement, `/prizmkit-test` runs after the Main-Agent code-review stage reaches `PASS` and before retrospective:
|
|
@@ -68,7 +68,8 @@ Documentation, comment, and formatting-only scopes use deterministic lightweight
|
|
|
68
68
|
3. Preserve complete command/environment/output values as requested. Mark the package `sensitivity=project-controlled`; the project owns access control, retention, and upload policy.
|
|
69
69
|
4. Treat code-level dependencies as mock-first. Real deployed test-environment validation is a separate authorized activity.
|
|
70
70
|
5. Never delete existing tests.
|
|
71
|
-
6.
|
|
71
|
+
6. Evidence-protocol recovery is separately bounded: allow at most two package initializations and two semantic preflight failures for one canonical change identity. After the budget is exhausted, return `TEST_BLOCKED`; do not create another versioned package or hand-edit finalized evidence.
|
|
72
|
+
7. For `TEST_FAIL`, classify and persist an evidence-backed `repair_scope` of `test-infrastructure`, `production`, `runtime`, `schema`, `dependency`, `public-interface`, or `unknown`. Test-infrastructure scope routes to implement then test; production-family scope routes to implement, code-review, then test. If evidence cannot support a safe classification, return `TEST_BLOCKED`.
|
|
72
73
|
|
|
73
74
|
If test infrastructure is missing, the infrastructure preparation state establishes only the necessary project-native infrastructure autonomously. Do not ask the user to install routine test dependencies in a headless run; verify dependency versions before editing manifests.
|
|
74
75
|
|
|
@@ -216,9 +217,11 @@ python3 ${SKILL_DIR}/scripts/validate_test_evidence.py \
|
|
|
216
217
|
--project-root <target-project-root>
|
|
217
218
|
```
|
|
218
219
|
|
|
219
|
-
|
|
220
|
+
The first pass validates the payload while the package remains `payload-finalized`; `validation.json` is intentionally empty until attestation. Builder-derived target hashes and lifecycle records are generated before this check. If this pass finds semantic errors, repair the mutable staging package and rerun finalize; never repair a sealed package.
|
|
220
221
|
|
|
221
|
-
|
|
222
|
+
The validator checks every authoritative record against its shipped schema, live target inventories, patch/diff binding, changed-file/module-root/exclusion/Regression Ring cross-links, generated-test snapshots, planned-test execution links, runner-generated receipts/request/raw-output hashes, successful behavior-risk mappings, production/unknown external-target safety, and package integrity.
|
|
223
|
+
|
|
224
|
+
### Second pass (attestation)
|
|
222
225
|
|
|
223
226
|
```bash
|
|
224
227
|
python3 ${SKILL_DIR}/scripts/validate_test_evidence.py \
|
|
@@ -227,7 +230,7 @@ python3 ${SKILL_DIR}/scripts/validate_test_evidence.py \
|
|
|
227
230
|
--attest
|
|
228
231
|
```
|
|
229
232
|
|
|
230
|
-
`--attest` writes an integrity/protocol attestation into `validation.json`, re-derives `test-report.md`, re-hashes final records in `manifest.json`, then runs a **second** strict validation against the refreshed package. A package is not a `TEST_PASS` until this second strict validation succeeds. Use builder replay when feasible to rerun recorded requests and create new linked receipts. Store validation output. Tampering, schema/identity/hash mismatch, source/test/lockfile drift, missing matrix mappings, wrong execution order, unjustified structured N/A, unresolved risk, or a false real-environment claim fails validation.
|
|
233
|
+
`--attest` writes an integrity/protocol attestation into `validation.json`, marks `EVIDENCE_VALIDATE` complete, re-derives `test-report.md`, re-hashes final records in `manifest.json`, seals the package, and then runs a **second** strict validation against the refreshed package. A package is not a `TEST_PASS` until this second strict validation succeeds. Use builder replay when feasible to rerun recorded requests and create new linked receipts. Store validation output. Tampering, schema/identity/hash mismatch, source/test/lockfile drift, missing matrix mappings, wrong execution order, unjustified structured N/A, unresolved risk, or a false real-environment claim fails validation.
|
|
231
234
|
|
|
232
235
|
## Resume and Idempotency
|
|
233
236
|
|
|
@@ -255,7 +258,7 @@ No conditional pass exists.
|
|
|
255
258
|
|
|
256
259
|
Before writing state, read `${SKILL_DIR}/references/workflow-state-protocol.md`.
|
|
257
260
|
|
|
258
|
-
On `TEST_PASS`, update the active workflow state with `status=TEST_PASS`, `stage_result=TEST_PASS`, and `next_stage=retrospective`. On `TEST_FAIL`, persist the evidence-backed `repair_scope`, set `status=TEST_FAIL` and `stage_result=TEST_FAIL`, and return to implementation; test-infrastructure scope selects `next_stage=implement` with a direct test resume, while production-family scope selects `next_stage=implement` and requires code review before testing again. On `TEST_BLOCKED`, preserve the blocker, do not modify production code speculatively, and return a blocked result. The authoritative verdict is the validated evidence record; the derived Markdown report is only a compatibility view. If an active orchestrator exists, return the terminal result and `next_stage` to it; do not invoke the next skill independently.
|
|
261
|
+
On `TEST_PASS`, update the active workflow state with `status=TEST_PASS`, `stage_result=TEST_PASS`, and `next_stage=retrospective`. On `TEST_FAIL`, persist the evidence-backed `repair_scope`, set `status=TEST_FAIL` and `stage_result=TEST_FAIL`, and return to implementation; test-infrastructure scope selects `next_stage=implement` with a direct test resume, while production-family scope selects `next_stage=implement` and requires code review before testing again. On `TEST_BLOCKED`, preserve the blocker and any validated finalized evidence paths, do not modify production code speculatively, and return a blocked result. A validator `PASS` means only that the evidence package is structurally valid; it does not mean the testing verdict passed. The authoritative verdict is the validated evidence record; the derived Markdown report is only a compatibility view. If an active orchestrator exists, return the terminal result and `next_stage` to it; do not invoke the next skill independently.
|
|
259
262
|
|
|
260
263
|
## Derived Report and Handoff
|
|
261
264
|
|
|
@@ -362,7 +362,7 @@
|
|
|
362
362
|
"executionReceipt": {
|
|
363
363
|
"type": "object",
|
|
364
364
|
"additionalProperties": false,
|
|
365
|
-
"required": ["receipt_format", "runner_path", "runner_sha256", "execution_id", "attempt_index", "request_path", "request_sha256", "purpose", "command", "cwd", "environment", "tool_versions", "layer", "test_ids", "external_targets", "exit_code", "stdout_path", "stderr_path", "stdout_sha256", "stderr_sha256", "selected_execution", "reliable", "started_at", "finished_at"],
|
|
365
|
+
"required": ["receipt_format", "runner_path", "runner_sha256", "execution_id", "attempt_index", "request_path", "request_sha256", "purpose", "command", "cwd", "environment", "tool_versions", "layer", "test_ids", "external_targets", "timeout_seconds", "exit_code", "stdout_path", "stderr_path", "stdout_sha256", "stderr_sha256", "selected_execution", "reliable", "started_at", "finished_at"],
|
|
366
366
|
"properties": {
|
|
367
367
|
"receipt_format": { "const": "prizmkit-runner-generated-v1" },
|
|
368
368
|
"runner_path": { "$ref": "#/$defs/path" },
|
|
@@ -381,6 +381,7 @@
|
|
|
381
381
|
"test_paths": { "type": "array", "items": { "$ref": "#/$defs/path" } },
|
|
382
382
|
"auto_bind": { "type": "boolean" },
|
|
383
383
|
"external_targets": { "type": "array", "items": { "$ref": "#/$defs/externalTarget" } },
|
|
384
|
+
"timeout_seconds": { "type": "number", "exclusiveMinimum": 0 },
|
|
384
385
|
"exit_code": { "type": "integer" },
|
|
385
386
|
"stdout_path": { "$ref": "#/$defs/path" },
|
|
386
387
|
"stderr_path": { "$ref": "#/$defs/path" },
|
|
@@ -82,7 +82,7 @@
|
|
|
82
82
|
"EVIDENCE_VALIDATE"
|
|
83
83
|
]
|
|
84
84
|
},
|
|
85
|
-
"status": { "enum": ["complete", "not_applicable"] },
|
|
85
|
+
"status": { "enum": ["complete", "pending", "not_applicable"] },
|
|
86
86
|
"outputs": { "type": "array", "items": { "type": "string", "minLength": 1 } },
|
|
87
87
|
"not_applicable": {
|
|
88
88
|
"type": ["object", "null"],
|
|
@@ -22,11 +22,14 @@ Hashes and validator receipts provide byte-level binding and drift detection, no
|
|
|
22
22
|
|
|
23
23
|
`source-change.patch` preserves the complete target diff. Snapshot generated/changed tests under `generated-tests/`, contracts under `contracts/`, and raw stdout/stderr under `raw/`. Never replace old execution/output records.
|
|
24
24
|
|
|
25
|
-
##
|
|
25
|
+
## Bounded Protocol Recovery
|
|
26
|
+
|
|
27
|
+
Evidence-protocol repair is separate from production/test repair. The builder permits only a bounded recovery budget for semantic-record preflight and package initialization. After the budget is exhausted, return `TEST_BLOCKED`; do not create another versioned evidence package or hand-edit authoritative records. A payload-finalized package remains mutable only until validator attestation; after attestation it is sealed and immutable. `Evidence validation: PASS` means only that the package is structurally valid; the testing verdict must be read separately from `Testing verdict: TEST_PASS | TEST_FAIL | TEST_BLOCKED`.
|
|
28
|
+
|
|
26
29
|
|
|
27
30
|
The evidence package is an audit of repository behavior, not a request to invent a parallel test suite. Discover and execute the smallest relevant native tests before adding anything. Add, modify, or delete tests only in the target repository. The builder's `prepare-tests` command derives stable test IDs and change status from canonical Git capture, and snapshots only added or modified repository-native tests. `generated-tests/` is an evidence archive, never an execution workspace; it may be empty when existing tests cover the behavior.
|
|
28
31
|
|
|
29
|
-
The canonical package lifecycle is `init → capture-change → inventory → prepare-tests → execute → finalize →
|
|
32
|
+
The canonical package lifecycle is `init → capture-change → inventory → prepare-tests → execute → finalize → validate --attest → resume`. Builder commands own `source-change.patch`, native test preparation, receipts, raw outputs, manifest, verdict, and initial report. The validator alone owns `validation.json` and refreshes the report during attestation. Inventory must consume a fresh capture and exactly preserve its complete changed-file set. Canonical behavior packages require a schema-valid `test-preparation.json` bound byte-for-byte to the builder-generated `test-plan.json` and its hashed semantic request. Evidence identity is derived from the resolved baseline, canonical patch hash, and scope hash; identity-changing resume creates a new package instead of rewriting the old directory.
|
|
30
33
|
|
|
31
34
|
|
|
32
35
|
Run exactly these states in order:
|
|
@@ -164,12 +167,12 @@ python3 ${SKILL_DIR}/scripts/validate_test_evidence.py \
|
|
|
164
167
|
--attest
|
|
165
168
|
```
|
|
166
169
|
|
|
167
|
-
`--attest` first validates the
|
|
168
|
-
|
|
169
|
-
When feasible, replay recorded requests through `build_test_evidence.py execute --replay-receipt ...`. Replay performs a new real execution and emits a new linked receipt; it does not pretend the attestation establishes what historically ran.
|
|
170
|
+
`--attest` first validates the mutable `payload-finalized` package while excluding only validator-owned attestation/view records, atomically writes `validation.json`, marks validation complete, refreshes final file hashes, seals the lifecycle, then immediately performs a complete second validation. Subsequent checks omit `--attest` and verify the stored attestation. This avoids trusting a caller-authored `passed` marker or creating a self-referential manifest. The attestation proves package integrity and protocol consistency only; it is not proof against a same-permission producer.
|
|
170
171
|
|
|
171
172
|
The validator checks all shipped JSON schemas, evidence-directory identity, live target-project inventories and aggregate hashes, changed-file/module-root/exclusion/Regression Ring cross-links, patch/diff binding, stage ordering and output ownership, generated-test snapshot existence, planned test/inventory/execution linkage, module/matrix completeness, successful selected runner receipts with request/raw-output binding, structured N/A and obvious signal conflicts, production/unknown external-target safety, strict verdict semantics, and honest mocked-versus-real claims. `TEST_PASS` requires deterministic validator success.
|
|
172
173
|
|
|
174
|
+
When feasible, replay recorded requests through `build_test_evidence.py execute --replay-receipt ...`. Replay performs a new real execution and emits a new linked receipt; it does not pretend the attestation establishes what historically ran.
|
|
175
|
+
|
|
173
176
|
## Resume and Invalidation
|
|
174
177
|
|
|
175
178
|
On invocation for an existing evidence target:
|
|
@@ -15,7 +15,10 @@ The model writes project-semantic requests after inspecting manifests, runner co
|
|
|
15
15
|
|
|
16
16
|
Never hand-author `executions.json` receipt fields. Never copy a command's claimed output into `raw/`. Invoke the builder so provenance is linked to the actual process.
|
|
17
17
|
|
|
18
|
-
##
|
|
18
|
+
## Producer Boundary and Recovery
|
|
19
|
+
|
|
20
|
+
The model supplies schema-shaped semantic requests and mutable semantic records only before payload finalization. The builder performs structural and derived-field preflight before creating a payload-finalized package. Validator attestation then seals it. Once sealed, the returned `evidence_dir`, `manifest_path`, `verdict_path`, and `validation_path` are the only valid handoff paths; do not continue using the old staging directory. A validator pass is not a testing pass: read the testing verdict separately.
|
|
21
|
+
|
|
19
22
|
|
|
20
23
|
Place a request under `requests/` with:
|
|
21
24
|
|
|
@@ -38,7 +41,7 @@ Use an argv array so the runner does not interpret a shell string. Required fiel
|
|
|
38
41
|
- selected `layer` and exactly one canonical binding (`test_ids`, `test_paths`, or `auto_bind: true`);
|
|
39
42
|
- structured `external_targets`.
|
|
40
43
|
|
|
41
|
-
Optional `timeout_seconds`, `attempt_policy`, and `concurrency` remain model-chosen. The builder
|
|
44
|
+
Optional `timeout_seconds`, `attempt_policy`, and `concurrency` remain model-chosen. The builder applies a bounded default and maximum timeout; it does not silently retry or claim concurrency that it did not execute.
|
|
42
45
|
|
|
43
46
|
The request path must be beneath the evidence directory. `cwd` resolves beneath the selected execution root. Missing commands produce an actual exit 127 receipt; missing cwd or escaped paths block before execution.
|
|
44
47
|
|
|
@@ -41,7 +41,7 @@ Treat evidence as auditable project-controlled provenance, not a defense against
|
|
|
41
41
|
|
|
42
42
|
## Builder Commands
|
|
43
43
|
|
|
44
|
-
Drive a new package through the canonical lifecycle. Semantic request files remain model-authored; authoritative package records remain tool-owned
|
|
44
|
+
Drive a new package through the canonical lifecycle. Semantic request files remain model-authored; authoritative package records remain tool-owned. The builder applies a bounded default timeout when the request omits one, preserves common native cache variables, and returns a bounded unreliable receipt on timeout. The normal user path should author one semantic request and one execution plan; use individual lifecycle commands only for recovery or protocol debugging.
|
|
45
45
|
|
|
46
46
|
```bash
|
|
47
47
|
python3 ${SKILL_DIR}/scripts/build_test_evidence.py --project-root <root> --evidence-dir <staging> init --baseline <commit>
|
|
@@ -49,14 +49,11 @@ python3 ${SKILL_DIR}/scripts/build_test_evidence.py --project-root <root> --evid
|
|
|
49
49
|
python3 ${SKILL_DIR}/scripts/build_test_evidence.py --project-root <root> --evidence-dir <staging> inventory --request requests/inventory.json
|
|
50
50
|
python3 ${SKILL_DIR}/scripts/build_test_evidence.py --project-root <root> --evidence-dir <staging> execute --request requests/focused.json
|
|
51
51
|
python3 ${SKILL_DIR}/scripts/build_test_evidence.py --project-root <root> --evidence-dir <staging> finalize --request requests/finalize.json
|
|
52
|
-
python3 ${SKILL_DIR}/scripts/build_test_evidence.py --project-root <root> --evidence-dir <staging> render-report
|
|
53
52
|
```
|
|
54
53
|
|
|
55
|
-
|
|
54
|
+
`<staging>` must be a child of `.prizmkit/test/evidence/`. After `finalize`, the package is `payload-finalized`; run the validator pre-check and then `validate_test_evidence.py --attest` to seal it. Do not treat the pre-attestation staging package as immutable or invoke `render-report` after sealing.
|
|
56
55
|
|
|
57
56
|
|
|
58
|
-
Write requests beneath the evidence directory, then invoke the bundled script:
|
|
59
|
-
|
|
60
57
|
```bash
|
|
61
58
|
python3 ${SKILL_DIR}/scripts/build_test_evidence.py \
|
|
62
59
|
--project-root <target-project-root> \
|
|
@@ -59,6 +59,12 @@ INVENTORY_RESERVED_ROOTS = (
|
|
|
59
59
|
|
|
60
60
|
_IGNORED_DIR_NAMES = {".git", "__pycache__", ".pytest_cache", ".mypy_cache"}
|
|
61
61
|
FINAL_RECORDS = {"manifest.json", "validation.json", "verdict.json", "test-report.md"}
|
|
62
|
+
MAX_PROTOCOL_PACKAGE_ATTEMPTS = 2
|
|
63
|
+
PROTOCOL_STATE_NAME = ".protocol-recovery.json"
|
|
64
|
+
DEFAULT_EXECUTION_TIMEOUT_SECONDS = 300.0
|
|
65
|
+
MAX_EXECUTION_TIMEOUT_SECONDS = 1800.0
|
|
66
|
+
MAX_EXECUTION_OUTPUT_BYTES = 16 * 1024 * 1024
|
|
67
|
+
CANONICAL_EVIDENCE_ROOT = (".prizmkit", "test", "evidence")
|
|
62
68
|
|
|
63
69
|
|
|
64
70
|
class RequestError(Exception):
|
|
@@ -112,6 +118,34 @@ def confined(root: Path, value: str, *, must_exist: bool = False, directory: boo
|
|
|
112
118
|
return candidate
|
|
113
119
|
|
|
114
120
|
|
|
121
|
+
def canonical_evidence_root(project_root: Path) -> Path:
|
|
122
|
+
return project_root.joinpath(*CANONICAL_EVIDENCE_ROOT).resolve()
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def require_canonical_evidence_dir(project_root: Path, evidence_dir: Path) -> None:
|
|
126
|
+
root = canonical_evidence_root(project_root)
|
|
127
|
+
try:
|
|
128
|
+
evidence_dir.resolve().relative_to(root)
|
|
129
|
+
except ValueError as exc:
|
|
130
|
+
raise RequestError(
|
|
131
|
+
"evidence directory must be under .prizmkit/test/evidence"
|
|
132
|
+
) from exc
|
|
133
|
+
if evidence_dir.resolve() == root:
|
|
134
|
+
raise RequestError("evidence directory must be a package child of .prizmkit/test/evidence")
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def max_timeout(value: Any) -> float:
|
|
138
|
+
if value is None:
|
|
139
|
+
return DEFAULT_EXECUTION_TIMEOUT_SECONDS
|
|
140
|
+
if isinstance(value, bool) or not isinstance(value, (int, float)) or value <= 0:
|
|
141
|
+
raise RequestError("timeout_seconds must be a positive number")
|
|
142
|
+
if value > MAX_EXECUTION_TIMEOUT_SECONDS:
|
|
143
|
+
raise RequestError(
|
|
144
|
+
f"timeout_seconds must not exceed {MAX_EXECUTION_TIMEOUT_SECONDS:g} seconds"
|
|
145
|
+
)
|
|
146
|
+
return float(value)
|
|
147
|
+
|
|
148
|
+
|
|
115
149
|
def evidence_relative(evidence_dir: Path, path: Path) -> str:
|
|
116
150
|
try:
|
|
117
151
|
return path.resolve().relative_to(evidence_dir.resolve()).as_posix()
|
|
@@ -792,6 +826,7 @@ def run_inventory(project_root: Path, evidence_dir: Path, request_path: Path, ou
|
|
|
792
826
|
|
|
793
827
|
|
|
794
828
|
def run_process(argv: list[str], cwd: Path, environment: dict[str, str], timeout: float | None) -> tuple[int, bytes, bytes]:
|
|
829
|
+
timeout_value = max_timeout(timeout)
|
|
795
830
|
creationflags = 0
|
|
796
831
|
process_kwargs: dict[str, Any] = {}
|
|
797
832
|
if os.name == "nt":
|
|
@@ -803,7 +838,7 @@ def run_process(argv: list[str], cwd: Path, environment: dict[str, str], timeout
|
|
|
803
838
|
creationflags=creationflags, **process_kwargs,
|
|
804
839
|
)
|
|
805
840
|
try:
|
|
806
|
-
stdout, stderr = process.communicate(timeout=
|
|
841
|
+
stdout, stderr = process.communicate(timeout=timeout_value)
|
|
807
842
|
return process.returncode, stdout, stderr
|
|
808
843
|
except subprocess.TimeoutExpired:
|
|
809
844
|
if os.name == "nt":
|
|
@@ -821,7 +856,14 @@ def run_process(argv: list[str], cwd: Path, environment: dict[str, str], timeout
|
|
|
821
856
|
except ProcessLookupError:
|
|
822
857
|
pass
|
|
823
858
|
stdout, stderr = process.communicate()
|
|
824
|
-
return 124, stdout, stderr + f"\nPrizmKit runner timeout after {
|
|
859
|
+
return 124, stdout, stderr + f"\nPrizmKit runner timeout after {timeout_value:g} seconds\n".encode()
|
|
860
|
+
|
|
861
|
+
|
|
862
|
+
def limit_output(value: bytes, label: str) -> bytes:
|
|
863
|
+
if len(value) <= MAX_EXECUTION_OUTPUT_BYTES:
|
|
864
|
+
return value
|
|
865
|
+
marker = f"\nPrizmKit runner {label} truncated after {MAX_EXECUTION_OUTPUT_BYTES} bytes\n".encode()
|
|
866
|
+
return value[:MAX_EXECUTION_OUTPUT_BYTES - len(marker)] + marker
|
|
825
867
|
|
|
826
868
|
|
|
827
869
|
def run_probe(argv: list[str], cwd: Path, environment: dict[str, str], timeout: float | None) -> dict[str, Any]:
|
|
@@ -829,7 +871,8 @@ def run_probe(argv: list[str], cwd: Path, environment: dict[str, str], timeout:
|
|
|
829
871
|
exit_code, stdout, stderr = run_process(argv, cwd, environment, timeout)
|
|
830
872
|
return {
|
|
831
873
|
"command": argv, "exit_code": exit_code,
|
|
832
|
-
"stdout": stdout
|
|
874
|
+
"stdout": limit_output(stdout, "probe stdout").decode(errors="replace"),
|
|
875
|
+
"stderr": limit_output(stderr, "probe stderr").decode(errors="replace"),
|
|
833
876
|
}
|
|
834
877
|
except OSError as exc:
|
|
835
878
|
return {"command": argv, "error": str(exc)}
|
|
@@ -881,11 +924,15 @@ def execute_request(
|
|
|
881
924
|
if original_binding_fields != {"test_ids"} or original_test_ids != request["test_ids"]:
|
|
882
925
|
write_json_atomic(request_path, request)
|
|
883
926
|
cwd = confined(project_root, request["cwd"] or ".", must_exist=True, directory=True)
|
|
884
|
-
complete_environment = {
|
|
927
|
+
complete_environment = {
|
|
928
|
+
key: value for key, value in os.environ.items()
|
|
929
|
+
if key in {"PATH", "HOME", "TMPDIR", "TMP", "TEMP", "GOMODCACHE", "GOPATH", "GOCACHE", "NPM_CONFIG_CACHE", "PIP_CACHE_DIR"}
|
|
930
|
+
}
|
|
931
|
+
complete_environment.setdefault("PATH", os.defpath)
|
|
885
932
|
if os.name == "nt" and os.environ.get("SYSTEMROOT"):
|
|
886
933
|
complete_environment["SYSTEMROOT"] = os.environ["SYSTEMROOT"]
|
|
887
934
|
complete_environment.update(request["environment"])
|
|
888
|
-
timeout = request.get("timeout_seconds")
|
|
935
|
+
timeout = max_timeout(request.get("timeout_seconds"))
|
|
889
936
|
tools = {
|
|
890
937
|
name: run_probe(argv, cwd, complete_environment, timeout)
|
|
891
938
|
for name, argv in request["tool_version_commands"].items()
|
|
@@ -904,19 +951,24 @@ def execute_request(
|
|
|
904
951
|
stderr_path = raw_dir / f"{execution_id}.stderr"
|
|
905
952
|
started = dt.datetime.now(dt.timezone.utc).isoformat()
|
|
906
953
|
timed_out = False
|
|
954
|
+
output_limited = False
|
|
907
955
|
execution_error = False
|
|
908
956
|
try:
|
|
909
957
|
exit_code, stdout, stderr = run_process(
|
|
910
958
|
request["command"], cwd, complete_environment, timeout,
|
|
911
959
|
)
|
|
912
960
|
timed_out = exit_code == 124 and b"PrizmKit runner timeout" in stderr
|
|
961
|
+
output_limited = (
|
|
962
|
+
len(stdout) > MAX_EXECUTION_OUTPUT_BYTES
|
|
963
|
+
or len(stderr) > MAX_EXECUTION_OUTPUT_BYTES
|
|
964
|
+
)
|
|
913
965
|
except OSError as exc:
|
|
914
966
|
exit_code = 127
|
|
915
967
|
stdout = b""
|
|
916
968
|
stderr = f"PrizmKit runner execution error: {exc}\n".encode()
|
|
917
969
|
execution_error = True
|
|
918
|
-
stdout_path.write_bytes(stdout)
|
|
919
|
-
stderr_path.write_bytes(stderr)
|
|
970
|
+
stdout_path.write_bytes(limit_output(stdout, "stdout"))
|
|
971
|
+
stderr_path.write_bytes(limit_output(stderr, "stderr"))
|
|
920
972
|
finished = dt.datetime.now(dt.timezone.utc).isoformat()
|
|
921
973
|
existing = load_json(evidence_dir / "executions.json") if (evidence_dir / "executions.json").exists() else []
|
|
922
974
|
if not isinstance(existing, list):
|
|
@@ -938,13 +990,14 @@ def execute_request(
|
|
|
938
990
|
"layer": request["layer"],
|
|
939
991
|
"test_ids": request["test_ids"],
|
|
940
992
|
"external_targets": request["external_targets"],
|
|
993
|
+
"timeout_seconds": timeout,
|
|
941
994
|
"exit_code": exit_code,
|
|
942
995
|
"stdout_path": evidence_relative(evidence_dir, stdout_path),
|
|
943
996
|
"stderr_path": evidence_relative(evidence_dir, stderr_path),
|
|
944
997
|
"stdout_sha256": file_sha256(stdout_path),
|
|
945
998
|
"stderr_sha256": file_sha256(stderr_path),
|
|
946
999
|
"selected_execution": selected_execution,
|
|
947
|
-
"reliable": probes_reliable and not timed_out and not execution_error,
|
|
1000
|
+
"reliable": probes_reliable and not timed_out and not output_limited and not execution_error,
|
|
948
1001
|
"started_at": started,
|
|
949
1002
|
"finished_at": finished,
|
|
950
1003
|
"replay_of": replay_of,
|
|
@@ -960,7 +1013,13 @@ def run_capture_change(
|
|
|
960
1013
|
baseline: str | None,
|
|
961
1014
|
output_name: str,
|
|
962
1015
|
) -> Path:
|
|
1016
|
+
lifecycle = load_lifecycle(evidence_dir, required=True)
|
|
1017
|
+
expected_baseline = lifecycle.get("baseline_commit")
|
|
963
1018
|
resolved_baseline = git_baseline(project_root, baseline)
|
|
1019
|
+
if isinstance(expected_baseline, str) and resolved_baseline != expected_baseline:
|
|
1020
|
+
raise RequestError(
|
|
1021
|
+
"capture baseline does not match the baseline pinned during init"
|
|
1022
|
+
)
|
|
964
1023
|
patch, status_entries = canonical_change_patch(project_root, resolved_baseline)
|
|
965
1024
|
patch_path = confined(evidence_dir, "source-change.patch")
|
|
966
1025
|
patch_path.parent.mkdir(parents=True, exist_ok=True)
|
|
@@ -978,18 +1037,23 @@ def run_capture_change(
|
|
|
978
1037
|
|
|
979
1038
|
|
|
980
1039
|
def run_init(project_root: Path, evidence_dir: Path, baseline: str | None, output_name: str) -> Path:
|
|
1040
|
+
require_canonical_evidence_dir(project_root, evidence_dir)
|
|
1041
|
+
align_protocol_state(project_root, protocol_run_key(project_root, baseline))
|
|
1042
|
+
require_protocol_budget(project_root, package_initialization=True)
|
|
981
1043
|
if evidence_dir.exists() and any(evidence_dir.iterdir()):
|
|
982
1044
|
if (evidence_dir / "manifest.json").exists():
|
|
983
1045
|
raise RequestError("evidence package is finalized; initialize a new evidence directory")
|
|
984
1046
|
raise RequestError("evidence directory is not empty; initialize a new evidence directory")
|
|
985
1047
|
evidence_dir.mkdir(parents=True, exist_ok=True)
|
|
1048
|
+
resolved_baseline = git_baseline(project_root, baseline)
|
|
1049
|
+
update_protocol_state(project_root, event="package-initialized", increment="package_initializations")
|
|
986
1050
|
for directory in ("requests", "receipts", "raw", "generated-tests", "contracts", "runner"):
|
|
987
1051
|
(evidence_dir / directory).mkdir(parents=True, exist_ok=True)
|
|
988
1052
|
write_json_atomic(evidence_dir / "executions.json", [])
|
|
989
1053
|
write_json_atomic(evidence_dir / "lifecycle.json", {
|
|
990
1054
|
"lifecycle_format": "prizmkit-canonical-lifecycle-v1",
|
|
991
1055
|
"status": "initialized",
|
|
992
|
-
"baseline_commit":
|
|
1056
|
+
"baseline_commit": resolved_baseline,
|
|
993
1057
|
"artifact_dir": None,
|
|
994
1058
|
"finalized": False,
|
|
995
1059
|
"immutable_identity": None,
|
|
@@ -998,12 +1062,147 @@ def run_init(project_root: Path, evidence_dir: Path, baseline: str | None, outpu
|
|
|
998
1062
|
write_json_atomic(output, {
|
|
999
1063
|
"command": "init",
|
|
1000
1064
|
"evidence_dir": ".",
|
|
1001
|
-
"baseline_commit":
|
|
1065
|
+
"baseline_commit": resolved_baseline,
|
|
1002
1066
|
"next": "capture-change",
|
|
1003
1067
|
})
|
|
1004
1068
|
return output
|
|
1005
1069
|
|
|
1006
1070
|
|
|
1071
|
+
def protocol_state_path(project_root: Path) -> Path:
|
|
1072
|
+
return canonical_evidence_root(project_root) / PROTOCOL_STATE_NAME
|
|
1073
|
+
|
|
1074
|
+
|
|
1075
|
+
def align_protocol_state(project_root: Path, run_key: str) -> None:
|
|
1076
|
+
state_path = protocol_state_path(project_root)
|
|
1077
|
+
if state_path.exists():
|
|
1078
|
+
state = ensure_object(load_json(state_path), "protocol recovery state")
|
|
1079
|
+
if state.get("run_key") == run_key:
|
|
1080
|
+
return
|
|
1081
|
+
write_json_atomic(state_path, {"run_key": run_key})
|
|
1082
|
+
|
|
1083
|
+
|
|
1084
|
+
def update_protocol_state(
|
|
1085
|
+
project_root: Path,
|
|
1086
|
+
*,
|
|
1087
|
+
event: str,
|
|
1088
|
+
error: str | None = None,
|
|
1089
|
+
increment: str | None = None,
|
|
1090
|
+
) -> dict[str, Any]:
|
|
1091
|
+
state_path = protocol_state_path(project_root)
|
|
1092
|
+
state: dict[str, Any] = {}
|
|
1093
|
+
if state_path.exists():
|
|
1094
|
+
state = ensure_object(load_json(state_path), "protocol recovery state")
|
|
1095
|
+
if increment is not None:
|
|
1096
|
+
count = state.get(increment, 0)
|
|
1097
|
+
if not isinstance(count, int) or isinstance(count, bool):
|
|
1098
|
+
count = 0
|
|
1099
|
+
state[increment] = count + 1
|
|
1100
|
+
state["last_event"] = event
|
|
1101
|
+
state["last_error"] = error
|
|
1102
|
+
state["max_package_initializations"] = MAX_PROTOCOL_PACKAGE_ATTEMPTS
|
|
1103
|
+
state["max_preflight_failures"] = MAX_PROTOCOL_PACKAGE_ATTEMPTS
|
|
1104
|
+
state["blocked"] = (
|
|
1105
|
+
state.get("package_initializations", 0) >= MAX_PROTOCOL_PACKAGE_ATTEMPTS
|
|
1106
|
+
or state.get("preflight_failures", 0) >= MAX_PROTOCOL_PACKAGE_ATTEMPTS
|
|
1107
|
+
)
|
|
1108
|
+
write_json_atomic(state_path, state)
|
|
1109
|
+
return state
|
|
1110
|
+
|
|
1111
|
+
|
|
1112
|
+
def require_protocol_budget(project_root: Path, *, package_initialization: bool = False) -> None:
|
|
1113
|
+
state_path = protocol_state_path(project_root)
|
|
1114
|
+
if not state_path.exists():
|
|
1115
|
+
return
|
|
1116
|
+
state = ensure_object(load_json(state_path), "protocol recovery state")
|
|
1117
|
+
if package_initialization and state.get("package_initializations", 0) >= MAX_PROTOCOL_PACKAGE_ATTEMPTS:
|
|
1118
|
+
raise RequestError(
|
|
1119
|
+
"evidence protocol package budget exhausted; stop and return TEST_BLOCKED "
|
|
1120
|
+
"instead of creating another evidence package"
|
|
1121
|
+
)
|
|
1122
|
+
if state.get("preflight_failures", 0) >= MAX_PROTOCOL_PACKAGE_ATTEMPTS:
|
|
1123
|
+
raise RequestError(
|
|
1124
|
+
"evidence protocol preflight budget exhausted; return TEST_BLOCKED "
|
|
1125
|
+
"instead of continuing semantic record repair"
|
|
1126
|
+
)
|
|
1127
|
+
|
|
1128
|
+
|
|
1129
|
+
def protocol_run_key(project_root: Path, baseline: str | None) -> str:
|
|
1130
|
+
resolved_baseline = git_baseline(project_root, baseline)
|
|
1131
|
+
patch, _ = canonical_change_patch(project_root, resolved_baseline)
|
|
1132
|
+
return canonical_sha256({
|
|
1133
|
+
"baseline_commit": resolved_baseline,
|
|
1134
|
+
"working_diff_sha256": hashlib.sha256(patch).hexdigest(),
|
|
1135
|
+
})
|
|
1136
|
+
|
|
1137
|
+
|
|
1138
|
+
def semantic_record_paths(change_class: str) -> set[str]:
|
|
1139
|
+
common = {"change-classification.json", "scope.json"}
|
|
1140
|
+
if change_class == "behavior":
|
|
1141
|
+
return common | {"behavior-risk-matrix.json", "infrastructure-changes.json"}
|
|
1142
|
+
return common | {"lightweight-verification.json"}
|
|
1143
|
+
|
|
1144
|
+
|
|
1145
|
+
def require_semantic_record_files(evidence_dir: Path, change_class: str) -> None:
|
|
1146
|
+
missing = sorted(
|
|
1147
|
+
relative for relative in semantic_record_paths(change_class)
|
|
1148
|
+
if not (evidence_dir / relative).is_file()
|
|
1149
|
+
)
|
|
1150
|
+
if missing:
|
|
1151
|
+
raise RequestError(
|
|
1152
|
+
"pre-finalize semantic records are incomplete: " + ", ".join(missing)
|
|
1153
|
+
+ "; keep this package mutable and repair it before finalize"
|
|
1154
|
+
)
|
|
1155
|
+
|
|
1156
|
+
|
|
1157
|
+
def preflight_semantic_records(evidence_dir: Path, change_class: str) -> None:
|
|
1158
|
+
require_semantic_record_files(evidence_dir, change_class)
|
|
1159
|
+
validator_path = Path(__file__).resolve().with_name("validate_test_evidence.py")
|
|
1160
|
+
try:
|
|
1161
|
+
import importlib.util
|
|
1162
|
+
spec = importlib.util.spec_from_file_location("prizmkit_test_validator_preflight", validator_path)
|
|
1163
|
+
if spec is None or spec.loader is None:
|
|
1164
|
+
raise RequestError("cannot load semantic preflight validator")
|
|
1165
|
+
module = importlib.util.module_from_spec(spec)
|
|
1166
|
+
spec.loader.exec_module(module)
|
|
1167
|
+
records_schema = module.load_schema("authoritative-records.schema.json")
|
|
1168
|
+
matrix_schema = module.load_schema("behavior-risk-matrix.schema.json")
|
|
1169
|
+
errors: list[str] = []
|
|
1170
|
+
definitions = {
|
|
1171
|
+
"change-classification.json": "changeClassification",
|
|
1172
|
+
"scope.json": "scope",
|
|
1173
|
+
"infrastructure-changes.json": "infrastructureChanges",
|
|
1174
|
+
"lightweight-verification.json": "lightweightVerification",
|
|
1175
|
+
}
|
|
1176
|
+
for relative in sorted(semantic_record_paths(change_class)):
|
|
1177
|
+
value = load_json(evidence_dir / relative)
|
|
1178
|
+
if relative == "behavior-risk-matrix.json":
|
|
1179
|
+
module.validate_schema(value, matrix_schema, matrix_schema, "behavior-risk-matrix", errors)
|
|
1180
|
+
else:
|
|
1181
|
+
module.validate_definition(value, definitions[relative], records_schema, relative, errors)
|
|
1182
|
+
if errors:
|
|
1183
|
+
raise RequestError("semantic preflight failed: " + "; ".join(errors[:20]))
|
|
1184
|
+
except ImportError as exc:
|
|
1185
|
+
raise RequestError(f"cannot run semantic preflight: {exc}") from exc
|
|
1186
|
+
|
|
1187
|
+
|
|
1188
|
+
def selected_layer_blockers(plan: dict[str, Any] | None, executions: list[Any]) -> list[str]:
|
|
1189
|
+
if not isinstance(plan, dict):
|
|
1190
|
+
return []
|
|
1191
|
+
selected_layers = {
|
|
1192
|
+
item.get("layer")
|
|
1193
|
+
for item in executions
|
|
1194
|
+
if isinstance(item, dict) and item.get("selected_execution") is True
|
|
1195
|
+
}
|
|
1196
|
+
blockers: list[str] = []
|
|
1197
|
+
for layer in plan.get("layers", []):
|
|
1198
|
+
if not isinstance(layer, dict) or layer.get("required") is not True:
|
|
1199
|
+
continue
|
|
1200
|
+
name = layer.get("name")
|
|
1201
|
+
if name not in selected_layers:
|
|
1202
|
+
blockers.append(f"required layer has no selected execution: {name}")
|
|
1203
|
+
return blockers
|
|
1204
|
+
|
|
1205
|
+
|
|
1007
1206
|
def load_lifecycle(evidence_dir: Path, *, required: bool = False) -> dict[str, Any]:
|
|
1008
1207
|
path = evidence_dir / "lifecycle.json"
|
|
1009
1208
|
if not path.exists():
|
|
@@ -1284,7 +1483,7 @@ def stage_output_owner(relative: str, change_class: str) -> str:
|
|
|
1284
1483
|
return "EVIDENCE_PACKAGE"
|
|
1285
1484
|
|
|
1286
1485
|
|
|
1287
|
-
def build_manifest(evidence_dir: Path, *, evidence_id: str, identity: dict[str, Any], target_hashes: dict[str, str], change_class: str, baseline_commit: str, final_verdict: str) -> dict[str, Any]:
|
|
1486
|
+
def build_manifest(evidence_dir: Path, *, evidence_id: str, identity: dict[str, Any], target_hashes: dict[str, str], change_class: str, baseline_commit: str, final_verdict: str, attested: bool = False) -> dict[str, Any]:
|
|
1288
1487
|
file_paths = sorted(
|
|
1289
1488
|
path.relative_to(evidence_dir).as_posix()
|
|
1290
1489
|
for path in evidence_dir.rglob("*")
|
|
@@ -1302,7 +1501,7 @@ def build_manifest(evidence_dir: Path, *, evidence_id: str, identity: dict[str,
|
|
|
1302
1501
|
outputs = sorted(outputs_by_stage.get(stage, []))
|
|
1303
1502
|
stages.append({
|
|
1304
1503
|
"name": stage,
|
|
1305
|
-
"status": "complete",
|
|
1504
|
+
"status": "complete" if stage != "EVIDENCE_VALIDATE" or attested else "pending",
|
|
1306
1505
|
"outputs": outputs,
|
|
1307
1506
|
"not_applicable": None,
|
|
1308
1507
|
})
|
|
@@ -1337,6 +1536,8 @@ def write_handoff_pointer(evidence_dir: Path, artifact_dir: str | None, project_
|
|
|
1337
1536
|
|
|
1338
1537
|
|
|
1339
1538
|
def run_finalize(project_root: Path, evidence_dir: Path, request: dict[str, Any], output_name: str) -> Path:
|
|
1539
|
+
require_canonical_evidence_dir(project_root, evidence_dir)
|
|
1540
|
+
require_protocol_budget(project_root)
|
|
1340
1541
|
allowed = {"request_version", "change_class", "baseline_commit", "scope_path", "environment_path", "artifact_dir", "blockers", "repair_scope"}
|
|
1341
1542
|
if set(request) - allowed:
|
|
1342
1543
|
raise RequestError(f"finalize request contains unsupported fields: {sorted(set(request) - allowed)}")
|
|
@@ -1351,13 +1552,32 @@ def run_finalize(project_root: Path, evidence_dir: Path, request: dict[str, Any]
|
|
|
1351
1552
|
if not capture_path.exists():
|
|
1352
1553
|
run_capture_change(project_root, evidence_dir, request.get("baseline_commit"), "change-capture.json")
|
|
1353
1554
|
capture = load_change_capture(project_root, evidence_dir, require_fresh=True)
|
|
1555
|
+
lifecycle_baseline = lifecycle.get("baseline_commit")
|
|
1354
1556
|
baseline = capture.get("baseline_commit")
|
|
1557
|
+
if not isinstance(lifecycle_baseline, str) or baseline != lifecycle_baseline:
|
|
1558
|
+
raise RequestError("canonical change capture baseline does not match init baseline")
|
|
1355
1559
|
patch_path = evidence_dir / "source-change.patch"
|
|
1356
1560
|
if not isinstance(baseline, str) or not patch_path.is_file():
|
|
1357
1561
|
raise RequestError("canonical change capture is incomplete")
|
|
1358
1562
|
change_class = request.get("change_class", "behavior")
|
|
1359
1563
|
if change_class not in {"behavior", "lightweight"}:
|
|
1360
1564
|
raise RequestError("change_class must be behavior or lightweight")
|
|
1565
|
+
try:
|
|
1566
|
+
require_semantic_record_files(evidence_dir, change_class)
|
|
1567
|
+
except RequestError as exc:
|
|
1568
|
+
state = update_protocol_state(
|
|
1569
|
+
project_root,
|
|
1570
|
+
event="preflight-failed",
|
|
1571
|
+
error=str(exc),
|
|
1572
|
+
increment="preflight_failures",
|
|
1573
|
+
)
|
|
1574
|
+
if state.get("blocked"):
|
|
1575
|
+
raise RequestError(
|
|
1576
|
+
str(exc) + "; evidence protocol recovery budget exhausted; return TEST_BLOCKED "
|
|
1577
|
+
"and do not initialize another package"
|
|
1578
|
+
) from exc
|
|
1579
|
+
raise
|
|
1580
|
+
|
|
1361
1581
|
artifact_dir = request.get("artifact_dir")
|
|
1362
1582
|
if artifact_dir is not None and not isinstance(artifact_dir, str):
|
|
1363
1583
|
raise RequestError("artifact_dir must be a project-relative directory")
|
|
@@ -1385,14 +1605,31 @@ def run_finalize(project_root: Path, evidence_dir: Path, request: dict[str, Any]
|
|
|
1385
1605
|
write_json_atomic(evidence_dir / "test-plan.json", plan)
|
|
1386
1606
|
scope["target_hashes"] = target_hashes
|
|
1387
1607
|
write_json_atomic(evidence_dir / scope_relative, scope)
|
|
1608
|
+
try:
|
|
1609
|
+
preflight_semantic_records(evidence_dir, change_class)
|
|
1610
|
+
except RequestError as exc:
|
|
1611
|
+
state = update_protocol_state(
|
|
1612
|
+
project_root,
|
|
1613
|
+
event="preflight-failed",
|
|
1614
|
+
error=str(exc),
|
|
1615
|
+
increment="preflight_failures",
|
|
1616
|
+
)
|
|
1617
|
+
if state.get("blocked"):
|
|
1618
|
+
raise RequestError(
|
|
1619
|
+
str(exc) + "; evidence protocol recovery budget exhausted; return TEST_BLOCKED "
|
|
1620
|
+
"and do not initialize another package"
|
|
1621
|
+
) from exc
|
|
1622
|
+
raise
|
|
1388
1623
|
identity = {
|
|
1389
1624
|
"baseline_commit": baseline,
|
|
1390
1625
|
"working_diff_sha256": file_sha256(patch_path),
|
|
1391
1626
|
"scope_sha256": canonical_sha256(scope),
|
|
1392
1627
|
}
|
|
1393
1628
|
evidence_id = canonical_sha256(identity)
|
|
1629
|
+
destination = evidence_dir.parent / evidence_id
|
|
1394
1630
|
executions = load_json(evidence_dir / "executions.json") if (evidence_dir / "executions.json").exists() else []
|
|
1395
1631
|
blockers = list(request.get("blockers", [])) if isinstance(request.get("blockers", []), list) else []
|
|
1632
|
+
blockers.extend(selected_layer_blockers(plan, executions if isinstance(executions, list) else []))
|
|
1396
1633
|
required_records = {
|
|
1397
1634
|
"change-classification.json", "scope.json", "target-inventory.json", "environment.json",
|
|
1398
1635
|
"source-change.patch",
|
|
@@ -1409,8 +1646,17 @@ def run_finalize(project_root: Path, evidence_dir: Path, request: dict[str, Any]
|
|
|
1409
1646
|
if not (evidence_dir / relative).is_file()
|
|
1410
1647
|
)
|
|
1411
1648
|
if missing_records:
|
|
1412
|
-
|
|
1413
|
-
|
|
1649
|
+
state = update_protocol_state(
|
|
1650
|
+
project_root,
|
|
1651
|
+
event="finalize-precondition-failed",
|
|
1652
|
+
error="required evidence records are missing: " + ", ".join(missing_records),
|
|
1653
|
+
increment="preflight_failures",
|
|
1654
|
+
)
|
|
1655
|
+
suffix = ""
|
|
1656
|
+
if state.get("blocked"):
|
|
1657
|
+
suffix = "; evidence protocol recovery budget exhausted; return TEST_BLOCKED"
|
|
1658
|
+
raise RequestError(
|
|
1659
|
+
"pre-finalize package records are incomplete: " + ", ".join(missing_records) + suffix
|
|
1414
1660
|
)
|
|
1415
1661
|
selected_unreliable = [
|
|
1416
1662
|
item for item in executions
|
|
@@ -1462,23 +1708,26 @@ def run_finalize(project_root: Path, evidence_dir: Path, request: dict[str, Any]
|
|
|
1462
1708
|
evidence_dir, evidence_id=evidence_id, identity=identity,
|
|
1463
1709
|
target_hashes=target_hashes, change_class=change_class,
|
|
1464
1710
|
baseline_commit=baseline, final_verdict=verdict,
|
|
1711
|
+
attested=False,
|
|
1465
1712
|
)
|
|
1466
1713
|
write_json_atomic(evidence_dir / "manifest.json", manifest)
|
|
1467
1714
|
run_render_report(evidence_dir, "test-report.md")
|
|
1468
1715
|
write_json_atomic(evidence_dir / "lifecycle.json", {
|
|
1469
1716
|
**lifecycle,
|
|
1470
|
-
"status": "finalized",
|
|
1471
|
-
"finalized":
|
|
1717
|
+
"status": "payload-finalized",
|
|
1718
|
+
"finalized": False,
|
|
1472
1719
|
"immutable_identity": identity,
|
|
1473
1720
|
})
|
|
1474
1721
|
output = confined(evidence_dir, output_name)
|
|
1722
|
+
final_evidence_relative = destination.relative_to(project_root).as_posix()
|
|
1475
1723
|
write_json_atomic(output, {
|
|
1476
1724
|
"command": "finalize",
|
|
1477
1725
|
"evidence_id": evidence_id,
|
|
1478
|
-
"
|
|
1479
|
-
"
|
|
1480
|
-
"
|
|
1481
|
-
"
|
|
1726
|
+
"evidence_dir": final_evidence_relative,
|
|
1727
|
+
"manifest_path": f"{final_evidence_relative}/manifest.json",
|
|
1728
|
+
"verdict_path": f"{final_evidence_relative}/verdict.json",
|
|
1729
|
+
"report_path": f"{final_evidence_relative}/test-report.md",
|
|
1730
|
+
"validation_path": f"{final_evidence_relative}/validation.json",
|
|
1482
1731
|
"next": "validate_test_evidence.py --attest",
|
|
1483
1732
|
})
|
|
1484
1733
|
# Second manifest: re-hashes after render-report produces test-report.md.
|
|
@@ -1486,9 +1735,9 @@ def run_finalize(project_root: Path, evidence_dir: Path, request: dict[str, Any]
|
|
|
1486
1735
|
evidence_dir, evidence_id=evidence_id, identity=identity,
|
|
1487
1736
|
target_hashes=target_hashes, change_class=change_class,
|
|
1488
1737
|
baseline_commit=baseline, final_verdict=verdict,
|
|
1738
|
+
attested=False,
|
|
1489
1739
|
)
|
|
1490
1740
|
write_json_atomic(evidence_dir / "manifest.json", manifest)
|
|
1491
|
-
destination = evidence_dir.parent / evidence_id
|
|
1492
1741
|
if destination != evidence_dir:
|
|
1493
1742
|
if destination.exists():
|
|
1494
1743
|
try:
|
|
@@ -1502,8 +1751,14 @@ def run_finalize(project_root: Path, evidence_dir: Path, request: dict[str, Any]
|
|
|
1502
1751
|
f"evidence identity already exists at {destination}; re-init a new package"
|
|
1503
1752
|
)
|
|
1504
1753
|
os.replace(evidence_dir, destination)
|
|
1505
|
-
|
|
1506
|
-
|
|
1754
|
+
final_evidence_dir = destination if destination != evidence_dir else evidence_dir
|
|
1755
|
+
write_handoff_pointer(final_evidence_dir, lifecycle.get("artifact_dir"), project_root)
|
|
1756
|
+
update_protocol_state(
|
|
1757
|
+
project_root,
|
|
1758
|
+
event="finalized",
|
|
1759
|
+
error="; ".join(verdict_record.get("blockers", [])) if verdict == "TEST_BLOCKED" else None,
|
|
1760
|
+
)
|
|
1761
|
+
return final_evidence_dir / output_name
|
|
1507
1762
|
|
|
1508
1763
|
|
|
1509
1764
|
def run_resume(
|
|
@@ -1619,10 +1874,7 @@ def main() -> int:
|
|
|
1619
1874
|
try:
|
|
1620
1875
|
if not project_root.is_dir():
|
|
1621
1876
|
raise RequestError(f"project root does not exist: {project_root}")
|
|
1622
|
-
|
|
1623
|
-
evidence_dir.relative_to(project_root)
|
|
1624
|
-
except ValueError:
|
|
1625
|
-
raise RequestError(f"evidence directory must be under project root: {evidence_dir}")
|
|
1877
|
+
require_canonical_evidence_dir(project_root, evidence_dir)
|
|
1626
1878
|
if args.subcommand == "init":
|
|
1627
1879
|
output = run_init(project_root, evidence_dir, args.baseline, args.output)
|
|
1628
1880
|
print(f"Evidence initialized: {output}")
|
|
@@ -39,6 +39,7 @@ LAYER_ORDER = {
|
|
|
39
39
|
"regression-ring": 4,
|
|
40
40
|
}
|
|
41
41
|
FINAL_RECORDS = {"validation.json", "verdict.json", "test-report.md"}
|
|
42
|
+
ATTESTATION_EXCLUDED_RECORDS = FINAL_RECORDS | {"lifecycle.json"}
|
|
42
43
|
ATTESTATION_MUTABLE = {"validation.json"}
|
|
43
44
|
COMMON_RECORDS = {
|
|
44
45
|
"change-classification.json", "scope.json", "target-inventory.json",
|
|
@@ -646,8 +647,12 @@ def validate_manifest(
|
|
|
646
647
|
if status == "not_applicable":
|
|
647
648
|
structured_na(na, records_schema, f"stage {name} not_applicable", errors)
|
|
648
649
|
require(outputs == [], f"not-applicable stage must not claim outputs: {name}", errors)
|
|
650
|
+
elif status == "pending":
|
|
651
|
+
require(na is None, f"pending stage must have null not_applicable: {name}", errors)
|
|
649
652
|
elif status == "complete":
|
|
650
653
|
require(na is None, f"complete stage must have null not_applicable: {name}", errors)
|
|
654
|
+
else:
|
|
655
|
+
require(False, f"stage has unknown status: {name}", errors)
|
|
651
656
|
for output in outputs if isinstance(outputs, list) else []:
|
|
652
657
|
entry = entry_map.get(output)
|
|
653
658
|
require(entry is not None, f"stage output is not a hashed evidence file: {name}/{output}", errors)
|
|
@@ -961,6 +966,16 @@ def validate_requests_and_receipts(
|
|
|
961
966
|
if isinstance(request, dict):
|
|
962
967
|
for key in ("purpose", "command", "cwd", "layer", "test_ids", "external_targets"):
|
|
963
968
|
require(receipt.get(key) == request.get(key), f"{location}: receipt {key} does not match request", errors)
|
|
969
|
+
requested_timeout = request.get("timeout_seconds")
|
|
970
|
+
require(
|
|
971
|
+
isinstance(receipt.get("timeout_seconds"), (int, float))
|
|
972
|
+
and not isinstance(receipt.get("timeout_seconds"), bool)
|
|
973
|
+
and receipt["timeout_seconds"] > 0,
|
|
974
|
+
f"{location}: receipt lacks a bounded execution timeout",
|
|
975
|
+
errors,
|
|
976
|
+
)
|
|
977
|
+
if requested_timeout is not None:
|
|
978
|
+
require(receipt.get("timeout_seconds") == requested_timeout, f"{location}: receipt timeout does not match request", errors)
|
|
964
979
|
expected_environment = request.get("environment")
|
|
965
980
|
actual_environment = receipt.get("environment")
|
|
966
981
|
require(isinstance(actual_environment, dict) and isinstance(actual_environment.get("PATH"), str), f"{location}: receipt lacks complete PATH environment", errors)
|
|
@@ -1179,9 +1194,32 @@ def validate_plan_and_scope_bindings(
|
|
|
1179
1194
|
records_schema: dict[str, Any],
|
|
1180
1195
|
errors: list[str],
|
|
1181
1196
|
) -> dict[str, dict[str, Any]]:
|
|
1182
|
-
|
|
1183
|
-
|
|
1184
|
-
|
|
1197
|
+
if not isinstance(scope, dict):
|
|
1198
|
+
errors.append("scope must be an object")
|
|
1199
|
+
if not isinstance(matrix, dict):
|
|
1200
|
+
errors.append("behavior-risk-matrix must be an object")
|
|
1201
|
+
scope_module = scope.get("affected_module") if isinstance(scope, dict) else None
|
|
1202
|
+
matrix_module = matrix.get("affected_module") if isinstance(matrix, dict) else None
|
|
1203
|
+
matrix_module_name = matrix_module.get("name") if isinstance(matrix_module, dict) else None
|
|
1204
|
+
matrix_boundary_source = matrix_module.get("boundary_source") if isinstance(matrix_module, dict) else None
|
|
1205
|
+
require(
|
|
1206
|
+
isinstance(scope_module, str) and scope_module == matrix_module_name,
|
|
1207
|
+
"matrix affected module does not match scope",
|
|
1208
|
+
errors,
|
|
1209
|
+
)
|
|
1210
|
+
require(
|
|
1211
|
+
isinstance(scope.get("boundary_source") if isinstance(scope, dict) else None, str)
|
|
1212
|
+
and scope.get("boundary_source") == matrix_boundary_source,
|
|
1213
|
+
"matrix boundary source does not match scope",
|
|
1214
|
+
errors,
|
|
1215
|
+
)
|
|
1216
|
+
require(
|
|
1217
|
+
scope.get("primary_scope") == matrix.get("primary_scope") if isinstance(scope, dict) and isinstance(matrix, dict) else False,
|
|
1218
|
+
"matrix primary scope does not match scope",
|
|
1219
|
+
errors,
|
|
1220
|
+
)
|
|
1221
|
+
if not isinstance(scope, dict) or not isinstance(matrix, dict) or not isinstance(matrix_module, dict):
|
|
1222
|
+
return {}
|
|
1185
1223
|
scope_ring = sorted((item.get("name"), item.get("kind")) for item in scope.get("regression_ring", []) if isinstance(item, dict))
|
|
1186
1224
|
matrix_ring = sorted((item.get("name"), item.get("kind")) for item in matrix.get("regression_ring", []) if isinstance(item, dict))
|
|
1187
1225
|
require(scope_ring == matrix_ring, "scope and matrix regression rings do not agree", errors)
|
|
@@ -1484,10 +1522,33 @@ def validate_verdict_and_attestation(
|
|
|
1484
1522
|
errors,
|
|
1485
1523
|
)
|
|
1486
1524
|
|
|
1525
|
+
if not verify_attestation:
|
|
1526
|
+
lifecycle = load_json(root / "lifecycle.json") if (root / "lifecycle.json").is_file() else {}
|
|
1527
|
+
if (root / "lifecycle.json").is_file():
|
|
1528
|
+
require(
|
|
1529
|
+
lifecycle.get("status") == "payload-finalized" and lifecycle.get("finalized") is False,
|
|
1530
|
+
"pre-attestation validation requires a payload-finalized mutable package",
|
|
1531
|
+
errors,
|
|
1532
|
+
)
|
|
1533
|
+
require(
|
|
1534
|
+
not validation,
|
|
1535
|
+
"pre-attestation package must not contain validator-owned validation data",
|
|
1536
|
+
errors,
|
|
1537
|
+
)
|
|
1538
|
+
|
|
1487
1539
|
if verify_attestation:
|
|
1488
1540
|
validate_definition(validation, "validation", records_schema, "validation", errors)
|
|
1541
|
+
lifecycle = load_json(root / "lifecycle.json") if (root / "lifecycle.json").is_file() else {}
|
|
1542
|
+
if (root / "lifecycle.json").is_file():
|
|
1543
|
+
require(
|
|
1544
|
+
lifecycle.get("status") in {"payload-finalized", "sealed"},
|
|
1545
|
+
"attested package has an invalid lifecycle state",
|
|
1546
|
+
errors,
|
|
1547
|
+
)
|
|
1548
|
+
if lifecycle.get("status") == "sealed":
|
|
1549
|
+
require(lifecycle.get("finalized") is True, "sealed package must be finalized", errors)
|
|
1489
1550
|
payload = sorted(
|
|
1490
|
-
({"path": path, "sha256": entry["sha256"]} for path, entry in entry_map.items() if path not in
|
|
1551
|
+
({"path": path, "sha256": entry["sha256"]} for path, entry in entry_map.items() if path not in ATTESTATION_EXCLUDED_RECORDS),
|
|
1491
1552
|
key=lambda item: item["path"],
|
|
1492
1553
|
)
|
|
1493
1554
|
replayed_ids = [
|
|
@@ -1609,17 +1670,21 @@ def validate_evidence(
|
|
|
1609
1670
|
errors.append(str(exc))
|
|
1610
1671
|
matrix = infrastructure = None
|
|
1611
1672
|
if all(isinstance(item, dict) for item in (matrix, infrastructure, plan)):
|
|
1673
|
+
matrix_error_count = len(errors)
|
|
1612
1674
|
validate_schema(matrix, matrix_schema, matrix_schema, "behavior-risk-matrix", errors)
|
|
1613
1675
|
validate_definition(infrastructure, "infrastructureChanges", records_schema, "infrastructure-changes", errors)
|
|
1614
|
-
|
|
1615
|
-
|
|
1616
|
-
|
|
1617
|
-
|
|
1618
|
-
|
|
1619
|
-
|
|
1620
|
-
|
|
1621
|
-
|
|
1622
|
-
|
|
1676
|
+
if len(errors) > matrix_error_count:
|
|
1677
|
+
errors.append("behavior semantic cross-link validation skipped because behavior-risk-matrix schema validation failed")
|
|
1678
|
+
else:
|
|
1679
|
+
stages = {item.get("name"): item for item in manifest.get("stages", []) if isinstance(item, dict)}
|
|
1680
|
+
for stage_name in CORE_BEHAVIOR_STAGES:
|
|
1681
|
+
require(stages.get(stage_name, {}).get("status") == "complete", f"behavior protocol stage cannot be N/A: {stage_name}", errors)
|
|
1682
|
+
test_map = validate_plan_and_scope_bindings(
|
|
1683
|
+
root, project_root, manifest, scope, matrix, plan, inventory_by_category, entry_map,
|
|
1684
|
+
executions, records_schema, errors,
|
|
1685
|
+
)
|
|
1686
|
+
validate_matrix_risks(manifest, matrix, test_map, executions, records_schema, errors)
|
|
1687
|
+
validate_infrastructure(manifest, infrastructure, executions, entry_map, root, errors)
|
|
1623
1688
|
else:
|
|
1624
1689
|
errors.append("behavior evidence records must be JSON objects")
|
|
1625
1690
|
elif change_class == "lightweight":
|
|
@@ -1722,6 +1787,16 @@ def main() -> int:
|
|
|
1722
1787
|
manifest = load_json(root / "manifest.json")
|
|
1723
1788
|
verdict_record = load_json(root / "verdict.json")
|
|
1724
1789
|
if args.attest:
|
|
1790
|
+
lifecycle_path = root / "lifecycle.json"
|
|
1791
|
+
lifecycle = load_json(lifecycle_path) if lifecycle_path.is_file() else {}
|
|
1792
|
+
if lifecycle_path.is_file() and (lifecycle.get("status") != "payload-finalized" or lifecycle.get("finalized") is not False):
|
|
1793
|
+
print("Test evidence validation: FAIL\n- attestation requires a payload-finalized mutable package")
|
|
1794
|
+
return 1
|
|
1795
|
+
manifest = load_json(root / "manifest.json")
|
|
1796
|
+
for stage in manifest.get("stages", []):
|
|
1797
|
+
if isinstance(stage, dict) and stage.get("name") == "EVIDENCE_VALIDATE":
|
|
1798
|
+
stage["status"] = "complete"
|
|
1799
|
+
atomic_write_json(root / "manifest.json", manifest)
|
|
1725
1800
|
executions = load_json(root / "executions.json")
|
|
1726
1801
|
replayed_ids = [
|
|
1727
1802
|
item.get("execution_id") for item in executions
|
|
@@ -1732,7 +1807,7 @@ def main() -> int:
|
|
|
1732
1807
|
payload = sorted(
|
|
1733
1808
|
(
|
|
1734
1809
|
{"path": entry["path"], "sha256": entry["sha256"]}
|
|
1735
|
-
for entry in manifest["files"] if entry["path"] not in
|
|
1810
|
+
for entry in manifest["files"] if entry["path"] not in ATTESTATION_EXCLUDED_RECORDS
|
|
1736
1811
|
),
|
|
1737
1812
|
key=lambda item: item["path"],
|
|
1738
1813
|
)
|
|
@@ -1755,10 +1830,16 @@ def main() -> int:
|
|
|
1755
1830
|
for error in errors:
|
|
1756
1831
|
print(f"- {error}")
|
|
1757
1832
|
return 1
|
|
1758
|
-
|
|
1759
|
-
|
|
1833
|
+
if lifecycle_path.is_file():
|
|
1834
|
+
atomic_write_json(root / "lifecycle.json", {
|
|
1835
|
+
**lifecycle,
|
|
1836
|
+
"status": "sealed",
|
|
1837
|
+
"finalized": True,
|
|
1838
|
+
})
|
|
1839
|
+
|
|
1840
|
+
print("Evidence validation: PASS")
|
|
1841
|
+
print(f"Testing verdict: {manifest['final_verdict']}")
|
|
1760
1842
|
print(f"Evidence ID: {manifest['evidence_id']}")
|
|
1761
|
-
print(f"Verdict: {manifest['final_verdict']}")
|
|
1762
1843
|
return 0
|
|
1763
1844
|
|
|
1764
1845
|
|