prizmkit 1.1.139 → 1.1.140
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bundled/VERSION.json +3 -3
- package/bundled/dev-pipeline/prizmkit_runtime/checkpoint_state.py +1 -1
- package/bundled/dev-pipeline/scripts/update-checkpoint.py +37 -11
- package/bundled/dev-pipeline/tests/test_checkpoint_state.py +55 -2
- package/bundled/skills/_metadata.json +1 -1
- package/bundled/skills/prizmkit-test/SKILL.md +9 -7
- package/bundled/skills/prizmkit-test/assets/authoritative-records.schema.json +2 -1
- package/bundled/skills/prizmkit-test/assets/evidence-manifest.schema.json +1 -1
- package/bundled/skills/prizmkit-test/references/evidence-protocol.md +5 -5
- package/bundled/skills/prizmkit-test/references/evidence-request-protocol.md +2 -2
- package/bundled/skills/prizmkit-test/references/trusted-evidence-execution.md +2 -5
- package/bundled/skills/prizmkit-test/scripts/build_test_evidence.py +125 -46
- package/bundled/skills/prizmkit-test/scripts/validate_test_evidence.py +56 -2
- package/package.json +1 -1
package/bundled/VERSION.json
CHANGED
|
@@ -668,10 +668,10 @@ def _evidence_validator_passes(
|
|
|
668
668
|
|
|
669
669
|
def _resolve_evidence_validator(project_root: Path) -> Path | None:
|
|
670
670
|
candidates = (
|
|
671
|
+
project_root / "core" / "skills" / "prizmkit-skill" / "prizmkit-test" / "scripts" / "validate_test_evidence.py",
|
|
671
672
|
project_root / ".claude" / "command-assets" / "prizmkit-test" / "scripts" / "validate_test_evidence.py",
|
|
672
673
|
project_root / ".agents" / "skills" / "prizmkit-test" / "scripts" / "validate_test_evidence.py",
|
|
673
674
|
project_root / ".codebuddy" / "skills" / "prizmkit-test" / "scripts" / "validate_test_evidence.py",
|
|
674
|
-
project_root / "core" / "skills" / "prizmkit-skill" / "prizmkit-test" / "scripts" / "validate_test_evidence.py",
|
|
675
675
|
)
|
|
676
676
|
return next((path for path in candidates if path.is_file()), None)
|
|
677
677
|
|
|
@@ -237,16 +237,43 @@ def _discover_test_evidence_paths(checkpoint_path, data):
|
|
|
237
237
|
except (OSError, ValueError):
|
|
238
238
|
continue
|
|
239
239
|
evidence_dir = report.parent
|
|
240
|
+
evidence_root = (project_root / ".prizmkit" / "test" / "evidence").resolve()
|
|
241
|
+
try:
|
|
242
|
+
relative_evidence = evidence_dir.resolve().relative_to(evidence_root)
|
|
243
|
+
except (OSError, ValueError):
|
|
244
|
+
continue
|
|
245
|
+
if len(relative_evidence.parts) != 1:
|
|
246
|
+
continue
|
|
240
247
|
paths = {
|
|
241
|
-
"manifest":
|
|
242
|
-
"verdict":
|
|
243
|
-
"validation":
|
|
248
|
+
"manifest": evidence_dir / "manifest.json",
|
|
249
|
+
"verdict": evidence_dir / "verdict.json",
|
|
250
|
+
"validation": evidence_dir / "validation.json",
|
|
244
251
|
}
|
|
245
|
-
if all(
|
|
252
|
+
if not all(path.is_file() for path in paths.values()):
|
|
253
|
+
continue
|
|
254
|
+
try:
|
|
255
|
+
manifest = json.loads(paths["manifest"].read_text(encoding="utf-8"))
|
|
256
|
+
verdict = json.loads(paths["verdict"].read_text(encoding="utf-8"))
|
|
257
|
+
validation = json.loads(paths["validation"].read_text(encoding="utf-8"))
|
|
258
|
+
except (OSError, UnicodeDecodeError, json.JSONDecodeError):
|
|
259
|
+
continue
|
|
260
|
+
if not all(isinstance(record, dict) for record in (manifest, verdict, validation)):
|
|
261
|
+
continue
|
|
262
|
+
if (
|
|
263
|
+
manifest.get("evidence_id") != evidence_dir.name
|
|
264
|
+
or manifest.get("final_verdict") != "TEST_BLOCKED"
|
|
265
|
+
or verdict.get("verdict") != "TEST_BLOCKED"
|
|
266
|
+
or validation.get("result") != "passed"
|
|
267
|
+
or validation.get("verdict") != "TEST_BLOCKED"
|
|
268
|
+
):
|
|
269
|
+
continue
|
|
270
|
+
try:
|
|
246
271
|
return {
|
|
247
|
-
key: str(
|
|
248
|
-
for key,
|
|
272
|
+
key: str(path.resolve().relative_to(project_root.resolve()))
|
|
273
|
+
for key, path in paths.items()
|
|
249
274
|
}
|
|
275
|
+
except (OSError, ValueError):
|
|
276
|
+
continue
|
|
250
277
|
return None
|
|
251
278
|
|
|
252
279
|
|
|
@@ -323,10 +350,7 @@ def update_checkpoint(
|
|
|
323
350
|
return {"ok": False, "error": verdict_error}
|
|
324
351
|
|
|
325
352
|
semantic_update = _normalize_semantic_update(semantic_update)
|
|
326
|
-
if (
|
|
327
|
-
semantic_update.get("stage_result") == "TEST_BLOCKED"
|
|
328
|
-
and not semantic_update.get("authoritative_evidence_paths")
|
|
329
|
-
):
|
|
353
|
+
if semantic_update.get("stage_result") == "TEST_BLOCKED" and not semantic_update.get("authoritative_evidence_paths"):
|
|
330
354
|
discovered = _discover_test_evidence_paths(checkpoint_path, data)
|
|
331
355
|
if discovered:
|
|
332
356
|
semantic_update["authoritative_evidence_paths"] = discovered
|
|
@@ -337,7 +361,9 @@ def update_checkpoint(
|
|
|
337
361
|
stage_result = semantic_update.get("stage_result")
|
|
338
362
|
if step.get("skill") == "prizmkit-test" and stage_result in TEST_RESULTS:
|
|
339
363
|
evidence = semantic_update.get("authoritative_evidence_paths") or step.get("authoritative_evidence_paths")
|
|
340
|
-
if
|
|
364
|
+
if stage_result != "TEST_BLOCKED" and (
|
|
365
|
+
not isinstance(evidence, dict) or set(evidence) != {"manifest", "verdict", "validation"}
|
|
366
|
+
):
|
|
341
367
|
return {
|
|
342
368
|
"ok": False,
|
|
343
369
|
"error": "Every terminal test verdict requires manifest, verdict, and validation evidence paths",
|
|
@@ -445,8 +445,22 @@ def test_blocked_test_handoff_discovers_finalized_evidence_paths(tmp_path):
|
|
|
445
445
|
artifact.mkdir(parents=True)
|
|
446
446
|
evidence = tmp_path / ".prizmkit" / "test" / "evidence" / "evidence-1"
|
|
447
447
|
evidence.mkdir(parents=True)
|
|
448
|
-
|
|
449
|
-
|
|
448
|
+
(evidence / "manifest.json").write_text(
|
|
449
|
+
json.dumps({"evidence_id": "evidence-1", "final_verdict": "TEST_BLOCKED"}),
|
|
450
|
+
encoding="utf-8",
|
|
451
|
+
)
|
|
452
|
+
(evidence / "verdict.json").write_text(
|
|
453
|
+
json.dumps({"verdict": "TEST_BLOCKED"}),
|
|
454
|
+
encoding="utf-8",
|
|
455
|
+
)
|
|
456
|
+
(evidence / "validation.json").write_text(
|
|
457
|
+
json.dumps({"result": "passed", "verdict": "TEST_BLOCKED"}),
|
|
458
|
+
encoding="utf-8",
|
|
459
|
+
)
|
|
460
|
+
(evidence / "lifecycle.json").write_text(
|
|
461
|
+
json.dumps({"status": "sealed", "finalized": True}),
|
|
462
|
+
encoding="utf-8",
|
|
463
|
+
)
|
|
450
464
|
report = evidence / "test-report.md"
|
|
451
465
|
report.write_text("blocked", encoding="utf-8")
|
|
452
466
|
(artifact / "test-report-path.txt").write_text(str(report), encoding="utf-8")
|
|
@@ -487,6 +501,45 @@ def test_blocked_test_handoff_discovers_finalized_evidence_paths(tmp_path):
|
|
|
487
501
|
|
|
488
502
|
|
|
489
503
|
|
|
504
|
+
def test_blocked_test_handoff_without_finalized_evidence_preserves_test_result(tmp_path):
|
|
505
|
+
artifact = tmp_path / ".prizmkit" / "specs" / "feature"
|
|
506
|
+
artifact.mkdir(parents=True)
|
|
507
|
+
payload = _family_checkpoint(
|
|
508
|
+
"feature-pipeline",
|
|
509
|
+
[
|
|
510
|
+
"prizmkit-implement", "prizmkit-code-review", "prizmkit-test",
|
|
511
|
+
"prizmkit-retrospective", "prizmkit-committer", "completion-summary",
|
|
512
|
+
],
|
|
513
|
+
semantic={"artifact_dir": ".prizmkit/specs/feature"},
|
|
514
|
+
)
|
|
515
|
+
payload["feature_state"]["artifact_dir"] = ".prizmkit/specs/feature"
|
|
516
|
+
payload["steps"][1]["stage_result"] = "REVIEW_PASS"
|
|
517
|
+
payload["steps"][2]["status"] = "in_progress"
|
|
518
|
+
payload["steps"][2].pop("stage_result")
|
|
519
|
+
for step in payload["steps"][3:]:
|
|
520
|
+
step["status"] = "pending"
|
|
521
|
+
step.pop("stage_result", None)
|
|
522
|
+
checkpoint = _write(artifact / "workflow-checkpoint.json", payload)
|
|
523
|
+
|
|
524
|
+
result = update_checkpoint(
|
|
525
|
+
str(checkpoint),
|
|
526
|
+
"prizmkit-test",
|
|
527
|
+
"failed",
|
|
528
|
+
semantic_update={
|
|
529
|
+
"stage_result": "TEST_BLOCKED",
|
|
530
|
+
"blocked_reason": "test_blocked",
|
|
531
|
+
"terminal_status": "WORKFLOW_BLOCKED",
|
|
532
|
+
},
|
|
533
|
+
)
|
|
534
|
+
|
|
535
|
+
persisted = json.loads(checkpoint.read_text(encoding="utf-8"))
|
|
536
|
+
assert result["ok"] is True
|
|
537
|
+
assert result["semantic"]["stage_result"] == "TEST_BLOCKED"
|
|
538
|
+
assert persisted["steps"][2]["stage_result"] == "TEST_BLOCKED"
|
|
539
|
+
assert persisted["feature_state"]["stage_result"] == "TEST_BLOCKED"
|
|
540
|
+
assert "authoritative_evidence_paths" not in persisted["steps"][2]
|
|
541
|
+
|
|
542
|
+
|
|
490
543
|
def test_update_checkpoint_rejects_test_pass_when_validator_replay_fails(tmp_path, monkeypatch):
|
|
491
544
|
paths = _write_test_evidence(tmp_path)
|
|
492
545
|
payload = _family_checkpoint(
|
|
@@ -20,18 +20,18 @@ Fix only the protocol mechanisms needed for safe, replayable evidence: project/e
|
|
|
20
20
|
The Main Agent supplies semantic requests only; it does not author authoritative package records. Initialize and drive one evidence directory through the builder in this order:
|
|
21
21
|
|
|
22
22
|
```text
|
|
23
|
-
init → capture-change → inventory → prepare-tests → execute → finalize →
|
|
23
|
+
init → capture-change → inventory → prepare-tests → execute → finalize → validate --attest → resume
|
|
24
24
|
```
|
|
25
25
|
|
|
26
26
|
Use `${SKILL_DIR}/scripts/build_test_evidence.py` for every lifecycle command:
|
|
27
27
|
|
|
28
|
-
- `init`: creates the package directories and
|
|
28
|
+
- `init`: creates the package directories and pins the baseline identity.
|
|
29
29
|
- `capture-change`: captures the complete canonical change, including untracked, deleted, renamed, and copied files; never substitute plain `git diff`. `inventory` then requires its changed-file set to match the capture exactly and rejects stale captures or unaccounted live paths.
|
|
30
30
|
- `inventory`: hashes current source/tests/contracts/lockfiles, rejects category overlap, excludes runtime-only paths, and validates directory roots plus the complete capture-derived changed-file set.
|
|
31
31
|
- `prepare-tests`: discovers repository-native test files from the canonical capture and inventory, generates stable builder-owned test IDs, derives existing/added/modified status, and snapshots only added/modified live test sources.
|
|
32
32
|
- `execute`: runs native project commands and owns `executions.json`, `receipts/`, and `raw/`. Each request uses exactly one binding mode: `test_ids`, `test_paths`, or `auto_bind=true`. The builder infers recognized package-wide or explicit file/directory/glob command scope, blocks opaque commands or mismatched declarations, and persists verified canonical `test_ids` only.
|
|
33
|
-
- `finalize`:
|
|
34
|
-
- `render-report`: derives `test-report.md`; it is never authoritative.
|
|
33
|
+
- `finalize`: derives builder-owned target hashes, runs semantic-record and required-layer checks, rechecks canonical capture freshness, blocks every selected unreliable execution, and owns the payload manifest, verdict, generated-test snapshots, and evidence identity. It leaves the package in a mutable `payload-finalized` state until validator attestation; malformed semantic records fail before package relocation.
|
|
34
|
+
- `render-report`: recovery/debugging command that derives `test-report.md`; normal finalization and attestation render it automatically, and it is never authoritative.
|
|
35
35
|
- `resume`: preserves prior receipts and refuses to rebind an identity-changing package; create a new package after scope/source identity changes.
|
|
36
36
|
|
|
37
37
|
`test-plan.json` is the only source of builder-generated test IDs. An execution request provides exactly one of native `test_paths`, `auto_bind=true`, or canonical `test_ids`; the actual command must expose the same deterministic scope, including all prepared tests for recognized package-wide commands. The builder blocks opaque or mismatched command scope and rewrites verified requests to canonical `test_ids` only before execution. Assess all five layers explicitly, but set `required=true` only when risk requires it; every omitted layer needs structured N/A evidence. The same binding applies test ↔ behavior ↔ execution; current-workspace receipts are the execution evidence.
|
|
@@ -217,9 +217,11 @@ python3 ${SKILL_DIR}/scripts/validate_test_evidence.py \
|
|
|
217
217
|
--project-root <target-project-root>
|
|
218
218
|
```
|
|
219
219
|
|
|
220
|
-
|
|
220
|
+
The first pass validates the payload while the package remains `payload-finalized`; `validation.json` is intentionally empty until attestation. Builder-derived target hashes and lifecycle records are generated before this check. If this pass finds semantic errors, repair the mutable staging package and rerun finalize; never repair a sealed package.
|
|
221
221
|
|
|
222
|
-
|
|
222
|
+
The validator checks every authoritative record against its shipped schema, live target inventories, patch/diff binding, changed-file/module-root/exclusion/Regression Ring cross-links, generated-test snapshots, planned-test execution links, runner-generated receipts/request/raw-output hashes, successful behavior-risk mappings, production/unknown external-target safety, and package integrity.
|
|
223
|
+
|
|
224
|
+
### Second pass (attestation)
|
|
223
225
|
|
|
224
226
|
```bash
|
|
225
227
|
python3 ${SKILL_DIR}/scripts/validate_test_evidence.py \
|
|
@@ -228,7 +230,7 @@ python3 ${SKILL_DIR}/scripts/validate_test_evidence.py \
|
|
|
228
230
|
--attest
|
|
229
231
|
```
|
|
230
232
|
|
|
231
|
-
`--attest` writes an integrity/protocol attestation into `validation.json`, re-derives `test-report.md`, re-hashes final records in `manifest.json`, then runs a **second** strict validation against the refreshed package. A package is not a `TEST_PASS` until this second strict validation succeeds. Use builder replay when feasible to rerun recorded requests and create new linked receipts. Store validation output. Tampering, schema/identity/hash mismatch, source/test/lockfile drift, missing matrix mappings, wrong execution order, unjustified structured N/A, unresolved risk, or a false real-environment claim fails validation.
|
|
233
|
+
`--attest` writes an integrity/protocol attestation into `validation.json`, marks `EVIDENCE_VALIDATE` complete, re-derives `test-report.md`, re-hashes final records in `manifest.json`, seals the package, and then runs a **second** strict validation against the refreshed package. A package is not a `TEST_PASS` until this second strict validation succeeds. Use builder replay when feasible to rerun recorded requests and create new linked receipts. Store validation output. Tampering, schema/identity/hash mismatch, source/test/lockfile drift, missing matrix mappings, wrong execution order, unjustified structured N/A, unresolved risk, or a false real-environment claim fails validation.
|
|
232
234
|
|
|
233
235
|
## Resume and Idempotency
|
|
234
236
|
|
|
@@ -362,7 +362,7 @@
|
|
|
362
362
|
"executionReceipt": {
|
|
363
363
|
"type": "object",
|
|
364
364
|
"additionalProperties": false,
|
|
365
|
-
"required": ["receipt_format", "runner_path", "runner_sha256", "execution_id", "attempt_index", "request_path", "request_sha256", "purpose", "command", "cwd", "environment", "tool_versions", "layer", "test_ids", "external_targets", "exit_code", "stdout_path", "stderr_path", "stdout_sha256", "stderr_sha256", "selected_execution", "reliable", "started_at", "finished_at"],
|
|
365
|
+
"required": ["receipt_format", "runner_path", "runner_sha256", "execution_id", "attempt_index", "request_path", "request_sha256", "purpose", "command", "cwd", "environment", "tool_versions", "layer", "test_ids", "external_targets", "timeout_seconds", "exit_code", "stdout_path", "stderr_path", "stdout_sha256", "stderr_sha256", "selected_execution", "reliable", "started_at", "finished_at"],
|
|
366
366
|
"properties": {
|
|
367
367
|
"receipt_format": { "const": "prizmkit-runner-generated-v1" },
|
|
368
368
|
"runner_path": { "$ref": "#/$defs/path" },
|
|
@@ -381,6 +381,7 @@
|
|
|
381
381
|
"test_paths": { "type": "array", "items": { "$ref": "#/$defs/path" } },
|
|
382
382
|
"auto_bind": { "type": "boolean" },
|
|
383
383
|
"external_targets": { "type": "array", "items": { "$ref": "#/$defs/externalTarget" } },
|
|
384
|
+
"timeout_seconds": { "type": "number", "exclusiveMinimum": 0 },
|
|
384
385
|
"exit_code": { "type": "integer" },
|
|
385
386
|
"stdout_path": { "$ref": "#/$defs/path" },
|
|
386
387
|
"stderr_path": { "$ref": "#/$defs/path" },
|
|
@@ -82,7 +82,7 @@
|
|
|
82
82
|
"EVIDENCE_VALIDATE"
|
|
83
83
|
]
|
|
84
84
|
},
|
|
85
|
-
"status": { "enum": ["complete", "not_applicable"] },
|
|
85
|
+
"status": { "enum": ["complete", "pending", "not_applicable"] },
|
|
86
86
|
"outputs": { "type": "array", "items": { "type": "string", "minLength": 1 } },
|
|
87
87
|
"not_applicable": {
|
|
88
88
|
"type": ["object", "null"],
|
|
@@ -24,12 +24,12 @@ Hashes and validator receipts provide byte-level binding and drift detection, no
|
|
|
24
24
|
|
|
25
25
|
## Bounded Protocol Recovery
|
|
26
26
|
|
|
27
|
-
Evidence-protocol repair is separate from production/test repair. The builder permits only a bounded recovery budget for semantic-record preflight and package initialization. After the budget is exhausted, return `TEST_BLOCKED`; do not create another versioned evidence package
|
|
27
|
+
Evidence-protocol repair is separate from production/test repair. The builder permits only a bounded recovery budget for semantic-record preflight and package initialization. After the budget is exhausted, return `TEST_BLOCKED`; do not create another versioned evidence package or hand-edit authoritative records. A payload-finalized package remains mutable only until validator attestation; after attestation it is sealed and immutable. `Evidence validation: PASS` means only that the package is structurally valid; the testing verdict must be read separately from `Testing verdict: TEST_PASS | TEST_FAIL | TEST_BLOCKED`.
|
|
28
28
|
|
|
29
29
|
|
|
30
30
|
The evidence package is an audit of repository behavior, not a request to invent a parallel test suite. Discover and execute the smallest relevant native tests before adding anything. Add, modify, or delete tests only in the target repository. The builder's `prepare-tests` command derives stable test IDs and change status from canonical Git capture, and snapshots only added or modified repository-native tests. `generated-tests/` is an evidence archive, never an execution workspace; it may be empty when existing tests cover the behavior.
|
|
31
31
|
|
|
32
|
-
The canonical package lifecycle is `init → capture-change → inventory → prepare-tests → execute → finalize →
|
|
32
|
+
The canonical package lifecycle is `init → capture-change → inventory → prepare-tests → execute → finalize → validate --attest → resume`. Builder commands own `source-change.patch`, native test preparation, receipts, raw outputs, manifest, verdict, and initial report. The validator alone owns `validation.json` and refreshes the report during attestation. Inventory must consume a fresh capture and exactly preserve its complete changed-file set. Canonical behavior packages require a schema-valid `test-preparation.json` bound byte-for-byte to the builder-generated `test-plan.json` and its hashed semantic request. Evidence identity is derived from the resolved baseline, canonical patch hash, and scope hash; identity-changing resume creates a new package instead of rewriting the old directory.
|
|
33
33
|
|
|
34
34
|
|
|
35
35
|
Run exactly these states in order:
|
|
@@ -167,12 +167,12 @@ python3 ${SKILL_DIR}/scripts/validate_test_evidence.py \
|
|
|
167
167
|
--attest
|
|
168
168
|
```
|
|
169
169
|
|
|
170
|
-
`--attest` first validates the
|
|
171
|
-
|
|
172
|
-
When feasible, replay recorded requests through `build_test_evidence.py execute --replay-receipt ...`. Replay performs a new real execution and emits a new linked receipt; it does not pretend the attestation establishes what historically ran.
|
|
170
|
+
`--attest` first validates the mutable `payload-finalized` package while excluding only validator-owned attestation/view records, atomically writes `validation.json`, marks validation complete, refreshes final file hashes, seals the lifecycle, then immediately performs a complete second validation. Subsequent checks omit `--attest` and verify the stored attestation. This avoids trusting a caller-authored `passed` marker or creating a self-referential manifest. The attestation proves package integrity and protocol consistency only; it is not proof against a same-permission producer.
|
|
173
171
|
|
|
174
172
|
The validator checks all shipped JSON schemas, evidence-directory identity, live target-project inventories and aggregate hashes, changed-file/module-root/exclusion/Regression Ring cross-links, patch/diff binding, stage ordering and output ownership, generated-test snapshot existence, planned test/inventory/execution linkage, module/matrix completeness, successful selected runner receipts with request/raw-output binding, structured N/A and obvious signal conflicts, production/unknown external-target safety, strict verdict semantics, and honest mocked-versus-real claims. `TEST_PASS` requires deterministic validator success.
|
|
175
173
|
|
|
174
|
+
When feasible, replay recorded requests through `build_test_evidence.py execute --replay-receipt ...`. Replay performs a new real execution and emits a new linked receipt; it does not pretend the attestation establishes what historically ran.
|
|
175
|
+
|
|
176
176
|
## Resume and Invalidation
|
|
177
177
|
|
|
178
178
|
On invocation for an existing evidence target:
|
|
@@ -17,7 +17,7 @@ Never hand-author `executions.json` receipt fields. Never copy a command's claim
|
|
|
17
17
|
|
|
18
18
|
## Producer Boundary and Recovery
|
|
19
19
|
|
|
20
|
-
The model supplies schema-shaped semantic requests and mutable semantic records only before finalization. The builder performs preflight
|
|
20
|
+
The model supplies schema-shaped semantic requests and mutable semantic records only before payload finalization. The builder performs structural and derived-field preflight before creating a payload-finalized package. Validator attestation then seals it. Once sealed, the returned `evidence_dir`, `manifest_path`, `verdict_path`, and `validation_path` are the only valid handoff paths; do not continue using the old staging directory. A validator pass is not a testing pass: read the testing verdict separately.
|
|
21
21
|
|
|
22
22
|
|
|
23
23
|
Place a request under `requests/` with:
|
|
@@ -41,7 +41,7 @@ Use an argv array so the runner does not interpret a shell string. Required fiel
|
|
|
41
41
|
- selected `layer` and exactly one canonical binding (`test_ids`, `test_paths`, or `auto_bind: true`);
|
|
42
42
|
- structured `external_targets`.
|
|
43
43
|
|
|
44
|
-
Optional `timeout_seconds`, `attempt_policy`, and `concurrency` remain model-chosen. The builder
|
|
44
|
+
Optional `timeout_seconds`, `attempt_policy`, and `concurrency` remain model-chosen. The builder applies a bounded default and maximum timeout; it does not silently retry or claim concurrency that it did not execute.
|
|
45
45
|
|
|
46
46
|
The request path must be beneath the evidence directory. `cwd` resolves beneath the selected execution root. Missing commands produce an actual exit 127 receipt; missing cwd or escaped paths block before execution.
|
|
47
47
|
|
|
@@ -41,7 +41,7 @@ Treat evidence as auditable project-controlled provenance, not a defense against
|
|
|
41
41
|
|
|
42
42
|
## Builder Commands
|
|
43
43
|
|
|
44
|
-
Drive a new package through the canonical lifecycle. Semantic request files remain model-authored; authoritative package records remain tool-owned
|
|
44
|
+
Drive a new package through the canonical lifecycle. Semantic request files remain model-authored; authoritative package records remain tool-owned. The builder applies a bounded default timeout when the request omits one, preserves common native cache variables, and returns a bounded unreliable receipt on timeout. The normal user path should author one semantic request and one execution plan; use individual lifecycle commands only for recovery or protocol debugging.
|
|
45
45
|
|
|
46
46
|
```bash
|
|
47
47
|
python3 ${SKILL_DIR}/scripts/build_test_evidence.py --project-root <root> --evidence-dir <staging> init --baseline <commit>
|
|
@@ -49,14 +49,11 @@ python3 ${SKILL_DIR}/scripts/build_test_evidence.py --project-root <root> --evid
|
|
|
49
49
|
python3 ${SKILL_DIR}/scripts/build_test_evidence.py --project-root <root> --evidence-dir <staging> inventory --request requests/inventory.json
|
|
50
50
|
python3 ${SKILL_DIR}/scripts/build_test_evidence.py --project-root <root> --evidence-dir <staging> execute --request requests/focused.json
|
|
51
51
|
python3 ${SKILL_DIR}/scripts/build_test_evidence.py --project-root <root> --evidence-dir <staging> finalize --request requests/finalize.json
|
|
52
|
-
python3 ${SKILL_DIR}/scripts/build_test_evidence.py --project-root <root> --evidence-dir <staging> render-report
|
|
53
52
|
```
|
|
54
53
|
|
|
55
|
-
|
|
54
|
+
`<staging>` must be a child of `.prizmkit/test/evidence/`. After `finalize`, the package is `payload-finalized`; run the validator pre-check and then `validate_test_evidence.py --attest` to seal it. Do not treat the pre-attestation staging package as immutable or invoke `render-report` after sealing.
|
|
56
55
|
|
|
57
56
|
|
|
58
|
-
Write requests beneath the evidence directory, then invoke the bundled script:
|
|
59
|
-
|
|
60
57
|
```bash
|
|
61
58
|
python3 ${SKILL_DIR}/scripts/build_test_evidence.py \
|
|
62
59
|
--project-root <target-project-root> \
|
|
@@ -61,6 +61,10 @@ _IGNORED_DIR_NAMES = {".git", "__pycache__", ".pytest_cache", ".mypy_cache"}
|
|
|
61
61
|
FINAL_RECORDS = {"manifest.json", "validation.json", "verdict.json", "test-report.md"}
|
|
62
62
|
MAX_PROTOCOL_PACKAGE_ATTEMPTS = 2
|
|
63
63
|
PROTOCOL_STATE_NAME = ".protocol-recovery.json"
|
|
64
|
+
DEFAULT_EXECUTION_TIMEOUT_SECONDS = 300.0
|
|
65
|
+
MAX_EXECUTION_TIMEOUT_SECONDS = 1800.0
|
|
66
|
+
MAX_EXECUTION_OUTPUT_BYTES = 16 * 1024 * 1024
|
|
67
|
+
CANONICAL_EVIDENCE_ROOT = (".prizmkit", "test", "evidence")
|
|
64
68
|
|
|
65
69
|
|
|
66
70
|
class RequestError(Exception):
|
|
@@ -114,6 +118,34 @@ def confined(root: Path, value: str, *, must_exist: bool = False, directory: boo
|
|
|
114
118
|
return candidate
|
|
115
119
|
|
|
116
120
|
|
|
121
|
+
def canonical_evidence_root(project_root: Path) -> Path:
|
|
122
|
+
return project_root.joinpath(*CANONICAL_EVIDENCE_ROOT).resolve()
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def require_canonical_evidence_dir(project_root: Path, evidence_dir: Path) -> None:
|
|
126
|
+
root = canonical_evidence_root(project_root)
|
|
127
|
+
try:
|
|
128
|
+
evidence_dir.resolve().relative_to(root)
|
|
129
|
+
except ValueError as exc:
|
|
130
|
+
raise RequestError(
|
|
131
|
+
"evidence directory must be under .prizmkit/test/evidence"
|
|
132
|
+
) from exc
|
|
133
|
+
if evidence_dir.resolve() == root:
|
|
134
|
+
raise RequestError("evidence directory must be a package child of .prizmkit/test/evidence")
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def max_timeout(value: Any) -> float:
|
|
138
|
+
if value is None:
|
|
139
|
+
return DEFAULT_EXECUTION_TIMEOUT_SECONDS
|
|
140
|
+
if isinstance(value, bool) or not isinstance(value, (int, float)) or value <= 0:
|
|
141
|
+
raise RequestError("timeout_seconds must be a positive number")
|
|
142
|
+
if value > MAX_EXECUTION_TIMEOUT_SECONDS:
|
|
143
|
+
raise RequestError(
|
|
144
|
+
f"timeout_seconds must not exceed {MAX_EXECUTION_TIMEOUT_SECONDS:g} seconds"
|
|
145
|
+
)
|
|
146
|
+
return float(value)
|
|
147
|
+
|
|
148
|
+
|
|
117
149
|
def evidence_relative(evidence_dir: Path, path: Path) -> str:
|
|
118
150
|
try:
|
|
119
151
|
return path.resolve().relative_to(evidence_dir.resolve()).as_posix()
|
|
@@ -794,6 +826,7 @@ def run_inventory(project_root: Path, evidence_dir: Path, request_path: Path, ou
|
|
|
794
826
|
|
|
795
827
|
|
|
796
828
|
def run_process(argv: list[str], cwd: Path, environment: dict[str, str], timeout: float | None) -> tuple[int, bytes, bytes]:
|
|
829
|
+
timeout_value = max_timeout(timeout)
|
|
797
830
|
creationflags = 0
|
|
798
831
|
process_kwargs: dict[str, Any] = {}
|
|
799
832
|
if os.name == "nt":
|
|
@@ -805,7 +838,7 @@ def run_process(argv: list[str], cwd: Path, environment: dict[str, str], timeout
|
|
|
805
838
|
creationflags=creationflags, **process_kwargs,
|
|
806
839
|
)
|
|
807
840
|
try:
|
|
808
|
-
stdout, stderr = process.communicate(timeout=
|
|
841
|
+
stdout, stderr = process.communicate(timeout=timeout_value)
|
|
809
842
|
return process.returncode, stdout, stderr
|
|
810
843
|
except subprocess.TimeoutExpired:
|
|
811
844
|
if os.name == "nt":
|
|
@@ -823,7 +856,14 @@ def run_process(argv: list[str], cwd: Path, environment: dict[str, str], timeout
|
|
|
823
856
|
except ProcessLookupError:
|
|
824
857
|
pass
|
|
825
858
|
stdout, stderr = process.communicate()
|
|
826
|
-
return 124, stdout, stderr + f"\nPrizmKit runner timeout after {
|
|
859
|
+
return 124, stdout, stderr + f"\nPrizmKit runner timeout after {timeout_value:g} seconds\n".encode()
|
|
860
|
+
|
|
861
|
+
|
|
862
|
+
def limit_output(value: bytes, label: str) -> bytes:
|
|
863
|
+
if len(value) <= MAX_EXECUTION_OUTPUT_BYTES:
|
|
864
|
+
return value
|
|
865
|
+
marker = f"\nPrizmKit runner {label} truncated after {MAX_EXECUTION_OUTPUT_BYTES} bytes\n".encode()
|
|
866
|
+
return value[:MAX_EXECUTION_OUTPUT_BYTES - len(marker)] + marker
|
|
827
867
|
|
|
828
868
|
|
|
829
869
|
def run_probe(argv: list[str], cwd: Path, environment: dict[str, str], timeout: float | None) -> dict[str, Any]:
|
|
@@ -831,7 +871,8 @@ def run_probe(argv: list[str], cwd: Path, environment: dict[str, str], timeout:
|
|
|
831
871
|
exit_code, stdout, stderr = run_process(argv, cwd, environment, timeout)
|
|
832
872
|
return {
|
|
833
873
|
"command": argv, "exit_code": exit_code,
|
|
834
|
-
"stdout": stdout
|
|
874
|
+
"stdout": limit_output(stdout, "probe stdout").decode(errors="replace"),
|
|
875
|
+
"stderr": limit_output(stderr, "probe stderr").decode(errors="replace"),
|
|
835
876
|
}
|
|
836
877
|
except OSError as exc:
|
|
837
878
|
return {"command": argv, "error": str(exc)}
|
|
@@ -883,11 +924,15 @@ def execute_request(
|
|
|
883
924
|
if original_binding_fields != {"test_ids"} or original_test_ids != request["test_ids"]:
|
|
884
925
|
write_json_atomic(request_path, request)
|
|
885
926
|
cwd = confined(project_root, request["cwd"] or ".", must_exist=True, directory=True)
|
|
886
|
-
complete_environment = {
|
|
927
|
+
complete_environment = {
|
|
928
|
+
key: value for key, value in os.environ.items()
|
|
929
|
+
if key in {"PATH", "HOME", "TMPDIR", "TMP", "TEMP", "GOMODCACHE", "GOPATH", "GOCACHE", "NPM_CONFIG_CACHE", "PIP_CACHE_DIR"}
|
|
930
|
+
}
|
|
931
|
+
complete_environment.setdefault("PATH", os.defpath)
|
|
887
932
|
if os.name == "nt" and os.environ.get("SYSTEMROOT"):
|
|
888
933
|
complete_environment["SYSTEMROOT"] = os.environ["SYSTEMROOT"]
|
|
889
934
|
complete_environment.update(request["environment"])
|
|
890
|
-
timeout = request.get("timeout_seconds")
|
|
935
|
+
timeout = max_timeout(request.get("timeout_seconds"))
|
|
891
936
|
tools = {
|
|
892
937
|
name: run_probe(argv, cwd, complete_environment, timeout)
|
|
893
938
|
for name, argv in request["tool_version_commands"].items()
|
|
@@ -906,19 +951,24 @@ def execute_request(
|
|
|
906
951
|
stderr_path = raw_dir / f"{execution_id}.stderr"
|
|
907
952
|
started = dt.datetime.now(dt.timezone.utc).isoformat()
|
|
908
953
|
timed_out = False
|
|
954
|
+
output_limited = False
|
|
909
955
|
execution_error = False
|
|
910
956
|
try:
|
|
911
957
|
exit_code, stdout, stderr = run_process(
|
|
912
958
|
request["command"], cwd, complete_environment, timeout,
|
|
913
959
|
)
|
|
914
960
|
timed_out = exit_code == 124 and b"PrizmKit runner timeout" in stderr
|
|
961
|
+
output_limited = (
|
|
962
|
+
len(stdout) > MAX_EXECUTION_OUTPUT_BYTES
|
|
963
|
+
or len(stderr) > MAX_EXECUTION_OUTPUT_BYTES
|
|
964
|
+
)
|
|
915
965
|
except OSError as exc:
|
|
916
966
|
exit_code = 127
|
|
917
967
|
stdout = b""
|
|
918
968
|
stderr = f"PrizmKit runner execution error: {exc}\n".encode()
|
|
919
969
|
execution_error = True
|
|
920
|
-
stdout_path.write_bytes(stdout)
|
|
921
|
-
stderr_path.write_bytes(stderr)
|
|
970
|
+
stdout_path.write_bytes(limit_output(stdout, "stdout"))
|
|
971
|
+
stderr_path.write_bytes(limit_output(stderr, "stderr"))
|
|
922
972
|
finished = dt.datetime.now(dt.timezone.utc).isoformat()
|
|
923
973
|
existing = load_json(evidence_dir / "executions.json") if (evidence_dir / "executions.json").exists() else []
|
|
924
974
|
if not isinstance(existing, list):
|
|
@@ -940,13 +990,14 @@ def execute_request(
|
|
|
940
990
|
"layer": request["layer"],
|
|
941
991
|
"test_ids": request["test_ids"],
|
|
942
992
|
"external_targets": request["external_targets"],
|
|
993
|
+
"timeout_seconds": timeout,
|
|
943
994
|
"exit_code": exit_code,
|
|
944
995
|
"stdout_path": evidence_relative(evidence_dir, stdout_path),
|
|
945
996
|
"stderr_path": evidence_relative(evidence_dir, stderr_path),
|
|
946
997
|
"stdout_sha256": file_sha256(stdout_path),
|
|
947
998
|
"stderr_sha256": file_sha256(stderr_path),
|
|
948
999
|
"selected_execution": selected_execution,
|
|
949
|
-
"reliable": probes_reliable and not timed_out and not execution_error,
|
|
1000
|
+
"reliable": probes_reliable and not timed_out and not output_limited and not execution_error,
|
|
950
1001
|
"started_at": started,
|
|
951
1002
|
"finished_at": finished,
|
|
952
1003
|
"replay_of": replay_of,
|
|
@@ -962,7 +1013,13 @@ def run_capture_change(
|
|
|
962
1013
|
baseline: str | None,
|
|
963
1014
|
output_name: str,
|
|
964
1015
|
) -> Path:
|
|
1016
|
+
lifecycle = load_lifecycle(evidence_dir, required=True)
|
|
1017
|
+
expected_baseline = lifecycle.get("baseline_commit")
|
|
965
1018
|
resolved_baseline = git_baseline(project_root, baseline)
|
|
1019
|
+
if isinstance(expected_baseline, str) and resolved_baseline != expected_baseline:
|
|
1020
|
+
raise RequestError(
|
|
1021
|
+
"capture baseline does not match the baseline pinned during init"
|
|
1022
|
+
)
|
|
966
1023
|
patch, status_entries = canonical_change_patch(project_root, resolved_baseline)
|
|
967
1024
|
patch_path = confined(evidence_dir, "source-change.patch")
|
|
968
1025
|
patch_path.parent.mkdir(parents=True, exist_ok=True)
|
|
@@ -980,21 +1037,23 @@ def run_capture_change(
|
|
|
980
1037
|
|
|
981
1038
|
|
|
982
1039
|
def run_init(project_root: Path, evidence_dir: Path, baseline: str | None, output_name: str) -> Path:
|
|
983
|
-
|
|
984
|
-
|
|
1040
|
+
require_canonical_evidence_dir(project_root, evidence_dir)
|
|
1041
|
+
align_protocol_state(project_root, protocol_run_key(project_root, baseline))
|
|
1042
|
+
require_protocol_budget(project_root, package_initialization=True)
|
|
985
1043
|
if evidence_dir.exists() and any(evidence_dir.iterdir()):
|
|
986
1044
|
if (evidence_dir / "manifest.json").exists():
|
|
987
1045
|
raise RequestError("evidence package is finalized; initialize a new evidence directory")
|
|
988
1046
|
raise RequestError("evidence directory is not empty; initialize a new evidence directory")
|
|
989
1047
|
evidence_dir.mkdir(parents=True, exist_ok=True)
|
|
990
|
-
|
|
1048
|
+
resolved_baseline = git_baseline(project_root, baseline)
|
|
1049
|
+
update_protocol_state(project_root, event="package-initialized", increment="package_initializations")
|
|
991
1050
|
for directory in ("requests", "receipts", "raw", "generated-tests", "contracts", "runner"):
|
|
992
1051
|
(evidence_dir / directory).mkdir(parents=True, exist_ok=True)
|
|
993
1052
|
write_json_atomic(evidence_dir / "executions.json", [])
|
|
994
1053
|
write_json_atomic(evidence_dir / "lifecycle.json", {
|
|
995
1054
|
"lifecycle_format": "prizmkit-canonical-lifecycle-v1",
|
|
996
1055
|
"status": "initialized",
|
|
997
|
-
"baseline_commit":
|
|
1056
|
+
"baseline_commit": resolved_baseline,
|
|
998
1057
|
"artifact_dir": None,
|
|
999
1058
|
"finalized": False,
|
|
1000
1059
|
"immutable_identity": None,
|
|
@@ -1003,29 +1062,18 @@ def run_init(project_root: Path, evidence_dir: Path, baseline: str | None, outpu
|
|
|
1003
1062
|
write_json_atomic(output, {
|
|
1004
1063
|
"command": "init",
|
|
1005
1064
|
"evidence_dir": ".",
|
|
1006
|
-
"baseline_commit":
|
|
1065
|
+
"baseline_commit": resolved_baseline,
|
|
1007
1066
|
"next": "capture-change",
|
|
1008
1067
|
})
|
|
1009
1068
|
return output
|
|
1010
1069
|
|
|
1011
1070
|
|
|
1012
|
-
def protocol_state_path(
|
|
1013
|
-
|
|
1014
|
-
# cannot reset the budget by choosing another evidence directory.
|
|
1015
|
-
return evidence_dir.parent / PROTOCOL_STATE_NAME
|
|
1016
|
-
|
|
1017
|
-
|
|
1018
|
-
def protocol_run_key(project_root: Path, baseline: str | None) -> str:
|
|
1019
|
-
resolved_baseline = git_baseline(project_root, baseline)
|
|
1020
|
-
patch, _ = canonical_change_patch(project_root, resolved_baseline)
|
|
1021
|
-
return canonical_sha256({
|
|
1022
|
-
"baseline_commit": resolved_baseline,
|
|
1023
|
-
"working_diff_sha256": hashlib.sha256(patch).hexdigest(),
|
|
1024
|
-
})
|
|
1071
|
+
def protocol_state_path(project_root: Path) -> Path:
|
|
1072
|
+
return canonical_evidence_root(project_root) / PROTOCOL_STATE_NAME
|
|
1025
1073
|
|
|
1026
1074
|
|
|
1027
|
-
def align_protocol_state(
|
|
1028
|
-
state_path = protocol_state_path(
|
|
1075
|
+
def align_protocol_state(project_root: Path, run_key: str) -> None:
|
|
1076
|
+
state_path = protocol_state_path(project_root)
|
|
1029
1077
|
if state_path.exists():
|
|
1030
1078
|
state = ensure_object(load_json(state_path), "protocol recovery state")
|
|
1031
1079
|
if state.get("run_key") == run_key:
|
|
@@ -1034,13 +1082,13 @@ def align_protocol_state(evidence_dir: Path, run_key: str) -> None:
|
|
|
1034
1082
|
|
|
1035
1083
|
|
|
1036
1084
|
def update_protocol_state(
|
|
1037
|
-
|
|
1085
|
+
project_root: Path,
|
|
1038
1086
|
*,
|
|
1039
1087
|
event: str,
|
|
1040
1088
|
error: str | None = None,
|
|
1041
1089
|
increment: str | None = None,
|
|
1042
1090
|
) -> dict[str, Any]:
|
|
1043
|
-
state_path = protocol_state_path(
|
|
1091
|
+
state_path = protocol_state_path(project_root)
|
|
1044
1092
|
state: dict[str, Any] = {}
|
|
1045
1093
|
if state_path.exists():
|
|
1046
1094
|
state = ensure_object(load_json(state_path), "protocol recovery state")
|
|
@@ -1061,8 +1109,8 @@ def update_protocol_state(
|
|
|
1061
1109
|
return state
|
|
1062
1110
|
|
|
1063
1111
|
|
|
1064
|
-
def require_protocol_budget(
|
|
1065
|
-
state_path = protocol_state_path(
|
|
1112
|
+
def require_protocol_budget(project_root: Path, *, package_initialization: bool = False) -> None:
|
|
1113
|
+
state_path = protocol_state_path(project_root)
|
|
1066
1114
|
if not state_path.exists():
|
|
1067
1115
|
return
|
|
1068
1116
|
state = ensure_object(load_json(state_path), "protocol recovery state")
|
|
@@ -1078,6 +1126,15 @@ def require_protocol_budget(evidence_dir: Path, *, package_initialization: bool
|
|
|
1078
1126
|
)
|
|
1079
1127
|
|
|
1080
1128
|
|
|
1129
|
+
def protocol_run_key(project_root: Path, baseline: str | None) -> str:
|
|
1130
|
+
resolved_baseline = git_baseline(project_root, baseline)
|
|
1131
|
+
patch, _ = canonical_change_patch(project_root, resolved_baseline)
|
|
1132
|
+
return canonical_sha256({
|
|
1133
|
+
"baseline_commit": resolved_baseline,
|
|
1134
|
+
"working_diff_sha256": hashlib.sha256(patch).hexdigest(),
|
|
1135
|
+
})
|
|
1136
|
+
|
|
1137
|
+
|
|
1081
1138
|
def semantic_record_paths(change_class: str) -> set[str]:
|
|
1082
1139
|
common = {"change-classification.json", "scope.json"}
|
|
1083
1140
|
if change_class == "behavior":
|
|
@@ -1085,7 +1142,7 @@ def semantic_record_paths(change_class: str) -> set[str]:
|
|
|
1085
1142
|
return common | {"lightweight-verification.json"}
|
|
1086
1143
|
|
|
1087
1144
|
|
|
1088
|
-
def
|
|
1145
|
+
def require_semantic_record_files(evidence_dir: Path, change_class: str) -> None:
|
|
1089
1146
|
missing = sorted(
|
|
1090
1147
|
relative for relative in semantic_record_paths(change_class)
|
|
1091
1148
|
if not (evidence_dir / relative).is_file()
|
|
@@ -1096,6 +1153,9 @@ def preflight_semantic_records(evidence_dir: Path, change_class: str) -> None:
|
|
|
1096
1153
|
+ "; keep this package mutable and repair it before finalize"
|
|
1097
1154
|
)
|
|
1098
1155
|
|
|
1156
|
+
|
|
1157
|
+
def preflight_semantic_records(evidence_dir: Path, change_class: str) -> None:
|
|
1158
|
+
require_semantic_record_files(evidence_dir, change_class)
|
|
1099
1159
|
validator_path = Path(__file__).resolve().with_name("validate_test_evidence.py")
|
|
1100
1160
|
try:
|
|
1101
1161
|
import importlib.util
|
|
@@ -1423,7 +1483,7 @@ def stage_output_owner(relative: str, change_class: str) -> str:
|
|
|
1423
1483
|
return "EVIDENCE_PACKAGE"
|
|
1424
1484
|
|
|
1425
1485
|
|
|
1426
|
-
def build_manifest(evidence_dir: Path, *, evidence_id: str, identity: dict[str, Any], target_hashes: dict[str, str], change_class: str, baseline_commit: str, final_verdict: str) -> dict[str, Any]:
|
|
1486
|
+
def build_manifest(evidence_dir: Path, *, evidence_id: str, identity: dict[str, Any], target_hashes: dict[str, str], change_class: str, baseline_commit: str, final_verdict: str, attested: bool = False) -> dict[str, Any]:
|
|
1427
1487
|
file_paths = sorted(
|
|
1428
1488
|
path.relative_to(evidence_dir).as_posix()
|
|
1429
1489
|
for path in evidence_dir.rglob("*")
|
|
@@ -1441,7 +1501,7 @@ def build_manifest(evidence_dir: Path, *, evidence_id: str, identity: dict[str,
|
|
|
1441
1501
|
outputs = sorted(outputs_by_stage.get(stage, []))
|
|
1442
1502
|
stages.append({
|
|
1443
1503
|
"name": stage,
|
|
1444
|
-
"status": "complete",
|
|
1504
|
+
"status": "complete" if stage != "EVIDENCE_VALIDATE" or attested else "pending",
|
|
1445
1505
|
"outputs": outputs,
|
|
1446
1506
|
"not_applicable": None,
|
|
1447
1507
|
})
|
|
@@ -1476,7 +1536,8 @@ def write_handoff_pointer(evidence_dir: Path, artifact_dir: str | None, project_
|
|
|
1476
1536
|
|
|
1477
1537
|
|
|
1478
1538
|
def run_finalize(project_root: Path, evidence_dir: Path, request: dict[str, Any], output_name: str) -> Path:
|
|
1479
|
-
|
|
1539
|
+
require_canonical_evidence_dir(project_root, evidence_dir)
|
|
1540
|
+
require_protocol_budget(project_root)
|
|
1480
1541
|
allowed = {"request_version", "change_class", "baseline_commit", "scope_path", "environment_path", "artifact_dir", "blockers", "repair_scope"}
|
|
1481
1542
|
if set(request) - allowed:
|
|
1482
1543
|
raise RequestError(f"finalize request contains unsupported fields: {sorted(set(request) - allowed)}")
|
|
@@ -1491,7 +1552,10 @@ def run_finalize(project_root: Path, evidence_dir: Path, request: dict[str, Any]
|
|
|
1491
1552
|
if not capture_path.exists():
|
|
1492
1553
|
run_capture_change(project_root, evidence_dir, request.get("baseline_commit"), "change-capture.json")
|
|
1493
1554
|
capture = load_change_capture(project_root, evidence_dir, require_fresh=True)
|
|
1555
|
+
lifecycle_baseline = lifecycle.get("baseline_commit")
|
|
1494
1556
|
baseline = capture.get("baseline_commit")
|
|
1557
|
+
if not isinstance(lifecycle_baseline, str) or baseline != lifecycle_baseline:
|
|
1558
|
+
raise RequestError("canonical change capture baseline does not match init baseline")
|
|
1495
1559
|
patch_path = evidence_dir / "source-change.patch"
|
|
1496
1560
|
if not isinstance(baseline, str) or not patch_path.is_file():
|
|
1497
1561
|
raise RequestError("canonical change capture is incomplete")
|
|
@@ -1499,10 +1563,10 @@ def run_finalize(project_root: Path, evidence_dir: Path, request: dict[str, Any]
|
|
|
1499
1563
|
if change_class not in {"behavior", "lightweight"}:
|
|
1500
1564
|
raise RequestError("change_class must be behavior or lightweight")
|
|
1501
1565
|
try:
|
|
1502
|
-
|
|
1566
|
+
require_semantic_record_files(evidence_dir, change_class)
|
|
1503
1567
|
except RequestError as exc:
|
|
1504
1568
|
state = update_protocol_state(
|
|
1505
|
-
|
|
1569
|
+
project_root,
|
|
1506
1570
|
event="preflight-failed",
|
|
1507
1571
|
error=str(exc),
|
|
1508
1572
|
increment="preflight_failures",
|
|
@@ -1513,6 +1577,7 @@ def run_finalize(project_root: Path, evidence_dir: Path, request: dict[str, Any]
|
|
|
1513
1577
|
"and do not initialize another package"
|
|
1514
1578
|
) from exc
|
|
1515
1579
|
raise
|
|
1580
|
+
|
|
1516
1581
|
artifact_dir = request.get("artifact_dir")
|
|
1517
1582
|
if artifact_dir is not None and not isinstance(artifact_dir, str):
|
|
1518
1583
|
raise RequestError("artifact_dir must be a project-relative directory")
|
|
@@ -1540,6 +1605,21 @@ def run_finalize(project_root: Path, evidence_dir: Path, request: dict[str, Any]
|
|
|
1540
1605
|
write_json_atomic(evidence_dir / "test-plan.json", plan)
|
|
1541
1606
|
scope["target_hashes"] = target_hashes
|
|
1542
1607
|
write_json_atomic(evidence_dir / scope_relative, scope)
|
|
1608
|
+
try:
|
|
1609
|
+
preflight_semantic_records(evidence_dir, change_class)
|
|
1610
|
+
except RequestError as exc:
|
|
1611
|
+
state = update_protocol_state(
|
|
1612
|
+
project_root,
|
|
1613
|
+
event="preflight-failed",
|
|
1614
|
+
error=str(exc),
|
|
1615
|
+
increment="preflight_failures",
|
|
1616
|
+
)
|
|
1617
|
+
if state.get("blocked"):
|
|
1618
|
+
raise RequestError(
|
|
1619
|
+
str(exc) + "; evidence protocol recovery budget exhausted; return TEST_BLOCKED "
|
|
1620
|
+
"and do not initialize another package"
|
|
1621
|
+
) from exc
|
|
1622
|
+
raise
|
|
1543
1623
|
identity = {
|
|
1544
1624
|
"baseline_commit": baseline,
|
|
1545
1625
|
"working_diff_sha256": file_sha256(patch_path),
|
|
@@ -1567,7 +1647,7 @@ def run_finalize(project_root: Path, evidence_dir: Path, request: dict[str, Any]
|
|
|
1567
1647
|
)
|
|
1568
1648
|
if missing_records:
|
|
1569
1649
|
state = update_protocol_state(
|
|
1570
|
-
|
|
1650
|
+
project_root,
|
|
1571
1651
|
event="finalize-precondition-failed",
|
|
1572
1652
|
error="required evidence records are missing: " + ", ".join(missing_records),
|
|
1573
1653
|
increment="preflight_failures",
|
|
@@ -1628,13 +1708,14 @@ def run_finalize(project_root: Path, evidence_dir: Path, request: dict[str, Any]
|
|
|
1628
1708
|
evidence_dir, evidence_id=evidence_id, identity=identity,
|
|
1629
1709
|
target_hashes=target_hashes, change_class=change_class,
|
|
1630
1710
|
baseline_commit=baseline, final_verdict=verdict,
|
|
1711
|
+
attested=False,
|
|
1631
1712
|
)
|
|
1632
1713
|
write_json_atomic(evidence_dir / "manifest.json", manifest)
|
|
1633
1714
|
run_render_report(evidence_dir, "test-report.md")
|
|
1634
1715
|
write_json_atomic(evidence_dir / "lifecycle.json", {
|
|
1635
1716
|
**lifecycle,
|
|
1636
|
-
"status": "finalized",
|
|
1637
|
-
"finalized":
|
|
1717
|
+
"status": "payload-finalized",
|
|
1718
|
+
"finalized": False,
|
|
1638
1719
|
"immutable_identity": identity,
|
|
1639
1720
|
})
|
|
1640
1721
|
output = confined(evidence_dir, output_name)
|
|
@@ -1654,6 +1735,7 @@ def run_finalize(project_root: Path, evidence_dir: Path, request: dict[str, Any]
|
|
|
1654
1735
|
evidence_dir, evidence_id=evidence_id, identity=identity,
|
|
1655
1736
|
target_hashes=target_hashes, change_class=change_class,
|
|
1656
1737
|
baseline_commit=baseline, final_verdict=verdict,
|
|
1738
|
+
attested=False,
|
|
1657
1739
|
)
|
|
1658
1740
|
write_json_atomic(evidence_dir / "manifest.json", manifest)
|
|
1659
1741
|
if destination != evidence_dir:
|
|
@@ -1672,7 +1754,7 @@ def run_finalize(project_root: Path, evidence_dir: Path, request: dict[str, Any]
|
|
|
1672
1754
|
final_evidence_dir = destination if destination != evidence_dir else evidence_dir
|
|
1673
1755
|
write_handoff_pointer(final_evidence_dir, lifecycle.get("artifact_dir"), project_root)
|
|
1674
1756
|
update_protocol_state(
|
|
1675
|
-
|
|
1757
|
+
project_root,
|
|
1676
1758
|
event="finalized",
|
|
1677
1759
|
error="; ".join(verdict_record.get("blockers", [])) if verdict == "TEST_BLOCKED" else None,
|
|
1678
1760
|
)
|
|
@@ -1792,10 +1874,7 @@ def main() -> int:
|
|
|
1792
1874
|
try:
|
|
1793
1875
|
if not project_root.is_dir():
|
|
1794
1876
|
raise RequestError(f"project root does not exist: {project_root}")
|
|
1795
|
-
|
|
1796
|
-
evidence_dir.relative_to(project_root)
|
|
1797
|
-
except ValueError:
|
|
1798
|
-
raise RequestError(f"evidence directory must be under project root: {evidence_dir}")
|
|
1877
|
+
require_canonical_evidence_dir(project_root, evidence_dir)
|
|
1799
1878
|
if args.subcommand == "init":
|
|
1800
1879
|
output = run_init(project_root, evidence_dir, args.baseline, args.output)
|
|
1801
1880
|
print(f"Evidence initialized: {output}")
|
|
@@ -39,6 +39,7 @@ LAYER_ORDER = {
|
|
|
39
39
|
"regression-ring": 4,
|
|
40
40
|
}
|
|
41
41
|
FINAL_RECORDS = {"validation.json", "verdict.json", "test-report.md"}
|
|
42
|
+
ATTESTATION_EXCLUDED_RECORDS = FINAL_RECORDS | {"lifecycle.json"}
|
|
42
43
|
ATTESTATION_MUTABLE = {"validation.json"}
|
|
43
44
|
COMMON_RECORDS = {
|
|
44
45
|
"change-classification.json", "scope.json", "target-inventory.json",
|
|
@@ -646,8 +647,12 @@ def validate_manifest(
|
|
|
646
647
|
if status == "not_applicable":
|
|
647
648
|
structured_na(na, records_schema, f"stage {name} not_applicable", errors)
|
|
648
649
|
require(outputs == [], f"not-applicable stage must not claim outputs: {name}", errors)
|
|
650
|
+
elif status == "pending":
|
|
651
|
+
require(na is None, f"pending stage must have null not_applicable: {name}", errors)
|
|
649
652
|
elif status == "complete":
|
|
650
653
|
require(na is None, f"complete stage must have null not_applicable: {name}", errors)
|
|
654
|
+
else:
|
|
655
|
+
require(False, f"stage has unknown status: {name}", errors)
|
|
651
656
|
for output in outputs if isinstance(outputs, list) else []:
|
|
652
657
|
entry = entry_map.get(output)
|
|
653
658
|
require(entry is not None, f"stage output is not a hashed evidence file: {name}/{output}", errors)
|
|
@@ -961,6 +966,16 @@ def validate_requests_and_receipts(
|
|
|
961
966
|
if isinstance(request, dict):
|
|
962
967
|
for key in ("purpose", "command", "cwd", "layer", "test_ids", "external_targets"):
|
|
963
968
|
require(receipt.get(key) == request.get(key), f"{location}: receipt {key} does not match request", errors)
|
|
969
|
+
requested_timeout = request.get("timeout_seconds")
|
|
970
|
+
require(
|
|
971
|
+
isinstance(receipt.get("timeout_seconds"), (int, float))
|
|
972
|
+
and not isinstance(receipt.get("timeout_seconds"), bool)
|
|
973
|
+
and receipt["timeout_seconds"] > 0,
|
|
974
|
+
f"{location}: receipt lacks a bounded execution timeout",
|
|
975
|
+
errors,
|
|
976
|
+
)
|
|
977
|
+
if requested_timeout is not None:
|
|
978
|
+
require(receipt.get("timeout_seconds") == requested_timeout, f"{location}: receipt timeout does not match request", errors)
|
|
964
979
|
expected_environment = request.get("environment")
|
|
965
980
|
actual_environment = receipt.get("environment")
|
|
966
981
|
require(isinstance(actual_environment, dict) and isinstance(actual_environment.get("PATH"), str), f"{location}: receipt lacks complete PATH environment", errors)
|
|
@@ -1507,10 +1522,33 @@ def validate_verdict_and_attestation(
|
|
|
1507
1522
|
errors,
|
|
1508
1523
|
)
|
|
1509
1524
|
|
|
1525
|
+
if not verify_attestation:
|
|
1526
|
+
lifecycle = load_json(root / "lifecycle.json") if (root / "lifecycle.json").is_file() else {}
|
|
1527
|
+
if (root / "lifecycle.json").is_file():
|
|
1528
|
+
require(
|
|
1529
|
+
lifecycle.get("status") == "payload-finalized" and lifecycle.get("finalized") is False,
|
|
1530
|
+
"pre-attestation validation requires a payload-finalized mutable package",
|
|
1531
|
+
errors,
|
|
1532
|
+
)
|
|
1533
|
+
require(
|
|
1534
|
+
not validation,
|
|
1535
|
+
"pre-attestation package must not contain validator-owned validation data",
|
|
1536
|
+
errors,
|
|
1537
|
+
)
|
|
1538
|
+
|
|
1510
1539
|
if verify_attestation:
|
|
1511
1540
|
validate_definition(validation, "validation", records_schema, "validation", errors)
|
|
1541
|
+
lifecycle = load_json(root / "lifecycle.json") if (root / "lifecycle.json").is_file() else {}
|
|
1542
|
+
if (root / "lifecycle.json").is_file():
|
|
1543
|
+
require(
|
|
1544
|
+
lifecycle.get("status") in {"payload-finalized", "sealed"},
|
|
1545
|
+
"attested package has an invalid lifecycle state",
|
|
1546
|
+
errors,
|
|
1547
|
+
)
|
|
1548
|
+
if lifecycle.get("status") == "sealed":
|
|
1549
|
+
require(lifecycle.get("finalized") is True, "sealed package must be finalized", errors)
|
|
1512
1550
|
payload = sorted(
|
|
1513
|
-
({"path": path, "sha256": entry["sha256"]} for path, entry in entry_map.items() if path not in
|
|
1551
|
+
({"path": path, "sha256": entry["sha256"]} for path, entry in entry_map.items() if path not in ATTESTATION_EXCLUDED_RECORDS),
|
|
1514
1552
|
key=lambda item: item["path"],
|
|
1515
1553
|
)
|
|
1516
1554
|
replayed_ids = [
|
|
@@ -1749,6 +1787,16 @@ def main() -> int:
|
|
|
1749
1787
|
manifest = load_json(root / "manifest.json")
|
|
1750
1788
|
verdict_record = load_json(root / "verdict.json")
|
|
1751
1789
|
if args.attest:
|
|
1790
|
+
lifecycle_path = root / "lifecycle.json"
|
|
1791
|
+
lifecycle = load_json(lifecycle_path) if lifecycle_path.is_file() else {}
|
|
1792
|
+
if lifecycle_path.is_file() and (lifecycle.get("status") != "payload-finalized" or lifecycle.get("finalized") is not False):
|
|
1793
|
+
print("Test evidence validation: FAIL\n- attestation requires a payload-finalized mutable package")
|
|
1794
|
+
return 1
|
|
1795
|
+
manifest = load_json(root / "manifest.json")
|
|
1796
|
+
for stage in manifest.get("stages", []):
|
|
1797
|
+
if isinstance(stage, dict) and stage.get("name") == "EVIDENCE_VALIDATE":
|
|
1798
|
+
stage["status"] = "complete"
|
|
1799
|
+
atomic_write_json(root / "manifest.json", manifest)
|
|
1752
1800
|
executions = load_json(root / "executions.json")
|
|
1753
1801
|
replayed_ids = [
|
|
1754
1802
|
item.get("execution_id") for item in executions
|
|
@@ -1759,7 +1807,7 @@ def main() -> int:
|
|
|
1759
1807
|
payload = sorted(
|
|
1760
1808
|
(
|
|
1761
1809
|
{"path": entry["path"], "sha256": entry["sha256"]}
|
|
1762
|
-
for entry in manifest["files"] if entry["path"] not in
|
|
1810
|
+
for entry in manifest["files"] if entry["path"] not in ATTESTATION_EXCLUDED_RECORDS
|
|
1763
1811
|
),
|
|
1764
1812
|
key=lambda item: item["path"],
|
|
1765
1813
|
)
|
|
@@ -1782,6 +1830,12 @@ def main() -> int:
|
|
|
1782
1830
|
for error in errors:
|
|
1783
1831
|
print(f"- {error}")
|
|
1784
1832
|
return 1
|
|
1833
|
+
if lifecycle_path.is_file():
|
|
1834
|
+
atomic_write_json(root / "lifecycle.json", {
|
|
1835
|
+
**lifecycle,
|
|
1836
|
+
"status": "sealed",
|
|
1837
|
+
"finalized": True,
|
|
1838
|
+
})
|
|
1785
1839
|
|
|
1786
1840
|
print("Evidence validation: PASS")
|
|
1787
1841
|
print(f"Testing verdict: {manifest['final_verdict']}")
|