prizmkit 1.1.121 → 1.1.122
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bundled/VERSION.json +3 -3
- package/bundled/dev-pipeline/prizmkit_runtime/cli.py +1 -0
- package/bundled/dev-pipeline/tests/test_unified_cli.py +10 -1
- package/bundled/skills/_metadata.json +1 -1
- package/bundled/skills/prizmkit-test/SKILL.md +4 -3
- package/bundled/skills/prizmkit-test/references/evidence-protocol.md +10 -3
- package/bundled/skills/prizmkit-test/references/evidence-request-protocol.md +2 -2
- package/bundled/skills/prizmkit-test/scripts/build_test_evidence.py +26 -10
- package/bundled/skills/prizmkit-test/scripts/validate_test_evidence.py +21 -5
- package/package.json +1 -1
package/bundled/VERSION.json
CHANGED
|
@@ -30,10 +30,11 @@ if str(PIPELINE_ROOT) not in sys.path:
|
|
|
30
30
|
sys.path.insert(0, str(PIPELINE_ROOT))
|
|
31
31
|
|
|
32
32
|
|
|
33
|
-
def run_cli(*args):
|
|
33
|
+
def run_cli(*args, env=None):
|
|
34
34
|
return subprocess.run(
|
|
35
35
|
[PYTHON_310_PLUS, str(CLI_PATH), *args],
|
|
36
36
|
cwd=REPO_ROOT,
|
|
37
|
+
env=env,
|
|
37
38
|
text=True,
|
|
38
39
|
stdout=subprocess.PIPE,
|
|
39
40
|
stderr=subprocess.PIPE,
|
|
@@ -72,6 +73,14 @@ def test_cli_help_exposes_python_runtime_command_groups():
|
|
|
72
73
|
assert "Traceback" not in result.stderr
|
|
73
74
|
|
|
74
75
|
|
|
76
|
+
def test_cli_help_preserves_runtime_identity_at_narrow_terminal_width():
|
|
77
|
+
result = run_cli("--help", env={**os.environ, "COLUMNS": "24"})
|
|
78
|
+
|
|
79
|
+
assert result.returncode == 0
|
|
80
|
+
assert "canonical Python runtime CLI" in result.stdout
|
|
81
|
+
assert "without shell wrappers" in result.stdout
|
|
82
|
+
|
|
83
|
+
|
|
75
84
|
def test_module_entrypoint_help_exposes_same_cli():
|
|
76
85
|
env = {**os.environ, "PYTHONPATH": str(PIPELINE_ROOT)}
|
|
77
86
|
result = subprocess.run(
|
|
@@ -145,8 +145,9 @@ Differentially prove each added/changed necessary behavior test:
|
|
|
145
145
|
|
|
146
146
|
- Prefer baseline failure plus current success in an isolated/uncontaminated environment.
|
|
147
147
|
- If baseline is inapplicable, use a minimal controlled mutation tied to the same risk.
|
|
148
|
-
- Bind proof to runner receipts, baseline commit,
|
|
149
|
-
-
|
|
148
|
+
- Bind proof to runner receipts, baseline commit, a stable source-snapshot hash, mutation apply/restore hashes, and complete cleanup. The source snapshot excludes runtime-managed `.prizmkit/state/`, evidence output under `.prizmkit/test/evidence/`, `.claude/worktrees/`, Git metadata, and Python bytecode caches because those paths change while the tested code remains identical.
|
|
149
|
+
- Keep source identity separate from evidence integrity: the stable snapshot detects behavior-bearing project drift, while `manifest.json` hashes every evidence file and detects report/output tampering.
|
|
150
|
+
- Completely remove mutations and verify the stable source snapshot is unchanged. Never rebind completed proof to the mutable live project tree.
|
|
150
151
|
- Classify each proof as `PROVEN`, structured `NOT_APPLICABLE`, or `UNPROVEN`.
|
|
151
152
|
|
|
152
153
|
Necessary `UNPROVEN` behavior, required flakiness, unavailable infrastructure after bounded recovery, failed cleanup, unreliable execution, or budget truncation yields `TEST_BLOCKED`.
|
|
@@ -170,7 +171,7 @@ Create `.prizmkit/test/evidence/<evidence-id>/` from the package template with:
|
|
|
170
171
|
- `contracts/` snapshots
|
|
171
172
|
- derived `test-report.md`
|
|
172
173
|
|
|
173
|
-
Hash baseline, diff, source, tests
|
|
174
|
+
Hash baseline, diff, the stable source snapshot, inventoried source/tests/contracts/lockfiles, environment, plan, and every evidence file. Runtime state, evidence output, linked Agent worktrees, Git metadata, and bytecode caches are outside source identity; evidence files remain independently content-addressed through `manifest.json`. Set `environment_claim=mocked-code-level-only` and `compatibility=legacy-test-report-interface-not-supported`.
|
|
174
175
|
|
|
175
176
|
### 9. EVIDENCE_VALIDATE
|
|
176
177
|
|
|
@@ -10,7 +10,14 @@ Write packages under `.prizmkit/test/evidence/<evidence-id>/`. Compute `evidence
|
|
|
10
10
|
- `working_diff_sha256`
|
|
11
11
|
- `scope_sha256`
|
|
12
12
|
|
|
13
|
-
Store those values as `evidence_id_inputs` in `manifest.json`. Record separate aggregate hashes for source, tests, contracts, lockfiles, full environment values, and plan inputs. Hash every evidence file except `manifest.json` in the manifest; do not create a self-referential manifest hash.
|
|
13
|
+
Store those values as `evidence_id_inputs` in `manifest.json`. Record separate aggregate hashes for inventoried source, tests, contracts, lockfiles, full environment values, and plan inputs. Hash every evidence file except `manifest.json` in the manifest; do not create a self-referential manifest hash.
|
|
14
|
+
|
|
15
|
+
Use two separate integrity domains:
|
|
16
|
+
|
|
17
|
+
- **Stable source snapshot:** hash behavior-bearing project files while excluding runtime-managed `.prizmkit/state/`, evidence output under `.prizmkit/test/evidence/`, linked `.claude/worktrees/`, Git metadata, and Python bytecode caches. Builder and validator use the same deterministic exclusion policy, and isolated current execution is copied from that same snapshot scope.
|
|
18
|
+
- **Evidence package:** hash each evidence record through `manifest.json`; any report, receipt, raw output, request, or proof mutation remains detectable even though evidence output is outside source identity.
|
|
19
|
+
|
|
20
|
+
Runtime-only churn must not invalidate a source proof. Drift in an inventoried source, test, contract, or lockfile must still fail live inventory validation. Never replace a completed proof's stable snapshot identity with a later hash of the mutable parent workspace.
|
|
14
21
|
|
|
15
22
|
Hashes and validator receipts provide byte-level binding and drift detection, not cryptographic non-repudiation. A caller with the same OS permissions can replace the runner, requests, project tree, or package before validation. Preserve runner/request hashes and use replay to improve auditability without claiming resistance to a malicious same-privilege producer.
|
|
16
23
|
|
|
@@ -126,7 +133,7 @@ For each added or changed necessary behavior test, prove the test:
|
|
|
126
133
|
2. If baseline execution is inapplicable (for example, the public behavior did not exist), apply a minimal controlled mutation tied to the same matrix risk, observe failure, restore it completely, and observe current success.
|
|
127
134
|
3. Record `PROVEN`, justified `NOT_APPLICABLE`, or `UNPROVEN`.
|
|
128
135
|
|
|
129
|
-
Record baseline/current/mutation runner receipt IDs, request hash, baseline commit,
|
|
136
|
+
Record baseline/current/mutation runner receipt IDs, request hash, baseline commit, stable source-snapshot and isolation hashes, mutation apply/restore hashes, expected failure signal match, and source-snapshot cleanup result. Necessary `UNPROVEN` behavior prevents `TEST_PASS`.
|
|
130
137
|
|
|
131
138
|
### EVIDENCE_PACKAGE
|
|
132
139
|
|
|
@@ -157,7 +164,7 @@ python3 ${SKILL_DIR}/scripts/validate_test_evidence.py \
|
|
|
157
164
|
|
|
158
165
|
When feasible, replay recorded requests through `build_test_evidence.py execute --replay-receipt ...`. Replay performs a new real execution and emits a new linked receipt; it does not pretend the attestation establishes what historically ran.
|
|
159
166
|
|
|
160
|
-
The validator checks all shipped JSON schemas, evidence-directory identity, live target-project inventories and aggregate hashes, changed-file/module-root/exclusion/Regression Ring cross-links, patch/diff binding, stage output ownership and predecessor hashes, generated-test snapshot existence, planned test/inventory/execution linkage, module/matrix completeness, successful selected runner receipts with request/raw-output binding, proof-linked isolated failures/current successes, controlled-mutation apply/restoration hashes, cleanup, structured N/A and obvious signal conflicts, production/unknown external-target safety, strict verdict semantics, and honest mocked-versus-real claims. `TEST_PASS` requires deterministic validator success.
|
|
167
|
+
The validator checks all shipped JSON schemas, evidence-directory identity, live target-project inventories and aggregate hashes, stable source-snapshot identity with runtime paths excluded, changed-file/module-root/exclusion/Regression Ring cross-links, patch/diff binding, stage output ownership and predecessor hashes, generated-test snapshot existence, planned test/inventory/execution linkage, module/matrix completeness, successful selected runner receipts with request/raw-output binding, proof-linked isolated failures/current successes, controlled-mutation apply/restoration hashes, cleanup, structured N/A and obvious signal conflicts, production/unknown external-target safety, strict verdict semantics, and honest mocked-versus-real claims. `TEST_PASS` requires deterministic validator success.
|
|
161
168
|
|
|
162
169
|
## Resume and Invalidation
|
|
163
170
|
|
|
@@ -41,11 +41,11 @@ Record:
|
|
|
41
41
|
|
|
42
42
|
- behavior ID and model-selected `baseline` or `controlled-mutation` method;
|
|
43
43
|
- the nested execution request;
|
|
44
|
-
- actual baseline commit and
|
|
44
|
+
- actual baseline commit and the stable source-snapshot hash produced with the builder's runtime exclusions;
|
|
45
45
|
- mutation patch path when applicable;
|
|
46
46
|
- concrete expected failure signals.
|
|
47
47
|
|
|
48
|
-
The builder creates isolated copies, executes the failing and current sides, records receipts, binds the differential request hash, and verifies that
|
|
48
|
+
The builder creates isolated copies from the same stable source-snapshot scope, executes the failing and current sides, records receipts, binds the differential request hash, and verifies that behavior-bearing project content is unchanged. Runtime state, evidence output, linked Agent worktrees, Git metadata, and Python bytecode caches do not participate in source identity. A proof is `PROVEN` only when the intended failure signal appears, the failing side exits nonzero, current exits zero, and cleanup/snapshot checks pass.
|
|
49
49
|
|
|
50
50
|
## Structured N/A
|
|
51
51
|
|
|
@@ -36,6 +36,12 @@ STAGES = [
|
|
|
36
36
|
]
|
|
37
37
|
CATEGORIES = ("source", "tests", "contracts", "lockfiles")
|
|
38
38
|
BLOCKED_EXTERNAL_CLASSES = {"production", "unknown"}
|
|
39
|
+
SNAPSHOT_VOLATILE_ROOTS = (
|
|
40
|
+
".prizmkit/state",
|
|
41
|
+
".prizmkit/test/evidence",
|
|
42
|
+
".claude/worktrees",
|
|
43
|
+
)
|
|
44
|
+
SNAPSHOT_VOLATILE_NAMES = {".git", "__pycache__"}
|
|
39
45
|
|
|
40
46
|
|
|
41
47
|
class RequestError(Exception):
|
|
@@ -176,14 +182,23 @@ def validate_execution_request(request: Any) -> dict[str, Any]:
|
|
|
176
182
|
return request
|
|
177
183
|
|
|
178
184
|
|
|
185
|
+
def is_volatile_snapshot_path(root: Path, candidate: Path, excluded: list[Path] | None = None) -> bool:
|
|
186
|
+
try:
|
|
187
|
+
relative = candidate.relative_to(root)
|
|
188
|
+
except ValueError:
|
|
189
|
+
return True
|
|
190
|
+
if any(part in SNAPSHOT_VOLATILE_NAMES for part in relative.parts):
|
|
191
|
+
return True
|
|
192
|
+
if any(relative == Path(prefix) or Path(prefix) in relative.parents for prefix in SNAPSHOT_VOLATILE_ROOTS):
|
|
193
|
+
return True
|
|
194
|
+
resolved = candidate.resolve()
|
|
195
|
+
return any(resolved == item.resolve() or item.resolve() in resolved.parents for item in (excluded or []))
|
|
196
|
+
|
|
197
|
+
|
|
179
198
|
def tree_sha256(root: Path, excluded: list[Path] | None = None) -> str:
|
|
180
|
-
excluded_resolved = [path.resolve() for path in (excluded or [])]
|
|
181
199
|
entries: list[dict[str, str]] = []
|
|
182
200
|
for candidate in sorted(root.rglob("*")):
|
|
183
|
-
if not candidate.is_file() or
|
|
184
|
-
continue
|
|
185
|
-
resolved = candidate.resolve()
|
|
186
|
-
if any(resolved == item or item in resolved.parents for item in excluded_resolved):
|
|
201
|
+
if not candidate.is_file() or is_volatile_snapshot_path(root, candidate, excluded):
|
|
187
202
|
continue
|
|
188
203
|
entries.append({"path": candidate.relative_to(root).as_posix(), "sha256": file_sha256(candidate)})
|
|
189
204
|
return canonical_sha256(entries)
|
|
@@ -454,13 +469,14 @@ def execute_request(
|
|
|
454
469
|
|
|
455
470
|
|
|
456
471
|
def copy_project_tree(project_root: Path, destination: Path, evidence_dir: Path) -> None:
|
|
457
|
-
ignored_path = evidence_dir.resolve().relative_to(project_root.resolve())
|
|
458
|
-
|
|
459
472
|
def ignore(directory: str, names: list[str]) -> set[str]:
|
|
460
|
-
ignored = {".git", "__pycache__"} & set(names)
|
|
461
473
|
directory_relative = Path(directory).resolve().relative_to(project_root.resolve())
|
|
462
|
-
|
|
463
|
-
|
|
474
|
+
ignored: set[str] = set()
|
|
475
|
+
for name in names:
|
|
476
|
+
relative = directory_relative / name
|
|
477
|
+
candidate = project_root / relative
|
|
478
|
+
if is_volatile_snapshot_path(project_root, candidate, [evidence_dir]):
|
|
479
|
+
ignored.add(name)
|
|
464
480
|
return ignored
|
|
465
481
|
|
|
466
482
|
shutil.copytree(project_root, destination, ignore=ignore)
|
|
@@ -51,6 +51,12 @@ LIGHTWEIGHT_RECORDS = {"lightweight-verification.json"}
|
|
|
51
51
|
SHA256_RE = re.compile(r"^[a-f0-9]{64}$")
|
|
52
52
|
UUID_RE = re.compile(r"^[a-f0-9-]{36}$")
|
|
53
53
|
BLOCKED_EXTERNAL_CLASSES = {"production", "unknown"}
|
|
54
|
+
SNAPSHOT_VOLATILE_ROOTS = (
|
|
55
|
+
".prizmkit/state",
|
|
56
|
+
".prizmkit/test/evidence",
|
|
57
|
+
".claude/worktrees",
|
|
58
|
+
)
|
|
59
|
+
SNAPSHOT_VOLATILE_NAMES = {".git", "__pycache__"}
|
|
54
60
|
RISK_CONFLICT_PATTERNS = {
|
|
55
61
|
"permission": re.compile(r"auth|tenant|role|permission|access|acl|rbac", re.I),
|
|
56
62
|
"concurrency": re.compile(r"lock|shared[ _-]?state|worker|queue|concurr|parallel|race", re.I),
|
|
@@ -1019,14 +1025,24 @@ def validate_matrix_risks(
|
|
|
1019
1025
|
errors.append(f"TEST_PASS has unresolved behavior risk: {behavior_id}/{risk_name}")
|
|
1020
1026
|
|
|
1021
1027
|
|
|
1022
|
-
def
|
|
1028
|
+
def is_volatile_snapshot_path(root: Path, candidate: Path, evidence_root: Path) -> bool:
|
|
1029
|
+
try:
|
|
1030
|
+
relative = candidate.relative_to(root)
|
|
1031
|
+
except ValueError:
|
|
1032
|
+
return True
|
|
1033
|
+
if any(part in SNAPSHOT_VOLATILE_NAMES for part in relative.parts):
|
|
1034
|
+
return True
|
|
1035
|
+
if any(relative == Path(prefix) or Path(prefix) in relative.parents for prefix in SNAPSHOT_VOLATILE_ROOTS):
|
|
1036
|
+
return True
|
|
1037
|
+
resolved = candidate.resolve()
|
|
1023
1038
|
excluded = evidence_root.resolve()
|
|
1039
|
+
return resolved == excluded or excluded in resolved.parents
|
|
1040
|
+
|
|
1041
|
+
|
|
1042
|
+
def tree_sha256(root: Path, evidence_root: Path) -> str:
|
|
1024
1043
|
entries: list[dict[str, str]] = []
|
|
1025
1044
|
for candidate in sorted(root.rglob("*")):
|
|
1026
|
-
if not candidate.is_file() or
|
|
1027
|
-
continue
|
|
1028
|
-
resolved = candidate.resolve()
|
|
1029
|
-
if resolved == excluded or excluded in resolved.parents:
|
|
1045
|
+
if not candidate.is_file() or is_volatile_snapshot_path(root, candidate, evidence_root):
|
|
1030
1046
|
continue
|
|
1031
1047
|
entries.append({"path": candidate.relative_to(root).as_posix(), "sha256": file_sha256(candidate)})
|
|
1032
1048
|
return canonical_sha256(entries)
|