prizmkit 1.1.86 → 1.1.87
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bundled/VERSION.json +3 -3
- package/bundled/dev-pipeline/scripts/generate-bootstrap-prompt.py +462 -4
- package/bundled/dev-pipeline/templates/bootstrap-prompt.md +1 -1
- package/bundled/dev-pipeline/templates/bootstrap-tier1.md +14 -0
- package/bundled/dev-pipeline/templates/bootstrap-tier2.md +18 -3
- package/bundled/dev-pipeline/templates/bootstrap-tier3.md +18 -3
- package/bundled/dev-pipeline/templates/sections/phase-commit-full.md +2 -2
- package/bundled/dev-pipeline/templates/sections/phase-commit.md +2 -2
- package/bundled/dev-pipeline/templates/sections/phase-prizmkit-test.md +260 -0
- package/bundled/dev-pipeline/templates/sections/phase-review-agent.md +5 -1
- package/bundled/dev-pipeline/templates/sections/phase-review-full.md +5 -1
- package/bundled/dev-pipeline/tests/test_generate_bootstrap_prompt.py +429 -0
- package/bundled/dev-pipeline-windows/scripts/generate-bootstrap-prompt.py +462 -4
- package/bundled/dev-pipeline-windows/templates/bootstrap-prompt.md +1 -1
- package/bundled/dev-pipeline-windows/templates/bootstrap-tier1.md +14 -0
- package/bundled/dev-pipeline-windows/templates/bootstrap-tier2.md +18 -3
- package/bundled/dev-pipeline-windows/templates/bootstrap-tier3.md +18 -3
- package/bundled/dev-pipeline-windows/templates/sections/phase-commit-full.md +2 -2
- package/bundled/dev-pipeline-windows/templates/sections/phase-commit.md +2 -2
- package/bundled/dev-pipeline-windows/templates/sections/phase-prizmkit-test.md +186 -0
- package/bundled/dev-pipeline-windows/templates/sections/phase-review-agent.md +5 -1
- package/bundled/dev-pipeline-windows/templates/sections/phase-review-full.md +5 -1
- package/bundled/skills/_metadata.json +1 -1
- package/bundled/skills/prizmkit-code-review/SKILL.md +7 -2
- package/bundled/skills/prizmkit-code-review/references/reviewer-agent-prompt.md +6 -2
- package/bundled/skills/prizmkit-test/SKILL.md +106 -14
- package/bundled/skills/prizmkit-test/references/boundary-coverage-protocol.md +220 -0
- package/bundled/skills/prizmkit-test/references/examples.md +82 -9
- package/bundled/skills/prizmkit-test/references/test-generation-steps.md +68 -12
- package/bundled/skills/prizmkit-test/references/test-report-template.md +73 -9
- package/bundled/skills/prizmkit-test/scripts/validate_boundary_report.py +347 -0
- package/bundled/skills-windows/prizmkit-code-review/SKILL.md +7 -2
- package/bundled/skills-windows/prizmkit-code-review/references/reviewer-agent-prompt.md +6 -2
- package/bundled/skills-windows/prizmkit-test/SKILL.md +106 -14
- package/bundled/skills-windows/prizmkit-test/references/boundary-coverage-protocol.md +220 -0
- package/bundled/skills-windows/prizmkit-test/references/examples.md +82 -9
- package/bundled/skills-windows/prizmkit-test/references/service-boundary-test-catalog.md +4 -4
- package/bundled/skills-windows/prizmkit-test/references/test-generation-steps.md +68 -12
- package/bundled/skills-windows/prizmkit-test/references/test-report-template.md +73 -9
- package/bundled/skills-windows/prizmkit-test/scripts/validate_boundary_report.py +347 -0
- package/package.json +1 -1
|
@@ -0,0 +1,347 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""
|
|
3
|
+
Validate a prizmkit-test report's boundary coverage gate.
|
|
4
|
+
|
|
5
|
+
Usage:
|
|
6
|
+
python3 scripts/validate_boundary_report.py --report .prizmkit/test/.../test-report.md
|
|
7
|
+
python3 scripts/validate_boundary_report.py --report .prizmkit/test/.../test-report.md --openapi openapi.yaml
|
|
8
|
+
|
|
9
|
+
The script intentionally performs deterministic structural checks only. It cannot prove that every
|
|
10
|
+
business boundary is semantically correct, but it prevents reports from silently claiming completion
|
|
11
|
+
when interfaces are missing from the Boundary Matrix or remain happy-path-only / boundary-missing.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
import argparse
|
|
17
|
+
import re
|
|
18
|
+
import sys
|
|
19
|
+
from pathlib import Path
|
|
20
|
+
from typing import Dict, List, Set, Tuple
|
|
21
|
+
|
|
22
|
+
ALLOWED_FINAL_STATUSES = {
|
|
23
|
+
"boundary-covered",
|
|
24
|
+
"generated-needs-review",
|
|
25
|
+
"skipped-with-reason",
|
|
26
|
+
}
|
|
27
|
+
INCOMPLETE_MARKERS = {
|
|
28
|
+
"boundary-missing",
|
|
29
|
+
"happy-path-only",
|
|
30
|
+
"unresolved",
|
|
31
|
+
}
|
|
32
|
+
REQUIRED_SECTIONS = [
|
|
33
|
+
"## Boundary Matrix",
|
|
34
|
+
"## Boundary Completion Gate",
|
|
35
|
+
"## Final Status",
|
|
36
|
+
]
|
|
37
|
+
REQUIRED_BOUNDARY_COLUMNS = [
|
|
38
|
+
"Module",
|
|
39
|
+
"Interface",
|
|
40
|
+
"Service Type",
|
|
41
|
+
"Happy Path",
|
|
42
|
+
"Request Validation",
|
|
43
|
+
"Auth / Permission / Ownership",
|
|
44
|
+
"Domain Invariants",
|
|
45
|
+
"Collection / Pagination",
|
|
46
|
+
"Date / Time",
|
|
47
|
+
"State Transitions",
|
|
48
|
+
"Dependency Failure",
|
|
49
|
+
"Response Contract",
|
|
50
|
+
"N/A Reasons",
|
|
51
|
+
"Final Status",
|
|
52
|
+
]
|
|
53
|
+
BOUNDARY_CATEGORY_COLUMNS = [
|
|
54
|
+
"Happy Path",
|
|
55
|
+
"Request Validation",
|
|
56
|
+
"Auth / Permission / Ownership",
|
|
57
|
+
"Domain Invariants",
|
|
58
|
+
"Collection / Pagination",
|
|
59
|
+
"Date / Time",
|
|
60
|
+
"State Transitions",
|
|
61
|
+
"Dependency Failure",
|
|
62
|
+
"Response Contract",
|
|
63
|
+
]
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def read_text(path: Path) -> str:
|
|
67
|
+
try:
|
|
68
|
+
return path.read_text(encoding="utf-8")
|
|
69
|
+
except FileNotFoundError:
|
|
70
|
+
raise SystemExit(f"FAIL: file not found: {path}")
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
HTTP_METHODS = {"GET", "POST", "PUT", "PATCH", "DELETE", "HEAD", "OPTIONS", "TRACE"}
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def extract_openapi_paths(openapi_text: str) -> List[str]:
|
|
77
|
+
"""Extract OpenAPI path keys without requiring PyYAML.
|
|
78
|
+
|
|
79
|
+
Supports common YAML forms inside the `paths:` mapping:
|
|
80
|
+
- /users:
|
|
81
|
+
- "/users/{id}":
|
|
82
|
+
- '/users/{id}':
|
|
83
|
+
"""
|
|
84
|
+
return [path for _method, path in extract_openapi_operations(openapi_text)]
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def extract_openapi_operations(openapi_text: str) -> List[Tuple[str, str]]:
|
|
88
|
+
"""Extract OpenAPI method+path operations without requiring PyYAML."""
|
|
89
|
+
operations: List[Tuple[str, str]] = []
|
|
90
|
+
in_paths = False
|
|
91
|
+
paths_indent = 0
|
|
92
|
+
current_path = ""
|
|
93
|
+
current_path_indent = 0
|
|
94
|
+
path_key_re = re.compile(
|
|
95
|
+
r"^\s*(?:(['\"])(/.*?)\1|(/[^:\s]+))\s*:\s*(?:#.*)?$"
|
|
96
|
+
)
|
|
97
|
+
method_re = re.compile(r"^\s*([A-Za-z]+)\s*:\s*(?:#.*)?$")
|
|
98
|
+
|
|
99
|
+
for raw_line in openapi_text.splitlines():
|
|
100
|
+
stripped = raw_line.strip()
|
|
101
|
+
if not stripped or stripped.startswith("#"):
|
|
102
|
+
continue
|
|
103
|
+
|
|
104
|
+
indent = len(raw_line) - len(raw_line.lstrip())
|
|
105
|
+
if not in_paths:
|
|
106
|
+
if re.match(r"^\s*paths\s*:\s*(?:#.*)?$", raw_line):
|
|
107
|
+
in_paths = True
|
|
108
|
+
paths_indent = indent
|
|
109
|
+
continue
|
|
110
|
+
|
|
111
|
+
if indent <= paths_indent:
|
|
112
|
+
break
|
|
113
|
+
|
|
114
|
+
path_match = path_key_re.match(raw_line)
|
|
115
|
+
if path_match:
|
|
116
|
+
current_path = path_match.group(2) or path_match.group(3) or ""
|
|
117
|
+
current_path_indent = indent
|
|
118
|
+
continue
|
|
119
|
+
|
|
120
|
+
method_match = method_re.match(raw_line)
|
|
121
|
+
if current_path and method_match and indent > current_path_indent:
|
|
122
|
+
method = method_match.group(1).upper()
|
|
123
|
+
if method in HTTP_METHODS:
|
|
124
|
+
operations.append((method, current_path))
|
|
125
|
+
|
|
126
|
+
return operations
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def extract_section(text: str, heading: str) -> str:
|
|
130
|
+
start = text.find(heading)
|
|
131
|
+
if start == -1:
|
|
132
|
+
return ""
|
|
133
|
+
rest = text[start + len(heading) :]
|
|
134
|
+
match = re.search(r"\n## ", rest)
|
|
135
|
+
if not match:
|
|
136
|
+
return rest
|
|
137
|
+
return rest[: match.start()]
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def split_markdown_row(line: str) -> List[str]:
|
|
141
|
+
return [cell.strip() for cell in line.strip().strip("|").split("|")]
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def parse_boundary_matrix(text: str) -> Tuple[List[str], List[Dict[str, str]]]:
|
|
145
|
+
section = extract_section(text, "## Boundary Matrix")
|
|
146
|
+
if not section:
|
|
147
|
+
return [], []
|
|
148
|
+
|
|
149
|
+
lines = [line.strip() for line in section.splitlines() if line.strip().startswith("|")]
|
|
150
|
+
header_index = -1
|
|
151
|
+
for idx, line in enumerate(lines):
|
|
152
|
+
cells = split_markdown_row(line)
|
|
153
|
+
if "Interface" in cells and "Final Status" in cells:
|
|
154
|
+
header_index = idx
|
|
155
|
+
break
|
|
156
|
+
if header_index == -1:
|
|
157
|
+
return [], []
|
|
158
|
+
|
|
159
|
+
headers = split_markdown_row(lines[header_index])
|
|
160
|
+
rows: List[Dict[str, str]] = []
|
|
161
|
+
for line in lines[header_index + 1 :]:
|
|
162
|
+
cells = split_markdown_row(line)
|
|
163
|
+
if all(re.fullmatch(r":?-{3,}:?", cell) for cell in cells):
|
|
164
|
+
continue
|
|
165
|
+
if len(cells) < len(headers):
|
|
166
|
+
cells.extend([""] * (len(headers) - len(cells)))
|
|
167
|
+
row = {header: cells[i] if i < len(cells) else "" for i, header in enumerate(headers)}
|
|
168
|
+
# Ignore example placeholder rows that were accidentally left in templates.
|
|
169
|
+
if row.get("Interface", "").strip(".` ") in {"...", ""}:
|
|
170
|
+
continue
|
|
171
|
+
rows.append(row)
|
|
172
|
+
return headers, rows
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
def has_inline_reason(value: str) -> bool:
|
|
176
|
+
normalized = value.strip().lower()
|
|
177
|
+
if not normalized:
|
|
178
|
+
return False
|
|
179
|
+
if "—" in value or " - " in value or ":" in value:
|
|
180
|
+
return True
|
|
181
|
+
# Accept phrases longer than a bare status marker.
|
|
182
|
+
bare = {"n/a", "na", "skipped", "skip"}
|
|
183
|
+
return normalized not in bare and (normalized.startswith("n/a ") or normalized.startswith("skipped "))
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
def extract_matrix_operations(interface_values: List[str]) -> Tuple[Set[Tuple[str, str]], Set[str]]:
|
|
187
|
+
"""Extract method+path and path-only tokens from Boundary Matrix Interface cells."""
|
|
188
|
+
operations: Set[Tuple[str, str]] = set()
|
|
189
|
+
path_only: Set[str] = set()
|
|
190
|
+
token_re = re.compile(
|
|
191
|
+
r"(?:(\b(?:GET|POST|PUT|PATCH|DELETE|HEAD|OPTIONS|TRACE)\b)\s+)?(/[^\s,`|)]+)",
|
|
192
|
+
flags=re.IGNORECASE,
|
|
193
|
+
)
|
|
194
|
+
for value in interface_values:
|
|
195
|
+
for match in token_re.finditer(value):
|
|
196
|
+
method = match.group(1).upper() if match.group(1) else ""
|
|
197
|
+
path = normalize_path(match.group(2))
|
|
198
|
+
if not path:
|
|
199
|
+
continue
|
|
200
|
+
if method:
|
|
201
|
+
operations.add((method, path))
|
|
202
|
+
else:
|
|
203
|
+
path_only.add(path)
|
|
204
|
+
return operations, path_only
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
def normalize_path(path: str) -> str:
|
|
208
|
+
normalized = path.strip().strip("'\"`")
|
|
209
|
+
if len(normalized) > 1:
|
|
210
|
+
normalized = normalized.rstrip("/")
|
|
211
|
+
return normalized
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
def validate_report(report_text: str, openapi_paths: List[str] | None,
|
|
215
|
+
openapi_operations: List[Tuple[str, str]] | None = None) -> List[str]:
|
|
216
|
+
errors: List[str] = []
|
|
217
|
+
|
|
218
|
+
for section in REQUIRED_SECTIONS:
|
|
219
|
+
if section not in report_text:
|
|
220
|
+
errors.append(f"missing required section: {section}")
|
|
221
|
+
|
|
222
|
+
headers, rows = parse_boundary_matrix(report_text)
|
|
223
|
+
if not rows:
|
|
224
|
+
errors.append("Boundary Matrix table is missing or empty")
|
|
225
|
+
return errors
|
|
226
|
+
|
|
227
|
+
for column in REQUIRED_BOUNDARY_COLUMNS:
|
|
228
|
+
if column not in headers:
|
|
229
|
+
errors.append(f"Boundary Matrix missing required column: {column}")
|
|
230
|
+
|
|
231
|
+
final_statuses = []
|
|
232
|
+
interface_values = []
|
|
233
|
+
for idx, row in enumerate(rows, start=1):
|
|
234
|
+
interface = row.get("Interface", "").strip()
|
|
235
|
+
interface_values.append(interface)
|
|
236
|
+
|
|
237
|
+
final_status = row.get("Final Status", "").strip().lower()
|
|
238
|
+
final_statuses.append(final_status)
|
|
239
|
+
if not final_status:
|
|
240
|
+
errors.append(f"row {idx} ({interface}) missing Final Status")
|
|
241
|
+
elif final_status not in ALLOWED_FINAL_STATUSES:
|
|
242
|
+
errors.append(
|
|
243
|
+
f"row {idx} ({interface}) has invalid/incomplete Final Status: {row.get('Final Status', '')}"
|
|
244
|
+
)
|
|
245
|
+
|
|
246
|
+
for column in BOUNDARY_CATEGORY_COLUMNS:
|
|
247
|
+
if column in headers and not row.get(column, "").strip():
|
|
248
|
+
errors.append(f"row {idx} ({interface}) missing boundary category value: {column}")
|
|
249
|
+
|
|
250
|
+
for header, value in row.items():
|
|
251
|
+
normalized = value.strip().lower()
|
|
252
|
+
if any(marker in normalized for marker in INCOMPLETE_MARKERS):
|
|
253
|
+
errors.append(f"row {idx} ({interface}) has incomplete marker in {header}: {value}")
|
|
254
|
+
|
|
255
|
+
n_a_reasons = row.get("N/A Reasons", "").strip()
|
|
256
|
+
for header, value in row.items():
|
|
257
|
+
if header in {"Module", "Interface", "Service Type", "Final Status", "N/A Reasons"}:
|
|
258
|
+
continue
|
|
259
|
+
normalized = value.strip().lower()
|
|
260
|
+
if normalized in {"n/a", "na", "skipped", "skip"} and not n_a_reasons:
|
|
261
|
+
errors.append(f"row {idx} ({interface}) marks {header} as {value} without N/A reason")
|
|
262
|
+
if ("skipped" in normalized or normalized.startswith("n/a")) and not (n_a_reasons or has_inline_reason(value)):
|
|
263
|
+
errors.append(f"row {idx} ({interface}) marks {header} as {value} without concrete reason")
|
|
264
|
+
|
|
265
|
+
if openapi_operations is not None:
|
|
266
|
+
matrix_operations, matrix_path_only = extract_matrix_operations(interface_values)
|
|
267
|
+
for method, path in openapi_operations:
|
|
268
|
+
normalized_path = normalize_path(path)
|
|
269
|
+
if (method, normalized_path) not in matrix_operations:
|
|
270
|
+
if normalized_path in matrix_path_only:
|
|
271
|
+
errors.append(
|
|
272
|
+
"OpenAPI operation missing from Boundary Matrix: "
|
|
273
|
+
f"{method} {path} (path-only row is ambiguous)"
|
|
274
|
+
)
|
|
275
|
+
else:
|
|
276
|
+
errors.append(
|
|
277
|
+
f"OpenAPI operation missing from Boundary Matrix: {method} {path}"
|
|
278
|
+
)
|
|
279
|
+
elif openapi_paths is not None:
|
|
280
|
+
_matrix_operations, matrix_path_only = extract_matrix_operations(interface_values)
|
|
281
|
+
missing_paths = [path for path in openapi_paths
|
|
282
|
+
if normalize_path(path) not in matrix_path_only]
|
|
283
|
+
for path in missing_paths:
|
|
284
|
+
errors.append(f"OpenAPI path missing from Boundary Matrix: {path}")
|
|
285
|
+
|
|
286
|
+
gate_section = extract_section(report_text, "## Boundary Completion Gate")
|
|
287
|
+
if gate_section:
|
|
288
|
+
for label in ["Interfaces total", "Boundary-covered", "Boundary-missing", "Happy-path-only", "Completion gate passed"]:
|
|
289
|
+
if label not in gate_section:
|
|
290
|
+
errors.append(f"Boundary Completion Gate missing metric: {label}")
|
|
291
|
+
for label in ["Boundary-missing", "Happy-path-only"]:
|
|
292
|
+
match = re.search(rf"{re.escape(label)}:\s*(\d+)", gate_section)
|
|
293
|
+
if match and int(match.group(1)) > 0:
|
|
294
|
+
errors.append(f"Boundary Completion Gate reports {label}: {match.group(1)}")
|
|
295
|
+
if re.search(r"Completion gate passed:\s*no", gate_section, flags=re.IGNORECASE):
|
|
296
|
+
errors.append("Boundary Completion Gate reports failure")
|
|
297
|
+
|
|
298
|
+
final_section = extract_section(report_text, "## Final Status")
|
|
299
|
+
if final_section:
|
|
300
|
+
if re.search(r"Ready for commit:\s*yes", final_section, flags=re.IGNORECASE):
|
|
301
|
+
if any(status != "boundary-covered" for status in final_statuses):
|
|
302
|
+
errors.append("Final Status says ready for commit but not every interface is boundary-covered")
|
|
303
|
+
review_match = re.search(r"Needs human review:\s*(.+)", final_section, flags=re.IGNORECASE)
|
|
304
|
+
if review_match and review_match.group(1).strip().lower() not in {"none", "no", "n/a", ""}:
|
|
305
|
+
errors.append("Final Status says ready for commit while human review is still needed")
|
|
306
|
+
|
|
307
|
+
return errors
|
|
308
|
+
|
|
309
|
+
|
|
310
|
+
def main() -> int:
|
|
311
|
+
parser = argparse.ArgumentParser(description="Validate prizmkit-test boundary coverage report")
|
|
312
|
+
parser.add_argument("--report", required=True, help="Path to test-report.md")
|
|
313
|
+
parser.add_argument("--openapi", help="Optional OpenAPI file whose paths must appear in Boundary Matrix")
|
|
314
|
+
args = parser.parse_args()
|
|
315
|
+
|
|
316
|
+
report_path = Path(args.report)
|
|
317
|
+
report_text = read_text(report_path)
|
|
318
|
+
|
|
319
|
+
openapi_paths = None
|
|
320
|
+
openapi_operations = None
|
|
321
|
+
if args.openapi:
|
|
322
|
+
openapi_path = Path(args.openapi)
|
|
323
|
+
if openapi_path.exists():
|
|
324
|
+
openapi_text = read_text(openapi_path)
|
|
325
|
+
openapi_operations = extract_openapi_operations(openapi_text)
|
|
326
|
+
openapi_paths = [path for _method, path in openapi_operations]
|
|
327
|
+
else:
|
|
328
|
+
print(f"WARN: OpenAPI file not found, skipping path coverage check: {openapi_path}")
|
|
329
|
+
|
|
330
|
+
errors = validate_report(report_text, openapi_paths, openapi_operations)
|
|
331
|
+
if errors:
|
|
332
|
+
print("Boundary report validation: FAIL")
|
|
333
|
+
for error in errors:
|
|
334
|
+
print(f"- {error}")
|
|
335
|
+
return 1
|
|
336
|
+
|
|
337
|
+
print("Boundary report validation: PASS")
|
|
338
|
+
print(f"Report: {report_path}")
|
|
339
|
+
if openapi_operations is not None:
|
|
340
|
+
print(f"OpenAPI operations checked: {len(openapi_operations)}")
|
|
341
|
+
elif openapi_paths is not None:
|
|
342
|
+
print(f"OpenAPI paths checked: {len(openapi_paths)}")
|
|
343
|
+
return 0
|
|
344
|
+
|
|
345
|
+
|
|
346
|
+
if __name__ == "__main__":
|
|
347
|
+
sys.exit(main())
|
|
@@ -32,8 +32,13 @@ The loop repeats until the Reviewer finds no issues or the max round limit is re
|
|
|
32
32
|
|
|
33
33
|
1. **Read spec.md** from the artifact directory — extract goals and acceptance criteria.
|
|
34
34
|
2. **Read plan.md** from the artifact directory — extract architecture decisions and completed tasks.
|
|
35
|
-
3. **Read
|
|
36
|
-
|
|
35
|
+
3. **Read scoped test report when present**:
|
|
36
|
+
- If `{artifact_dir}/test-report-path.txt` exists, read the path, then read that `/prizmkit-test` report.
|
|
37
|
+
- Extract `Scope`, `Generated / Updated Tests`, `In-Scope Failures`, `Baseline Failures`, `Out of Scope Gaps`, `Boundary Matrix` / `Boundary Completion Gate` when present, and `Verdict`.
|
|
38
|
+
- If `context-snapshot.md` contains `## PrizmKit Test Gate`, read that summary too.
|
|
39
|
+
- If this is a feature artifact under `.prizmkit/specs/` and no test report pointer or summary exists, pass `No scoped /prizmkit-test report found` into the Reviewer context so the reviewer can flag the missing gate.
|
|
40
|
+
4. **Read dev rules** (if configured): If `.prizmkit/prizm-docs/root.prizm` exists, read it and check for a `RULES:` line. If present, read all referenced `.prizmkit/rules/<layer>-rules.md` files. If `root.prizm` or a referenced rules file does not exist, continue with "No custom dev rules configured — use general best practices."
|
|
41
|
+
5. **Capture workspace diff**: run `git diff` (unstaged) + `git diff --cached` (staged) + `git status` to understand the full scope of changes. For new files in git status, note their paths for the Reviewer to read.
|
|
37
42
|
- If no changes are detected, skip Phase 1 and proceed to Phase 2 with verdict PASS, rounds 0, and an empty findings list. Always write `review-report.md`; downstream pipeline steps use it as the review gate.
|
|
38
43
|
|
|
39
44
|
## Phase 1: Review-Fix Loop
|
|
@@ -11,6 +11,9 @@ You are a code reviewer. Review workspace changes against the spec goals, plan d
|
|
|
11
11
|
## Plan Decisions
|
|
12
12
|
{architecture decisions and task list from plan.md}
|
|
13
13
|
|
|
14
|
+
## Scoped Test Report
|
|
15
|
+
{summary from artifact_dir/test-report-path.txt and test-report.md, including Scope, Generated / Updated Tests, In-Scope Failures, Baseline Failures, Out of Scope Gaps, Boundary Matrix / Boundary Completion Gate if present, and Verdict. If no scoped report exists for a feature artifact, write "No scoped /prizmkit-test report found."}
|
|
16
|
+
|
|
14
17
|
## Dev Rules (per-layer conventions)
|
|
15
18
|
{rules from .prizmkit/rules/<layer>-rules.md, or "No custom dev rules configured — use general best practices."}
|
|
16
19
|
|
|
@@ -34,7 +37,8 @@ Evaluate the changes across these dimensions (focus on what's relevant):
|
|
|
34
37
|
3. **Completeness**: Files that should have been changed but weren't? Missing tests, types, imports, exports?
|
|
35
38
|
4. **Consistency**: Do changes follow the project's existing patterns, naming conventions, and code style?
|
|
36
39
|
5. **Security**: Hardcoded secrets, injection vulnerabilities, unsafe operations.
|
|
37
|
-
6. **
|
|
40
|
+
6. **Test quality**: If a scoped `/prizmkit-test` report is present, verify the report scope is `this-change`, the artifact dir matches the feature, generated/updated tests assert real behavior rather than mock success or empty assertions, in-scope failures are not mislabeled as baseline failures, baseline failures are genuinely unrelated or pre-existing, out-of-scope gaps were not turned into unrelated test changes, and the report verdict is compatible with proceeding.
|
|
41
|
+
7. **Rules compliance**: (Skip this dimension if no dev rules were provided.) Do changes follow the per-layer dev rules? Flag violations of framework conventions, naming patterns, state management, or other rules defined for that layer.
|
|
38
42
|
|
|
39
43
|
## Output Format
|
|
40
44
|
Respond with EXACTLY this format:
|
|
@@ -46,7 +50,7 @@ Respond with EXACTLY this format:
|
|
|
46
50
|
|
|
47
51
|
#### Finding N
|
|
48
52
|
- **Severity**: high | medium | low
|
|
49
|
-
- **Dimension**: goal-alignment | defect | completeness | consistency | security | rules-compliance
|
|
53
|
+
- **Dimension**: goal-alignment | defect | completeness | consistency | security | test-quality | rules-compliance
|
|
50
54
|
- **Location**: filepath:line (or "project-level")
|
|
51
55
|
- **Problem**: What is wrong and why it matters
|
|
52
56
|
- **Suggestion**: Recommended fix approach
|
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
---
|
|
2
2
|
name: "prizmkit-test"
|
|
3
|
-
description: "Full-stack test generation and orchestration. Detects architecture, discovers/runs existing tests, analyzes coverage gaps, generates missing
|
|
3
|
+
description: "Full-stack test generation and orchestration. Detects architecture, discovers/runs existing tests, analyzes coverage gaps, generates missing unit/integration/E2E tests including business-specific boundary cases, validates every interface with a required boundary coverage matrix, and outputs a unified report. Use after completing development to verify quality before deploy. Trigger on: 'test', 'run tests', 'check quality', 'verify code', 'generate tests', 'test coverage', 'fill test gaps', 'boundary tests', 'edge cases', 'quality check', '测试', '验证', '跑测试', '补测试', '边界测试'. (project)"
|
|
4
4
|
---
|
|
5
5
|
|
|
6
6
|
# PrizmKit Test
|
|
7
7
|
|
|
8
|
-
A comprehensive test generation and orchestration skill. Discovers existing tests, runs them, compares coverage against spec acceptance criteria and module interfaces, generates missing tests at three levels (unit → integration → E2E), and produces a unified report.
|
|
8
|
+
A comprehensive test generation and orchestration skill. Discovers existing tests, runs them, compares coverage against spec acceptance criteria and module interfaces, generates missing tests at three levels (unit → integration → E2E), verifies business-specific boundary coverage for every in-scope interface, and produces a unified report.
|
|
9
9
|
|
|
10
10
|
### When to Use
|
|
11
11
|
- After completing one or more features/refactors/bugfixes
|
|
@@ -34,11 +34,26 @@ Before execution, load context once:
|
|
|
34
34
|
2. **Project config**: Read `.prizmkit/config.json` (tech stack, AI CLI config).
|
|
35
35
|
3. **Dependencies**: Read `package.json` or equivalent to detect test framework and project type.
|
|
36
36
|
|
|
37
|
+
For `scope=this-change`, load only:
|
|
38
|
+
- `.prizmkit/prizm-docs/root.prizm`
|
|
39
|
+
- relevant L1/L2 docs inferred from in-scope changed files
|
|
40
|
+
- `artifact_dir/spec.md`
|
|
41
|
+
- `artifact_dir/plan.md`
|
|
42
|
+
- `artifact_dir/context-snapshot.md` if present
|
|
43
|
+
- `artifact_dir/review-report.md` if present
|
|
44
|
+
- changed source files and directly related existing tests
|
|
45
|
+
|
|
46
|
+
Do not load all modules or all specs merely because they exist.
|
|
47
|
+
|
|
37
48
|
## Input
|
|
38
49
|
|
|
39
50
|
| Parameter | Required | Description |
|
|
40
51
|
|-----------|----------|-------------|
|
|
41
|
-
| `scope` | No |
|
|
52
|
+
| `scope` | No | `full-project`, `module:<name>`, `feature:<slug>`, or `this-change`. In headless mode, defaults to `this-change` only when `artifact_dir` points to `.prizmkit/specs/<FEATURE_SLUG>/`; otherwise defaults to full project. |
|
|
53
|
+
| `artifact_dir` | No | Feature artifact directory. For `scope=this-change`, must be `.prizmkit/specs/<FEATURE_SLUG>/`. Do not infer this-change scope from `.prizmkit/bugfix/` or `.prizmkit/refactor/`. |
|
|
54
|
+
| `diff_base` | No | Git ref used to compute changed files for this feature. |
|
|
55
|
+
| `changed_files` | No | Explicit caller-provided changed file list. Takes precedence over `diff_base`. |
|
|
56
|
+
| `generation_policy` | No | Defaults to `in-scope-only` for `scope=this-change`. Out-of-scope gaps are reported but not generated. |
|
|
42
57
|
|
|
43
58
|
## Execution
|
|
44
59
|
|
|
@@ -72,7 +87,14 @@ Before execution, load context once:
|
|
|
72
87
|
2. **Single module** — pick from L1 doc module names (e.g., "auth", "payment")
|
|
73
88
|
3. **Single feature** — pick from `.prizmkit/specs/###-*/` directories
|
|
74
89
|
|
|
75
|
-
**Headless mode
|
|
90
|
+
**Headless mode**:
|
|
91
|
+
1. If `scope=this-change`, require `artifact_dir=.prizmkit/specs/<FEATURE_SLUG>/` and run in feature-scoped mode.
|
|
92
|
+
2. If `artifact_dir=.prizmkit/specs/<FEATURE_SLUG>/` is provided without `scope`, treat it as `scope=this-change`.
|
|
93
|
+
3. If `scope=full-project` is explicitly provided, run full-project mode even if an artifact directory exists.
|
|
94
|
+
4. If `artifact_dir` points to `.prizmkit/bugfix/` or `.prizmkit/refactor/`, do not infer `scope=this-change`.
|
|
95
|
+
5. If neither `scope` nor feature `artifact_dir` is provided, default to full project.
|
|
96
|
+
|
|
97
|
+
`scope=this-change` is feature-only. It is valid only when `artifact_dir` is under `.prizmkit/specs/<FEATURE_SLUG>/`. Do not automatically apply `scope=this-change` to `.prizmkit/bugfix/<BUG_ID>/` or `.prizmkit/refactor/<REFACTOR_ID>/`. If the caller passes `scope=this-change` with a bugfix/refactor artifact directory, stop and report that this mode is unsupported for that artifact type. Bugfix and refactor workflows keep their existing reproduction-test and behavior-preservation test gates.
|
|
76
98
|
|
|
77
99
|
### Phase 2: Run Existing Tests
|
|
78
100
|
|
|
@@ -83,13 +105,37 @@ Before execution, load context once:
|
|
|
83
105
|
- Which test files exist (for gap analysis)
|
|
84
106
|
- Raw output (saved to report directory)
|
|
85
107
|
|
|
86
|
-
3.
|
|
108
|
+
3. Classify existing test failures before continuing:
|
|
109
|
+
- For `scope=this-change`, failures in changed tests, tests mapped to in-scope source files, or tests directly covering feature acceptance criteria are `In-Scope Failures` unless a captured pre-feature baseline proves they already failed before this feature.
|
|
110
|
+
- For `scope=this-change`, unrelated failures are `Baseline Failures`; record them without failing the scoped verdict.
|
|
111
|
+
- For other scopes, record failures as existing test failures and do not assume they are caused by this test-generation pass.
|
|
112
|
+
|
|
113
|
+
4. If `scope=this-change` has any unexplained `In-Scope Failures`, the final report verdict must be `NEEDS_FIXES` or `BLOCKED`, not `PASS`. Do not hide them as baseline failures.
|
|
87
114
|
|
|
88
115
|
### Phase 3: Coverage Gap Analysis
|
|
89
116
|
|
|
90
117
|
Staleness of `.prizmkit/prizm-docs/` was already checked during Context Loading (see Precondition). Gap analysis proceeds with the available data.
|
|
91
118
|
|
|
92
|
-
|
|
119
|
+
For `scope=this-change`:
|
|
120
|
+
|
|
121
|
+
1. Build the in-scope source set using this priority:
|
|
122
|
+
- `changed_files` parameter
|
|
123
|
+
- `git diff --name-only <diff_base>`
|
|
124
|
+
- files/tasks listed in `artifact_dir/plan.md`
|
|
125
|
+
- File Manifest in `artifact_dir/context-snapshot.md`
|
|
126
|
+
- direct file/module mapping from `artifact_dir/spec.md` acceptance criteria
|
|
127
|
+
2. Build the in-scope test set:
|
|
128
|
+
- tests already changed in this feature
|
|
129
|
+
- tests matching in-scope source files by project naming convention
|
|
130
|
+
- directly affected integration/E2E tests for feature acceptance criteria
|
|
131
|
+
3. Analyze only:
|
|
132
|
+
- changed/new public interfaces
|
|
133
|
+
- feature acceptance criteria from `artifact_dir/spec.md`
|
|
134
|
+
- integration boundaries directly affected by in-scope files
|
|
135
|
+
- UI/E2E flows directly affected by this feature
|
|
136
|
+
4. For unrelated historical gaps, record them under `Out of Scope Gaps`; do not generate tests and do not fail the feature verdict because of them.
|
|
137
|
+
|
|
138
|
+
For other scopes, compare what exists against what should exist, across three levels:
|
|
93
139
|
|
|
94
140
|
**Unit test gaps** — for each module in scope:
|
|
95
141
|
- Read the corresponding L2 `.prizm` doc INTERFACES section to get exported functions/classes. If no L2 doc exists for a module, analyze source files directly to identify exported functions/classes.
|
|
@@ -106,30 +152,76 @@ Compare what exists against what should exist, across three levels:
|
|
|
106
152
|
- Check existing E2E test files against these criteria
|
|
107
153
|
- Flag uncovered criteria
|
|
108
154
|
|
|
109
|
-
### Phase 4:
|
|
155
|
+
### Phase 4: Boundary Coverage Matrix
|
|
156
|
+
|
|
157
|
+
Phase 3 answers "which interfaces are missing tests?" Phase 4 answers "which business boundaries are missing for each interface?" Build this matrix before generating tests so the run does not mistake broad happy-path coverage for real safety coverage.
|
|
158
|
+
|
|
159
|
+
A path, method, function, or class is not considered covered merely because a happy-path test exists; happy path proves the interface works once, while boundary tests prove it behaves safely under realistic invalid, unauthorized, edge, and failure states.
|
|
160
|
+
|
|
161
|
+
Read `${SKILL_DIR}/references/boundary-coverage-protocol.md` for the required matrix format, category definitions, and completion gate. For every in-scope interface and API endpoint, classify applicable categories such as request validation, auth/permission/ownership, domain invariants, collection/pagination, date/time boundaries, state transitions, dependency failures, and response contract checks. In `scope=this-change`, build this matrix only for the in-scope feature interfaces/endpoints; unrelated interfaces belong in `Out of Scope Gaps`.
|
|
162
|
+
|
|
163
|
+
For each category, mark one of:
|
|
164
|
+
- `covered` — an existing test already asserts the boundary behavior
|
|
165
|
+
- `generated` — this run generated and passed the test
|
|
166
|
+
- `needs-review` — a valid generated test found behavior mismatch
|
|
167
|
+
- `skipped` — not applicable, with a concrete reason
|
|
110
168
|
|
|
111
|
-
|
|
169
|
+
If an interface has only happy-path coverage, mark it `boundary-missing` and generate applicable boundary tests in Phase 5. Do not mark happy-path-only coverage as `covered`.
|
|
112
170
|
|
|
113
|
-
|
|
171
|
+
### Phase 5: Generate Missing and Boundary Tests
|
|
114
172
|
|
|
115
|
-
|
|
173
|
+
Read `${SKILL_DIR}/references/test-generation-steps.md` for the detailed generation procedure — boundary-matrix-driven workflow, priority order (unit → integration → E2E), failure handling rules, and generation patterns for each test level.
|
|
174
|
+
|
|
175
|
+
When generating unit tests for service-like functions or API/integration tests for endpoints with business-specific behavior, use `${SKILL_DIR}/references/service-boundary-test-catalog.md` via the generation procedure so boundary cases reflect the interface's business responsibility, not only generic null/empty inputs.
|
|
176
|
+
|
|
177
|
+
For `scope=this-change`:
|
|
178
|
+
- Generate or update tests only for the in-scope source/test set.
|
|
179
|
+
- Check for equivalent existing tests before writing new tests.
|
|
180
|
+
- Do not create tests for unrelated modules just because coverage is missing.
|
|
181
|
+
- Do not refactor unrelated tests.
|
|
182
|
+
- Do not modify unrelated production code.
|
|
183
|
+
- Do not expand to full-project coverage unless the user explicitly requested `scope=full-project`.
|
|
184
|
+
- Generated tests must assert real behavior, boundaries, or integration contracts; do not add empty assertions or mock-success-only tests.
|
|
185
|
+
|
|
186
|
+
### Phase 6: Unified Report and Boundary Validation
|
|
116
187
|
|
|
117
188
|
Create `.prizmkit/test/{YYYY_MM_DD_HH_MM_SS}_testresult/` directory. Read `${SKILL_DIR}/references/test-report-template.md` for the full report format and artifact list.
|
|
118
189
|
|
|
190
|
+
For `scope=this-change`, the report must include:
|
|
191
|
+
- `## Scope` with mode, artifact dir, diff base, changed-files source, generation policy, in-scope source files, and in-scope test files
|
|
192
|
+
- `## Existing Tests Run`
|
|
193
|
+
- `## Generated / Updated Tests`
|
|
194
|
+
- `## In-Scope Failures`
|
|
195
|
+
- `## Baseline Failures`
|
|
196
|
+
- `## Out of Scope Gaps`
|
|
197
|
+
- `## Verdict` with `PASS`, `NEEDS_FIXES`, or `BLOCKED`
|
|
198
|
+
|
|
199
|
+
`Out of Scope Gaps` are informational only for `scope=this-change`: they must not trigger generated tests and must not fail the verdict.
|
|
200
|
+
|
|
201
|
+
After writing the report, run the deterministic boundary-report validator when the project has an OpenAPI file or the report contains a Boundary Matrix:
|
|
202
|
+
|
|
203
|
+
```bash
|
|
204
|
+
python3 ${SKILL_DIR}/scripts/validate_boundary_report.py \
|
|
205
|
+
--report .prizmkit/test/{timestamp}_testresult/test-report.md \
|
|
206
|
+
--openapi openapi.yaml
|
|
207
|
+
```
|
|
208
|
+
|
|
209
|
+
If the project has no OpenAPI file, omit `--openapi`. If validation fails, keep the report but do not claim the run is complete; report that tests may pass while boundary coverage remains incomplete.
|
|
210
|
+
|
|
119
211
|
## Output
|
|
120
212
|
|
|
121
213
|
- Test report: `.prizmkit/test/{timestamp}_testresult/test-report.md`
|
|
122
214
|
- Generated test files written to project test directories
|
|
123
|
-
- Existing test output, generated test copies,
|
|
215
|
+
- Existing test output, generated test copies, E2E artifacts, and boundary validation output in the report directory
|
|
124
216
|
|
|
125
217
|
## Recovery
|
|
126
218
|
|
|
127
219
|
If the session is interrupted:
|
|
128
220
|
- Check `.prizmkit/test/` for the most recent report directory — it contains what was completed before interruption
|
|
129
|
-
- Re-run `/prizmkit-test` — it starts fresh, but Phase 2 will skip tests that already pass, and Phase 3 will re-evaluate gaps
|
|
221
|
+
- Re-run `/prizmkit-test` — it starts fresh, but Phase 2 will skip tests that already pass, and Phase 3/4 will re-evaluate gaps and boundary coverage
|
|
130
222
|
|
|
131
223
|
## Examples
|
|
132
224
|
|
|
133
|
-
Read `${SKILL_DIR}/references/examples.md` for worked examples of full-project
|
|
225
|
+
Read `${SKILL_DIR}/references/examples.md` for worked examples of full-project, single-feature, and happy-path-only remediation runs.
|
|
134
226
|
|
|
135
|
-
**HANDOFF:** Independent skill — no handoff. User may proceed to `/prizmkit-committer` if
|
|
227
|
+
**HANDOFF:** Independent skill — no handoff. User may proceed to `/prizmkit-committer` only if tests pass and the boundary completion gate passes. If tests are marked `needs-review` or boundary validation fails, fix issues manually or run another targeted `/prizmkit-test` pass.
|