speccle 0.11.0 → 0.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +290 -3
- package/dist/calibration.js +156 -0
- package/dist/changeset.js +141 -0
- package/dist/claims.js +5 -1
- package/dist/cli.js +596 -4
- package/dist/config.js +117 -0
- package/dist/doctor.js +141 -0
- package/dist/init.js +5 -4
- package/dist/lenses.js +68 -0
- package/dist/remedy.js +133 -0
- package/dist/render.js +323 -2
- package/dist/reviewinit.js +139 -0
- package/dist/reviewrun.js +516 -0
- package/dist/risk.js +228 -0
- package/dist/skills.js +63 -0
- package/dist/update.js +48 -0
- package/dist/verify.js +92 -0
- package/lenses/accessibility.md +39 -0
- package/lenses/architecture.md +38 -0
- package/lenses/correctness.md +36 -0
- package/lenses/house-conventions.md +37 -0
- package/lenses/performance.md +40 -0
- package/lenses/risk.md +45 -0
- package/lenses/security.md +42 -0
- package/lenses/test-quality.md +39 -0
- package/package.json +7 -5
- package/skills/carve-feature/SKILL.md +183 -0
- package/skills/carve-feature/references/convention.md +239 -0
- package/skills/conform/SKILL.md +129 -0
- package/skills/conform/references/convention.md +239 -0
- package/skills/feature/SKILL.md +103 -0
- package/skills/implement-feature/SKILL.md +152 -0
- package/skills/implement-feature/references/convention.md +239 -0
- package/skills/plan-feature/SKILL.md +125 -0
- package/skills/plan-feature/references/convention.md +239 -0
- package/skills/review/SKILL.md +190 -0
- package/skills/spec-feature/SKILL.md +163 -0
- package/skills/spec-feature/references/convention.md +239 -0
- package/skills/strengthen/SKILL.md +184 -0
- package/skills/strengthen/references/heatmap.md +39 -0
- package/skills/strengthen/references/stack.md +23 -0
package/README.md
CHANGED
|
@@ -1,17 +1,39 @@
|
|
|
1
1
|
# speccle
|
|
2
2
|
|
|
3
|
-
The deterministic tooling the skills invoke
|
|
3
|
+
The deterministic tooling the skills invoke — one bin:
|
|
4
4
|
|
|
5
5
|
```
|
|
6
|
+
speccle init # record repo facts + vendor the skills into .claude/skills/
|
|
7
|
+
speccle doctor # report staleness across the CLI, skills, and strength stack
|
|
8
|
+
speccle update # refresh the vendored skills; print the CLI + stack fix commands
|
|
6
9
|
speccle lint # enforce the convention over a repo's specs
|
|
7
10
|
speccle claims # join criteria to the test names that claim them
|
|
11
|
+
speccle verify # run .speccle/checks/ over a change set: cross-file invariants
|
|
12
|
+
speccle risk # score a change set from spec-aware signals; gate on the threshold
|
|
13
|
+
speccle calibrate # record / report the calibration evidence a threshold moves on
|
|
14
|
+
speccle remedy # record / recall the known remedy for a class of finding
|
|
15
|
+
speccle review init # scaffold the opt-in CI review driver (a GitHub Actions workflow)
|
|
16
|
+
speccle review run # review a pull request with the lens panel and post one review
|
|
8
17
|
speccle strength # oracle-strength heatmap: per-criterion killed ÷ covered
|
|
9
18
|
speccle strength init # provision the strength stack into a target
|
|
10
19
|
```
|
|
11
20
|
|
|
21
|
+
- `init` / `doctor` / `update` — the setup and staleness surface: `init` records the
|
|
22
|
+
repo's test facts in `.speccle/config.json` and vendors the skills into
|
|
23
|
+
`.claude/skills/`; `doctor` reports whether those skills and the strength stack still
|
|
24
|
+
match this CLI; `update` refreshes the skills forward and prints the CLI + stack fix
|
|
25
|
+
commands. See [Install](https://github.com/matthewalton/speccle/blob/main/README.md#install)
|
|
26
|
+
and [Updating](https://github.com/matthewalton/speccle/blob/main/README.md#updating).
|
|
12
27
|
- `lint` — enforce the [convention](https://github.com/matthewalton/speccle/blob/main/docs/convention.md) over a repo's specs.
|
|
13
28
|
- `claims` — join every criterion to the test names carrying its id, statically. No
|
|
14
29
|
reports needed, so it is cheap enough to gate on.
|
|
30
|
+
- `verify` / `risk` — the change-set surface. Both read the working tree's pending change
|
|
31
|
+
by default, or a committed range with `--base <ref>`, measured at the merge base — which
|
|
32
|
+
is what a CI run needs, where the working tree is clean. `risk` exits 1 at or above the
|
|
33
|
+
review threshold, so it works as a status check.
|
|
34
|
+
- `calibrate` / `remedy` — the meta loop's two records: the evidence a review threshold
|
|
35
|
+
moves on, and the known-correct remedy for a class of finding.
|
|
36
|
+
- `review init` / `review run` — the CI driver. See [review](#review) below.
|
|
15
37
|
- `strength` — join specs + Stryker mutation report + coverage into per-criterion
|
|
16
38
|
`killed ÷ covered`.
|
|
17
39
|
- `strength init` — the setup `strength` measures against: install the stack's
|
|
@@ -21,8 +43,9 @@ The bin is named after the package; every command is an explicit subcommand (a b
|
|
|
21
43
|
invocation is a usage error, exit code 2). `strength` names the measurement — oracle
|
|
22
44
|
strength — not the heatmap rendering of it.
|
|
23
45
|
|
|
24
|
-
|
|
25
|
-
typed JSON, never calls an LLM (see [CONTEXT.md](https://github.com/matthewalton/speccle/blob/main/CONTEXT.md)).
|
|
46
|
+
Every command here but one is a **Speccle tool**: deterministic, independently runnable,
|
|
47
|
+
emits typed JSON, never calls an LLM (see [CONTEXT.md](https://github.com/matthewalton/speccle/blob/main/CONTEXT.md)).
|
|
48
|
+
The exception is `review run`, the CI driver, which calls a model by definition.
|
|
26
49
|
|
|
27
50
|
## lint
|
|
28
51
|
|
|
@@ -71,6 +94,227 @@ failure mode is a false alarm, never a silent pass. `--json` emits the typed
|
|
|
71
94
|
`ClaimsReport` (see [`src/claims.ts`](src/claims.ts)). Exit codes: `0` every criterion
|
|
72
95
|
claimed and no unknown claims, `1` otherwise, `2` usage error.
|
|
73
96
|
|
|
97
|
+
## verify
|
|
98
|
+
|
|
99
|
+
```sh
|
|
100
|
+
speccle verify [path] [--json] [--base <ref>]
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
Runs every check in `.speccle/checks/` over a **change set**. Its reason to exist is the
|
|
104
|
+
class of invariant no linter can hold, because the fact is about the whole change rather
|
|
105
|
+
than any one file: _a changed `@Model` requires a round-trip test in the same change._
|
|
106
|
+
|
|
107
|
+
One check, one JSON file:
|
|
108
|
+
|
|
109
|
+
```json
|
|
110
|
+
{
|
|
111
|
+
"when": { "path": "features/**/src/*.ts", "contains": "@Model" },
|
|
112
|
+
"require": { "path": "features/**/src/*.test.ts", "contains": "roundTrip" },
|
|
113
|
+
"message": "a changed @Model needs a round-trip test in the same change",
|
|
114
|
+
"because": "PR #412 shipped a schema change with no round-trip coverage"
|
|
115
|
+
}
|
|
116
|
+
```
|
|
117
|
+
|
|
118
|
+
`when` is the trigger — absent means always enforced, and a check whose trigger never fires
|
|
119
|
+
reports as **inactive**, not as a pass. Then exactly one requirement: `require` breaches when
|
|
120
|
+
no changed file matches it, `forbid` breaches when any does, naming the offenders. Each is a
|
|
121
|
+
**predicate** — a glob over root-relative posix paths (`**` spans directories), optionally
|
|
122
|
+
narrowed by a `contains` regex the file's current content must match. `because` records the
|
|
123
|
+
finding that created the check and prints on breach, so a breach still explains itself a year
|
|
124
|
+
later.
|
|
125
|
+
|
|
126
|
+
```
|
|
127
|
+
model-roundtrip a changed @Model needs a round-trip test in the same change
|
|
128
|
+
because PR #412 shipped a schema change with no round-trip coverage
|
|
129
|
+
|
|
130
|
+
1 check, 1 breach
|
|
131
|
+
```
|
|
132
|
+
|
|
133
|
+
The change set is the working tree's pending change by default, or the commits between
|
|
134
|
+
`--base <ref>` and HEAD, measured from their merge base — what a CI run needs, where the tree
|
|
135
|
+
is clean and the change is committed. A repo that has authored no checks passes trivially; a
|
|
136
|
+
malformed check file is a hard error naming the file, because a check that silently does
|
|
137
|
+
nothing is worse than no check at all.
|
|
138
|
+
|
|
139
|
+
`--json` emits the typed `VerifyReport` (see [`src/verify.ts`](src/verify.ts)). Exit codes:
|
|
140
|
+
`0` no breach, `1` at least one breach, `2` usage error.
|
|
141
|
+
|
|
142
|
+
## risk
|
|
143
|
+
|
|
144
|
+
```sh
|
|
145
|
+
speccle risk [path] [--json] [--base <ref>] [--dialect <name>]
|
|
146
|
+
```
|
|
147
|
+
|
|
148
|
+
Scores a change set from spec-aware signals — facts only Speccle can see, because only
|
|
149
|
+
Speccle knows which code is governed by which spec. The score decides **how supervised the
|
|
150
|
+
change is**: at or above the review threshold a human is required, and review stops at
|
|
151
|
+
findings instead of fixing unasked.
|
|
152
|
+
|
|
153
|
+
Four baseline signals ship with the CLI and apply in every repo unmodified:
|
|
154
|
+
|
|
155
|
+
| signal | weight | fires on |
|
|
156
|
+
| -------------------- | -----: | --------------------------------------------------------------------- |
|
|
157
|
+
| `criterion-retired` | 4 | a criterion vanished from a changed `SPEC.md` |
|
|
158
|
+
| `spec-silent-change` | 3 | production source changed in a governed slice whose `SPEC.md` did not |
|
|
159
|
+
| `unclaimed-change` | 3 | changed code lives in a slice with a criterion no test claims |
|
|
160
|
+
| `criterion-reworded` | 2 | a criterion's statement changed |
|
|
161
|
+
|
|
162
|
+
```
|
|
163
|
+
spec-silent-change +3 production source changed in a governed slice whose SPEC.md did not
|
|
164
|
+
features/checkout/src/checkout.ts
|
|
165
|
+
unclaimed-change +3 changed code lives in a slice with a criterion no test claims
|
|
166
|
+
CHECKOUT-2
|
|
167
|
+
|
|
168
|
+
score 6 ≥ threshold 3 — human required, review stops at findings
|
|
169
|
+
this score is a floor — a risk lens may escalate it, never lower it
|
|
170
|
+
```
|
|
171
|
+
|
|
172
|
+
Every fired signal carries its evidence — the files, criterion ids, or slices it fired on —
|
|
173
|
+
so the number is auditable rather than asserted. The score is a **floor**: a risk lens reading
|
|
174
|
+
the change may escalate beyond it, never below it.
|
|
175
|
+
|
|
176
|
+
The default review threshold is `3`, deliberately low, so most changes are supervised. A repo
|
|
177
|
+
may reweight in `.speccle/risk.json` — the one sanctioned piece of configurable judgement here,
|
|
178
|
+
because what counts as consequential really is repo-specific:
|
|
179
|
+
|
|
180
|
+
```json
|
|
181
|
+
{
|
|
182
|
+
"threshold": 4,
|
|
183
|
+
"weights": { "criterion-reworded": 0 },
|
|
184
|
+
"signals": [
|
|
185
|
+
{
|
|
186
|
+
"id": "migration-without-rollback",
|
|
187
|
+
"weight": 5,
|
|
188
|
+
"when": { "path": "db/migrations/*.sql" },
|
|
189
|
+
"message": "a migration changed",
|
|
190
|
+
"because": "the incident on 2026-03-02"
|
|
191
|
+
}
|
|
192
|
+
]
|
|
193
|
+
}
|
|
194
|
+
```
|
|
195
|
+
|
|
196
|
+
A `0` weight mutes a baseline signal, and a muted signal is not computed at all. Only a human
|
|
197
|
+
edits this file, and a malformed one is a hard error rather than a silently-ignored weight —
|
|
198
|
+
a quietly dropped signal is exactly the invisible reduction of supervision the score exists to
|
|
199
|
+
prevent.
|
|
200
|
+
|
|
201
|
+
`--base` moves both the change set and the criterion baseline to the merge base. That pairing
|
|
202
|
+
is load-bearing: on a branch, HEAD already contains the change, so diffing a changed `SPEC.md`
|
|
203
|
+
against HEAD would compare it with itself and no criterion would ever read as retired or
|
|
204
|
+
reworded.
|
|
205
|
+
|
|
206
|
+
`--json` emits the typed `RiskReport` (see [`src/risk.ts`](src/risk.ts)). Exit codes: `0`
|
|
207
|
+
below the review threshold (review may fix and report), `1` at or above it (a human is
|
|
208
|
+
required), `2` usage error. Exit `1` is the verdict, not a failure — it is what makes the
|
|
209
|
+
command usable as a status check.
|
|
210
|
+
|
|
211
|
+
## calibrate
|
|
212
|
+
|
|
213
|
+
```sh
|
|
214
|
+
speccle calibrate record [path] --needed-human <true|false> --found-real <true|false>
|
|
215
|
+
[--escalated] [--note <text>] [--dialect <name>] [--json]
|
|
216
|
+
speccle calibrate report [path] [--json]
|
|
217
|
+
```
|
|
218
|
+
|
|
219
|
+
The evidence a review threshold moves on. `record` appends one line to
|
|
220
|
+
`.speccle/calibration.jsonl` per reviewed change: `risk` computes the deterministic floor —
|
|
221
|
+
score, threshold, the signals that fired — and the caller supplies the honest human verdict.
|
|
222
|
+
|
|
223
|
+
The two are recorded side by side and never conflated, which is the whole point. `--needed-human`
|
|
224
|
+
and `--found-real` are required and never defaulted, because recording the gate's verdict as if
|
|
225
|
+
it were the human's produces exactly the dishonest data a threshold must not rise on.
|
|
226
|
+
|
|
227
|
+
```
|
|
228
|
+
recorded .speccle/calibration.jsonl — 1 entry
|
|
229
|
+
score 6 vs threshold 3 — floor required a human
|
|
230
|
+
verdict: needed a human, found something real
|
|
231
|
+
signals: spec-silent-change, unclaimed-change
|
|
232
|
+
```
|
|
233
|
+
|
|
234
|
+
`report` reads the record back and answers three questions arithmetically: which signals fired
|
|
235
|
+
but never on a change that mattered, which fired on every change that genuinely needed a human,
|
|
236
|
+
and what threshold the record would support.
|
|
237
|
+
|
|
238
|
+
```
|
|
239
|
+
1 reviewed change — 1 needed a human, floor gated 1
|
|
240
|
+
|
|
241
|
+
spec-silent-change 1/1 mattered — on every change that needed a human
|
|
242
|
+
unclaimed-change 1/1 mattered — on every change that needed a human
|
|
243
|
+
|
|
244
|
+
proposals — evidence, not instructions (only a human reduces supervision):
|
|
245
|
+
spec-silent-change and unclaimed-change fired on every change that needed a human — reliable so far
|
|
246
|
+
the record would support a review threshold up to 6 (now 3) — only a human may raise it
|
|
247
|
+
```
|
|
248
|
+
|
|
249
|
+
`supportedThreshold` is the cheapest change that needed a human — conservative on purpose,
|
|
250
|
+
since a floor any higher would have missed it. Escalation is excluded from that arithmetic: a
|
|
251
|
+
safety net must never be the reason a floor sits high. Two counters name the failure directions
|
|
252
|
+
in plain terms — `floorMisses` (needed a human, and neither the floor nor a lens caught it) and
|
|
253
|
+
`overSupervised` (the floor gated a change the verdict says was fine).
|
|
254
|
+
|
|
255
|
+
The report **proposes and never applies**. Nothing here edits `.speccle/risk.json`; only a
|
|
256
|
+
human reduces supervision.
|
|
257
|
+
|
|
258
|
+
`--json` emits the typed `RecordReport` / `CalibrationReport` (see
|
|
259
|
+
[`src/calibration.ts`](src/calibration.ts)). Both subcommands exit `0` whenever they produced a
|
|
260
|
+
report — the verdict is `risk`'s job — and `2` on a usage error, including a missing verdict
|
|
261
|
+
flag or a malformed record line, which is reported with its line number rather than skipped.
|
|
262
|
+
|
|
263
|
+
## remedy
|
|
264
|
+
|
|
265
|
+
```sh
|
|
266
|
+
speccle remedy record [path] --class <handle> --finding <text> --fix <text>
|
|
267
|
+
--route <check|criterion|lens|none> [--artefact <ref>]
|
|
268
|
+
[--note <text>] [--json]
|
|
269
|
+
speccle remedy recall [path] --class <handle> [--json]
|
|
270
|
+
```
|
|
271
|
+
|
|
272
|
+
The memory that makes a repeat finding get the same answer twice. `record` appends one line to
|
|
273
|
+
`.speccle/remedies.jsonl`: what the finding was, the fix applied to the code this time, and the
|
|
274
|
+
**prevention route** chosen so the class stops recurring.
|
|
275
|
+
|
|
276
|
+
| route | prevention artefact |
|
|
277
|
+
| ----------- | ------------------------------ |
|
|
278
|
+
| `check` | a `.speccle/checks/` path |
|
|
279
|
+
| `criterion` | a `SPEC.md` criterion id |
|
|
280
|
+
| `lens` | a `.speccle/lenses/` path |
|
|
281
|
+
| `none` | names none — an honest one-off |
|
|
282
|
+
|
|
283
|
+
Every route but `none` must name its `--artefact`: a record is only worth consulting if it
|
|
284
|
+
points at the prevention it chose. `none` is the honest escape — a finding whose class is not
|
|
285
|
+
worth preventing prevents nothing, and must not pretend otherwise.
|
|
286
|
+
|
|
287
|
+
```
|
|
288
|
+
recorded .speccle/remedies.jsonl — 1 remedy
|
|
289
|
+
missing-model-roundtrip-test: a changed @Model shipped with no round-trip test
|
|
290
|
+
fix: added a round-trip test over the changed model
|
|
291
|
+
remedy: check → .speccle/checks/model-roundtrip.json
|
|
292
|
+
```
|
|
293
|
+
|
|
294
|
+
`recall` looks a class up before routing a finding fresh. Matching is deterministic on the
|
|
295
|
+
class handle's tokens — lowercased alphanumeric runs — and an entry matches when one side's
|
|
296
|
+
tokens are a subset of the other's, so `missing-roundtrip` still recalls
|
|
297
|
+
`missing-model-roundtrip-test`. Most-recent first: the latest answer to a class wins.
|
|
298
|
+
|
|
299
|
+
```
|
|
300
|
+
1 prior remedy for "missing-roundtrip" — reuse to fix consistently:
|
|
301
|
+
|
|
302
|
+
missing-model-roundtrip-test (2026-07-24T19:39:15.815Z)
|
|
303
|
+
finding: a changed @Model shipped with no round-trip test
|
|
304
|
+
fix: added a round-trip test over the changed model
|
|
305
|
+
remedy: check → .speccle/checks/model-roundtrip.json
|
|
306
|
+
```
|
|
307
|
+
|
|
308
|
+
The `--class` handle is the reviewer's judgement, not the tool's — it only stores and retrieves,
|
|
309
|
+
the same honesty split `calibrate` draws around the human verdict.
|
|
310
|
+
|
|
311
|
+
`--json` emits the typed `RemedyRecordReport` / `RemedyRecallReport` (see
|
|
312
|
+
[`src/remedy.ts`](src/remedy.ts)). Both subcommands exit `0` whenever they produced a report,
|
|
313
|
+
**including a recall that matched nothing** — no prior remedy is an answer ("route it fresh,
|
|
314
|
+
then record it"), not a failure. `2` is a usage error: a missing required flag, an unknown
|
|
315
|
+
route, an artefact on a `none` remedy, or a malformed record line, reported with its line
|
|
316
|
+
number rather than skipped.
|
|
317
|
+
|
|
74
318
|
## strength
|
|
75
319
|
|
|
76
320
|
```sh
|
|
@@ -116,6 +360,49 @@ no spec declares are reported too.
|
|
|
116
360
|
command exits `0` whenever it produced a report — judging a diff against a threshold is a
|
|
117
361
|
separate concern.
|
|
118
362
|
|
|
363
|
+
## review
|
|
364
|
+
|
|
365
|
+
The outer loop has two drivers. The **local driver** is the `review` skill: it fans the
|
|
366
|
+
lenses as subagents in a session and needs no key. The **CI driver** is these two commands,
|
|
367
|
+
and it is **opt-in**, because it needs a metered `ANTHROPIC_API_KEY`.
|
|
368
|
+
|
|
369
|
+
```sh
|
|
370
|
+
speccle review init [path] [--json]
|
|
371
|
+
```
|
|
372
|
+
|
|
373
|
+
Writes one file — `.github/workflows/speccle-review.yml` — pinned to the version of the CLI
|
|
374
|
+
that wrote it. Nothing else is vendored: the driver ships in this tarball and the workflow
|
|
375
|
+
fetches it from npm, so the code doing the reviewing never comes from the branch under
|
|
376
|
+
review. Re-running moves the pin, which is how a repo updates the driver.
|
|
377
|
+
|
|
378
|
+
```sh
|
|
379
|
+
speccle review run --pr <number> [path] [--repo <owner/name>] [--base <ref>]
|
|
380
|
+
[--force] [--model <id>] [--dry-run] [--json]
|
|
381
|
+
```
|
|
382
|
+
|
|
383
|
+
Fans every lens in `.speccle/lenses/` over the pull request's change set — one model call
|
|
384
|
+
per lens, findings forced into shape by a tool schema — and posts them as a single review
|
|
385
|
+
with inline comments. It **finds and comments only**: it never edits the tree, commits, or
|
|
386
|
+
pushes, because a fix has to re-run the checks-gate and be revertible, which is the local
|
|
387
|
+
driver's job.
|
|
388
|
+
|
|
389
|
+
- Skips `risk.md` (it escalates authority rather than reporting findings) and an unauthored
|
|
390
|
+
`house-conventions.md`, exactly as the local driver does.
|
|
391
|
+
- Posts one automatic review per pull request, recognised by a marker in its own body;
|
|
392
|
+
`--force` overrides that, which is what the workflow's `@review` rerun path passes.
|
|
393
|
+
- A finding that cannot anchor to a line in the diff is carried in the summary rather than
|
|
394
|
+
dropped, and if GitHub rejects the anchors outright the retry carries every finding in
|
|
395
|
+
the body.
|
|
396
|
+
- Reads `ANTHROPIC_API_KEY` and `GITHUB_TOKEN` from the environment; `--dry-run` reports
|
|
397
|
+
what would be posted and posts nothing. The default model is overridable with
|
|
398
|
+
`SPECCLE_REVIEW_MODEL`.
|
|
399
|
+
|
|
400
|
+
The risk verdict leads the summary, and the workflow runs `speccle risk --base` as its own
|
|
401
|
+
step so the **status check stays deterministic** — it survives a bad API day. Whether a
|
|
402
|
+
failing check blocks the merge is branch protection: GitHub's setting, the repo's call.
|
|
403
|
+
|
|
404
|
+
This is the one command here that calls a model.
|
|
405
|
+
|
|
119
406
|
## strength init
|
|
120
407
|
|
|
121
408
|
```sh
|
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
import { appendFile, mkdir, readFile } from "node:fs/promises";
|
|
2
|
+
import { dirname, join, resolve } from "node:path";
|
|
3
|
+
import { isDirectory, messageOf } from "./changeset.js";
|
|
4
|
+
import { reviewThreshold, risk } from "./risk.js";
|
|
5
|
+
/** Where the meta loop keeps its calibration record (ADR-0042): append-only, one entry per line. */
|
|
6
|
+
export const CALIBRATION_FILE = ".speccle/calibration.jsonl";
|
|
7
|
+
/**
|
|
8
|
+
* Appends one calibration entry: `risk` computes the deterministic floor (score, signals,
|
|
9
|
+
* threshold, humanRequired) over the change set, the caller supplies the honest human verdict.
|
|
10
|
+
* The floor and the verdict are recorded side by side and never conflated.
|
|
11
|
+
*/
|
|
12
|
+
export async function recordCalibration(target, input, options = {}) {
|
|
13
|
+
const root = resolve(target);
|
|
14
|
+
if (!(await isDirectory(root)))
|
|
15
|
+
throw new Error(`path not found: ${target}`);
|
|
16
|
+
const { now, ...riskOptions } = options;
|
|
17
|
+
const assessment = await risk(root, riskOptions);
|
|
18
|
+
const stamp = (now ?? (() => new Date().toISOString()))();
|
|
19
|
+
const entry = {
|
|
20
|
+
at: stamp,
|
|
21
|
+
score: assessment.score,
|
|
22
|
+
threshold: assessment.threshold,
|
|
23
|
+
humanRequired: assessment.humanRequired,
|
|
24
|
+
escalated: input.escalated ?? false,
|
|
25
|
+
signals: assessment.signals.map((signal) => signal.id),
|
|
26
|
+
verdict: { neededHuman: input.neededHuman, foundReal: input.foundReal },
|
|
27
|
+
...(input.note !== undefined && input.note !== "" && { note: input.note }),
|
|
28
|
+
};
|
|
29
|
+
const file = join(root, CALIBRATION_FILE);
|
|
30
|
+
await mkdir(dirname(file), { recursive: true });
|
|
31
|
+
await appendFile(file, JSON.stringify(entry) + "\n");
|
|
32
|
+
return { root, file: CALIBRATION_FILE, entry, count: (await readEntries(root)).length };
|
|
33
|
+
}
|
|
34
|
+
/**
|
|
35
|
+
* Reads the calibration record and answers ADR-0042's three questions deterministically: which
|
|
36
|
+
* signals never corresponded to a real problem, which fired on every change a human was needed
|
|
37
|
+
* for, and the threshold the record would support. It proposes; it never applies.
|
|
38
|
+
*/
|
|
39
|
+
export async function calibrationReport(target) {
|
|
40
|
+
const root = resolve(target);
|
|
41
|
+
if (!(await isDirectory(root)))
|
|
42
|
+
throw new Error(`path not found: ${target}`);
|
|
43
|
+
const entries = await readEntries(root);
|
|
44
|
+
const currentThreshold = await reviewThreshold(root);
|
|
45
|
+
const mattered = (entry) => entry.verdict.neededHuman || entry.verdict.foundReal;
|
|
46
|
+
const neededEntries = entries.filter((entry) => entry.verdict.neededHuman);
|
|
47
|
+
const ids = [...new Set(entries.flatMap((entry) => entry.signals))].sort();
|
|
48
|
+
const signals = ids.map((id) => {
|
|
49
|
+
const fired = entries.filter((entry) => entry.signals.includes(id));
|
|
50
|
+
const firedOnMattered = fired.filter(mattered).length;
|
|
51
|
+
return {
|
|
52
|
+
id,
|
|
53
|
+
fired: fired.length,
|
|
54
|
+
firedOnMattered,
|
|
55
|
+
neverUseful: fired.length > 0 && firedOnMattered === 0,
|
|
56
|
+
firedOnEveryNeeded: neededEntries.length > 0 && neededEntries.every((entry) => entry.signals.includes(id)),
|
|
57
|
+
};
|
|
58
|
+
});
|
|
59
|
+
// Conservative on purpose (ADR-0041 favours more supervision): the floor could rise no higher
|
|
60
|
+
// than the cheapest change that genuinely needed a human, or the floor alone would have missed
|
|
61
|
+
// it. Escalation is ignored here — a safety net must not be the reason a floor sits high.
|
|
62
|
+
const supportedThreshold = neededEntries.length === 0 ? null : Math.min(...neededEntries.map((entry) => entry.score));
|
|
63
|
+
const floorMisses = entries.filter((entry) => entry.verdict.neededHuman && !entry.humanRequired && !entry.escalated).length;
|
|
64
|
+
const overSupervised = entries.filter((entry) => entry.humanRequired && !entry.verdict.neededHuman).length;
|
|
65
|
+
return {
|
|
66
|
+
root,
|
|
67
|
+
file: CALIBRATION_FILE,
|
|
68
|
+
count: entries.length,
|
|
69
|
+
neededByHuman: neededEntries.length,
|
|
70
|
+
gatedByFloor: entries.filter((entry) => entry.humanRequired).length,
|
|
71
|
+
floorMisses,
|
|
72
|
+
overSupervised,
|
|
73
|
+
currentThreshold,
|
|
74
|
+
supportedThreshold,
|
|
75
|
+
signals,
|
|
76
|
+
proposals: proposalsFrom(entries.length, signals, currentThreshold, supportedThreshold),
|
|
77
|
+
};
|
|
78
|
+
}
|
|
79
|
+
function proposalsFrom(count, signals, currentThreshold, supportedThreshold) {
|
|
80
|
+
if (count === 0)
|
|
81
|
+
return ["no calibration entries yet — review some changes to build the record"];
|
|
82
|
+
const out = [];
|
|
83
|
+
const never = signals.filter((signal) => signal.neverUseful).map((signal) => signal.id);
|
|
84
|
+
if (never.length > 0) {
|
|
85
|
+
out.push(`${and(never)} fired but never on a change that mattered — weigh their weight, not a licence to delete`);
|
|
86
|
+
}
|
|
87
|
+
const reliable = signals.filter((signal) => signal.firedOnEveryNeeded).map((signal) => signal.id);
|
|
88
|
+
if (reliable.length > 0) {
|
|
89
|
+
out.push(`${and(reliable)} fired on every change that needed a human — reliable so far`);
|
|
90
|
+
}
|
|
91
|
+
if (supportedThreshold === null) {
|
|
92
|
+
out.push("no change has needed a human yet — the record cannot support a threshold move");
|
|
93
|
+
}
|
|
94
|
+
else if (supportedThreshold > currentThreshold) {
|
|
95
|
+
out.push(`the record would support a review threshold up to ${supportedThreshold} (now ${currentThreshold}) — only a human may raise it`);
|
|
96
|
+
}
|
|
97
|
+
else if (supportedThreshold < currentThreshold) {
|
|
98
|
+
out.push(`a change that needed a human scored ${supportedThreshold}, below the threshold of ${currentThreshold} — the floor would miss it; only a human may lower the threshold`);
|
|
99
|
+
}
|
|
100
|
+
else {
|
|
101
|
+
out.push(`the threshold of ${currentThreshold} matches the record — no move indicated`);
|
|
102
|
+
}
|
|
103
|
+
return out;
|
|
104
|
+
}
|
|
105
|
+
async function readEntries(root) {
|
|
106
|
+
let raw;
|
|
107
|
+
try {
|
|
108
|
+
raw = await readFile(join(root, CALIBRATION_FILE), "utf8");
|
|
109
|
+
}
|
|
110
|
+
catch {
|
|
111
|
+
return []; // no record yet: nothing has been calibrated.
|
|
112
|
+
}
|
|
113
|
+
const entries = [];
|
|
114
|
+
const lines = raw.split("\n");
|
|
115
|
+
for (let i = 0; i < lines.length; i++) {
|
|
116
|
+
const line = lines[i].trim();
|
|
117
|
+
if (line === "")
|
|
118
|
+
continue;
|
|
119
|
+
let entry;
|
|
120
|
+
try {
|
|
121
|
+
entry = JSON.parse(line);
|
|
122
|
+
}
|
|
123
|
+
catch (err) {
|
|
124
|
+
throw new Error(`${CALIBRATION_FILE}:${i + 1} is not valid JSON: ${messageOf(err)}`);
|
|
125
|
+
}
|
|
126
|
+
assertEntry(entry, i + 1);
|
|
127
|
+
entries.push(entry);
|
|
128
|
+
}
|
|
129
|
+
return entries;
|
|
130
|
+
}
|
|
131
|
+
// A malformed entry must fail loudly, naming the line — a silently-dropped entry is dishonest
|
|
132
|
+
// calibration data, the one thing ADR-0042 will not tolerate in the record.
|
|
133
|
+
function assertEntry(entry, line) {
|
|
134
|
+
const at = `${CALIBRATION_FILE}:${line}`;
|
|
135
|
+
const isNumber = (value) => typeof value === "number" && Number.isFinite(value);
|
|
136
|
+
if (!isNumber(entry.score) || !isNumber(entry.threshold)) {
|
|
137
|
+
throw new Error(`${at}: "score" and "threshold" must be numbers`);
|
|
138
|
+
}
|
|
139
|
+
if (typeof entry.humanRequired !== "boolean" || typeof entry.escalated !== "boolean") {
|
|
140
|
+
throw new Error(`${at}: "humanRequired" and "escalated" must be booleans`);
|
|
141
|
+
}
|
|
142
|
+
if (!Array.isArray(entry.signals) || entry.signals.some((id) => typeof id !== "string")) {
|
|
143
|
+
throw new Error(`${at}: "signals" must be an array of signal ids`);
|
|
144
|
+
}
|
|
145
|
+
const verdict = entry.verdict;
|
|
146
|
+
const fields = verdict;
|
|
147
|
+
if (typeof verdict !== "object" ||
|
|
148
|
+
verdict === null ||
|
|
149
|
+
typeof fields.neededHuman !== "boolean" ||
|
|
150
|
+
typeof fields.foundReal !== "boolean") {
|
|
151
|
+
throw new Error(`${at}: "verdict" needs boolean "neededHuman" and "foundReal"`);
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
function and(ids) {
|
|
155
|
+
return ids.length <= 1 ? (ids[0] ?? "") : `${ids.slice(0, -1).join(", ")} and ${ids.at(-1)}`;
|
|
156
|
+
}
|
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
import { spawnSync } from "node:child_process";
|
|
2
|
+
import { readFile, stat } from "node:fs/promises";
|
|
3
|
+
import { join } from "node:path";
|
|
4
|
+
/** The changed files a predicate matches — by path glob, then by content when `contains` is set. */
|
|
5
|
+
export async function matching(predicate, changed, read) {
|
|
6
|
+
const pattern = globToRegExp(predicate.path);
|
|
7
|
+
const contains = predicate.contains === undefined ? undefined : new RegExp(predicate.contains);
|
|
8
|
+
const matches = [];
|
|
9
|
+
for (const file of changed) {
|
|
10
|
+
if (!pattern.test(file))
|
|
11
|
+
continue;
|
|
12
|
+
if (contains !== undefined) {
|
|
13
|
+
const source = await read(file);
|
|
14
|
+
if (source === undefined || !contains.test(source))
|
|
15
|
+
continue;
|
|
16
|
+
}
|
|
17
|
+
matches.push(file);
|
|
18
|
+
}
|
|
19
|
+
return matches;
|
|
20
|
+
}
|
|
21
|
+
export async function anyMatches(predicate, changed, read) {
|
|
22
|
+
return (await matching(predicate, changed, read)).length > 0;
|
|
23
|
+
}
|
|
24
|
+
/**
|
|
25
|
+
* A reader for the change set's current content, caching each file so a file read by several
|
|
26
|
+
* predicates is only touched once. Missing files read as undefined rather than throwing.
|
|
27
|
+
*/
|
|
28
|
+
export function contentReader(root) {
|
|
29
|
+
const cache = new Map();
|
|
30
|
+
return async (file) => {
|
|
31
|
+
if (!cache.has(file))
|
|
32
|
+
cache.set(file, await readMaybe(join(root, file)));
|
|
33
|
+
return cache.get(file);
|
|
34
|
+
};
|
|
35
|
+
}
|
|
36
|
+
/** The pending change set: everything the working tree differs from its last commit by. */
|
|
37
|
+
export function gitChangeSet(root) {
|
|
38
|
+
// --untracked-files=all lists new files individually; the default collapses a wholly
|
|
39
|
+
// untracked directory to its name, hiding the files that must be seen.
|
|
40
|
+
const args = ["status", "--porcelain", "--untracked-files=all"];
|
|
41
|
+
const stdout = git(root, args);
|
|
42
|
+
if (stdout === undefined) {
|
|
43
|
+
throw new Error("could not read a change set from git — run this inside a git repository");
|
|
44
|
+
}
|
|
45
|
+
const changed = new Set();
|
|
46
|
+
for (const line of stdout.split("\n")) {
|
|
47
|
+
if (line === "")
|
|
48
|
+
continue;
|
|
49
|
+
// Porcelain: two status chars, a space, then the path — or "old -> new" for a rename.
|
|
50
|
+
const path = line.slice(3);
|
|
51
|
+
changed.add(path.includes(" -> ") ? path.slice(path.indexOf(" -> ") + 4) : path);
|
|
52
|
+
}
|
|
53
|
+
return [...changed];
|
|
54
|
+
}
|
|
55
|
+
/**
|
|
56
|
+
* The committed change set between a base ref and HEAD — the change set a CI driver reviews,
|
|
57
|
+
* where the working tree is clean and the change lives in commits instead. Measured from the
|
|
58
|
+
* **merge base**, not the base's tip, so commits that landed on the base after this branch
|
|
59
|
+
* left it are not attributed to this change; that is the set a pull request shows.
|
|
60
|
+
*/
|
|
61
|
+
export function gitRangeChangeSet(root, base) {
|
|
62
|
+
const mergeBase = git(root, ["merge-base", base, "HEAD"])?.trim();
|
|
63
|
+
if (mergeBase === undefined || mergeBase === "") {
|
|
64
|
+
// The likeliest cause in CI by far: a shallow checkout that fetched no shared history.
|
|
65
|
+
throw new Error(`no merge base between "${base}" and HEAD — fetch enough history for the two to share a commit`);
|
|
66
|
+
}
|
|
67
|
+
// A rename reports its destination path only, matching the working tree's "old -> new".
|
|
68
|
+
const stdout = git(root, ["diff", "--name-only", mergeBase, "HEAD"]);
|
|
69
|
+
if (stdout === undefined)
|
|
70
|
+
throw new Error(`could not diff "${base}" against HEAD`);
|
|
71
|
+
const changed = stdout.split("\n").filter((line) => line !== "");
|
|
72
|
+
return { changed: [...new Set(changed)], baseline: mergeBase };
|
|
73
|
+
}
|
|
74
|
+
/** Git's stdout, or undefined when the command could not run or failed. */
|
|
75
|
+
function git(root, args) {
|
|
76
|
+
const result = spawnSync("git", args, { cwd: root, encoding: "utf8" });
|
|
77
|
+
return result.error === undefined && result.status === 0 ? result.stdout : undefined;
|
|
78
|
+
}
|
|
79
|
+
/** Validates a predicate's shape, naming the offending file — a silently broken predicate is worse than none. */
|
|
80
|
+
export function assertPredicate(predicate, key, file) {
|
|
81
|
+
if (predicate === undefined)
|
|
82
|
+
return;
|
|
83
|
+
if (typeof predicate.path !== "string" || predicate.path === "") {
|
|
84
|
+
throw new Error(`${file}: "${key}" needs a non-empty "path" glob`);
|
|
85
|
+
}
|
|
86
|
+
if (predicate.contains !== undefined) {
|
|
87
|
+
try {
|
|
88
|
+
new RegExp(predicate.contains);
|
|
89
|
+
}
|
|
90
|
+
catch (err) {
|
|
91
|
+
throw new Error(`${file}: "${key}.contains" is not a valid regex: ${messageOf(err)}`);
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
/**
|
|
96
|
+
* A minimatch-style glob over posix paths: `**` spans directory boundaries, a single `*`
|
|
97
|
+
* and `?` stay within one segment.
|
|
98
|
+
*/
|
|
99
|
+
function globToRegExp(glob) {
|
|
100
|
+
let re = "";
|
|
101
|
+
for (let i = 0; i < glob.length; i++) {
|
|
102
|
+
const char = glob[i];
|
|
103
|
+
if (char === "*") {
|
|
104
|
+
if (glob[i + 1] === "*") {
|
|
105
|
+
i++;
|
|
106
|
+
if (glob[i + 1] === "/") {
|
|
107
|
+
i++;
|
|
108
|
+
re += "(?:.*/)?"; // `**/` also matches zero directories
|
|
109
|
+
}
|
|
110
|
+
else
|
|
111
|
+
re += ".*";
|
|
112
|
+
}
|
|
113
|
+
else
|
|
114
|
+
re += "[^/]*";
|
|
115
|
+
}
|
|
116
|
+
else if (char === "?")
|
|
117
|
+
re += "[^/]";
|
|
118
|
+
else
|
|
119
|
+
re += char.replace(/[.+^${}()|[\]\\]/g, "\\$&");
|
|
120
|
+
}
|
|
121
|
+
return new RegExp(`^${re}$`);
|
|
122
|
+
}
|
|
123
|
+
async function readMaybe(path) {
|
|
124
|
+
try {
|
|
125
|
+
return await readFile(path, "utf8");
|
|
126
|
+
}
|
|
127
|
+
catch {
|
|
128
|
+
return undefined;
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
export async function isDirectory(path) {
|
|
132
|
+
try {
|
|
133
|
+
return (await stat(path)).isDirectory();
|
|
134
|
+
}
|
|
135
|
+
catch {
|
|
136
|
+
return false;
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
export function messageOf(err) {
|
|
140
|
+
return err instanceof Error ? err.message : String(err);
|
|
141
|
+
}
|
package/dist/claims.js
CHANGED
|
@@ -1,13 +1,17 @@
|
|
|
1
1
|
import { readFile, stat } from "node:fs/promises";
|
|
2
2
|
import { dirname, join, resolve } from "node:path";
|
|
3
|
+
import { readConfig } from "./config.js";
|
|
3
4
|
import { DEFAULT_DIALECT, resolveDialect } from "./dialects.js";
|
|
4
5
|
import { discoverSpecs, discoverTests } from "./discover.js";
|
|
5
6
|
import { compareCriterionIds, parseSpec, readClaimedIds } from "./spec.js";
|
|
6
7
|
export async function claims(target, options = {}) {
|
|
7
|
-
const dialect = resolveDialect(options.dialect ?? DEFAULT_DIALECT);
|
|
8
8
|
const root = resolve(target);
|
|
9
9
|
if (!(await isDirectory(root)))
|
|
10
10
|
throw new Error(`path not found: ${target}`);
|
|
11
|
+
// An explicit --dialect wins outright; otherwise `.speccle/config.json` is the source
|
|
12
|
+
// of truth, falling back to the default only when the repo has recorded nothing.
|
|
13
|
+
const declared = options.dialect ?? (await readConfig(root))?.dialect;
|
|
14
|
+
const dialect = resolveDialect(declared ?? DEFAULT_DIALECT);
|
|
11
15
|
const specFiles = await discoverSpecs(root);
|
|
12
16
|
const specs = await Promise.all(specFiles.map(async (file) => parseSpec(await readFile(join(root, file), "utf8"), file)));
|
|
13
17
|
const criteria = new Map();
|