supercov 0.0.43 → 0.0.45
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +23 -14
- package/docs/agent-loop.md +128 -39
- package/docs/assertion-agent.md +156 -0
- package/docs/assertion-evidence.md +9 -135
- package/docs/assertion-maps.md +252 -0
- package/docs/assertions.md +82 -0
- package/docs/cli.md +38 -23
- package/docs/coverage-model.md +18 -6
- package/docs/evidence.md +6 -6
- package/docs/getting-started.md +60 -72
- package/docs/performance.md +4 -4
- package/docs/troubleshooting.md +8 -8
- package/docs/verification.md +2 -2
- package/docs/workspace-isolation.md +1 -1
- package/package.json +34 -33
- package/runtime/javascript/nodeAssertAdapter.mjs +32 -8
- package/runtime/javascript/nodeTest.mjs +13 -5
- package/runtime/javascript/runnerEvidence.mjs +33 -11
- package/runtime/javascript/runtime.mjs +27 -23
- package/schemas/assertions.schema.json +276 -0
- package/analyzers/typescript/README.md +0 -55
- package/analyzers/typescript/bin/compiler-identity.mjs +0 -78
- package/analyzers/typescript/bin/identity.mjs +0 -67
- package/analyzers/typescript/bin/query.mjs +0 -29
- package/analyzers/typescript/dist/analyze.js +0 -3972
- package/analyzers/typescript/dist/archive.js +0 -309
- package/analyzers/typescript/dist/build-identity.json +0 -1
- package/analyzers/typescript/dist/compiler.js +0 -32
- package/analyzers/typescript/dist/frontend.js +0 -75
- package/analyzers/typescript/dist/native-frontend.js +0 -271
- package/analyzers/typescript/dist/pragmas.js +0 -143
- package/analyzers/typescript/dist/types.js +0 -1
- package/analyzers/typescript/package.json +0 -27
- package/analyzers/typescript/src/analyze.ts +0 -4538
- package/analyzers/typescript/src/archive.ts +0 -438
- package/analyzers/typescript/src/compiler.ts +0 -49
- package/analyzers/typescript/src/frontend.ts +0 -136
- package/analyzers/typescript/src/native-frontend.ts +0 -315
- package/analyzers/typescript/src/pragmas.ts +0 -218
- package/analyzers/typescript/src/types.ts +0 -45
- package/analyzers/typescript/tsconfig.json +0 -12
- package/docs/code-verification.md +0 -182
|
@@ -0,0 +1,252 @@
|
|
|
1
|
+
# Assertion maps
|
|
2
|
+
|
|
3
|
+
Each normal test run creates `assertions.json` in its run directory. Any coding
|
|
4
|
+
agent can edit that file to describe what specific assertions observe and which
|
|
5
|
+
statements contribute to those observations. Rust handles syntax, references,
|
|
6
|
+
input freshness, reuse and reporting. Supercov does not run an embedded model or
|
|
7
|
+
reconstruct the agent's semantic reasoning with a static analyzer.
|
|
8
|
+
|
|
9
|
+
## Run, investigate, save
|
|
10
|
+
|
|
11
|
+
```sh
|
|
12
|
+
supercov -- npm test
|
|
13
|
+
supercov runs latest assertions --json
|
|
14
|
+
# Pin the returned run ID; edit data.map using matching current project files.
|
|
15
|
+
supercov runs <run> assertions report --view changes --json
|
|
16
|
+
supercov runs <run> assertions validate --json
|
|
17
|
+
# After investigating, copy expectedBasis tokens into the corresponding entries.
|
|
18
|
+
supercov runs <run> assertions check --require-mappings --require-observed --json
|
|
19
|
+
supercov runs <run>
|
|
20
|
+
```
|
|
21
|
+
|
|
22
|
+
There is no `init` or `review` command. `validate` is read-only: it returns the
|
|
23
|
+
expected acknowledgements, and the agent saves them in the one editable file.
|
|
24
|
+
A token says that the author acknowledges these particular inputs and this
|
|
25
|
+
particular claim. It is not proof, a signature, or a model confidence score.
|
|
26
|
+
|
|
27
|
+
## File format, version 2
|
|
28
|
+
|
|
29
|
+
```json
|
|
30
|
+
{
|
|
31
|
+
"schemaVersion": 2,
|
|
32
|
+
"assertions": [{
|
|
33
|
+
"id": "a_example",
|
|
34
|
+
"at": {
|
|
35
|
+
"file": "tests/value.test.ts", "line": 5, "column": 3,
|
|
36
|
+
"text": "assert.equal(value(), 1)"
|
|
37
|
+
},
|
|
38
|
+
"observes": ["The returned number equals one."],
|
|
39
|
+
"flows": [{
|
|
40
|
+
"id": "return-value",
|
|
41
|
+
"basis": null,
|
|
42
|
+
"appliesTo": [{ "file": "tests/value.test.ts", "name": "value" }],
|
|
43
|
+
"explanation": "The function's returned number reaches the equality assertion through value().",
|
|
44
|
+
"nodes": [{
|
|
45
|
+
"id": "return",
|
|
46
|
+
"at": { "file": "src/value.ts", "line": 2, "column": 3, "text": "return 1;" }
|
|
47
|
+
}],
|
|
48
|
+
"edges": [{ "from": "return", "to": "$assertion", "kind": "data" }],
|
|
49
|
+
"countsAsAsserted": ["return"],
|
|
50
|
+
"watch": []
|
|
51
|
+
}]
|
|
52
|
+
}]
|
|
53
|
+
}
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
This is an illustrative draft; its anchors and test selector must match a real
|
|
57
|
+
run before it can receive credit. `basis: null` explicitly means unacknowledged.
|
|
58
|
+
`supercov assertions schema` exports the schema generated from the same Rust
|
|
59
|
+
types as the parser. The published editor schema is `schemas/assertions.schema.json`.
|
|
60
|
+
`supercov assertions validate --file <path> --json` checks JSON shape without a run.
|
|
61
|
+
|
|
62
|
+
| Field | Meaning |
|
|
63
|
+
| --- | --- |
|
|
64
|
+
| `id`, `at` | Persistent assertion identity and exact assertion expression. |
|
|
65
|
+
| `observes` | What the predicate distinguishes; an existence check does not check every field. |
|
|
66
|
+
| `flows` | Recorded explanations. An empty array means none are recorded; there is no known total. |
|
|
67
|
+
| `questions` | Optional assertion-level investigation questions. They do not establish completeness. |
|
|
68
|
+
| Flow `id` | Stable within its assertion; `assertion-id/flow-id` identifies the claim. |
|
|
69
|
+
| Flow `basis` | Null, or the opaque `scov2:<64 lowercase hex digits>` token supplied by validation. |
|
|
70
|
+
| `appliesTo` | Explicit project-relative test file and exact displayed test name. Empty means no execution credit. |
|
|
71
|
+
| `nodes`, `edges` | Exact source anchors and authored relationships ending at the reserved `$assertion` sink. |
|
|
72
|
+
| `countsAsAsserted` | The node IDs judged asserted. Context nodes earn no automatic credit. |
|
|
73
|
+
| `watch` | Additional whole-file dependencies as path strings. |
|
|
74
|
+
| Flow `questions` | Optional unresolved questions; any entry blocks this flow's credit. |
|
|
75
|
+
|
|
76
|
+
Anchors use project-relative `/` paths, one-based lines and one-based UTF-8 byte
|
|
77
|
+
columns. Preserve exact text, including multiline expressions. IDs use letters,
|
|
78
|
+
digits, `_`, `-`, or `.`. Edges have `from`, `to`, `kind`, and optional explanatory
|
|
79
|
+
`basis` text; that edge text is distinct from the flow's acknowledgement token.
|
|
80
|
+
Every counted node needs a path through the authored edges to `$assertion`.
|
|
81
|
+
The checker traverses this JSON graph only; it does not infer source dependencies.
|
|
82
|
+
A counted node must match one measured statement exactly to earn credit. A block
|
|
83
|
+
or guard does not implicitly credit its nested body.
|
|
84
|
+
|
|
85
|
+
A zero-credit explanation is useful for a fixture-only check or an absence check:
|
|
86
|
+
explain the observation and use `countsAsAsserted: []`. An absent event cannot
|
|
87
|
+
credit an unexecuted body. An executed guard can receive credit only when the
|
|
88
|
+
author judges that the predicate observes its behavior.
|
|
89
|
+
|
|
90
|
+
## Percentage and report states
|
|
91
|
+
|
|
92
|
+
The regular `supercov runs <run>` report reads the map on demand. Its primary
|
|
93
|
+
percentage is the union of eligible, explicitly counted measured statements,
|
|
94
|
+
divided by **all measured statements** in the run. Unexecuted and unanchored
|
|
95
|
+
measured statements remain in the denominator. Duplicate claims count once.
|
|
96
|
+
Line credit requires all measured statements on that line to be credited.
|
|
97
|
+
|
|
98
|
+
Eligibility requires valid references, current input acknowledgement, no open
|
|
99
|
+
flow questions, a passing run, an exact passing assertion occurrence, and
|
|
100
|
+
statement execution attributed to the same selected passing test. File + name
|
|
101
|
+
selectors must resolve unambiguously. A missing or unobserved selector contributes
|
|
102
|
+
nothing; an observed sibling selector can still contribute. Coexecution supports
|
|
103
|
+
the agent's claim but does not establish causality or temporal ordering.
|
|
104
|
+
|
|
105
|
+
| `summary.status` | Public percentage |
|
|
106
|
+
| --- | --- |
|
|
107
|
+
| `available` | Numeric, including a meaningful 0 when a current observed explanation credits nothing. |
|
|
108
|
+
| `notAssessed` | Null: no flow explanations have been recorded. |
|
|
109
|
+
| `pending` | Null: change impact remains outstanding, or no eligible current explanation exists. |
|
|
110
|
+
| `unavailable` | Null or an unavailable report: failed run, invalid identities, mismatching source or unreadable map. |
|
|
111
|
+
| `notApplicable` | Null: no measured statements exist. |
|
|
112
|
+
|
|
113
|
+
For example:
|
|
114
|
+
|
|
115
|
+
```text
|
|
116
|
+
Assertions 62.50% (50/80) — agent-assessed statements, whole run
|
|
117
|
+
18 assertions with flows; 7 without; 29 current flows; 3 stale; 2 draft
|
|
118
|
+
Assertions not assessed — No recorded flow explanations
|
|
119
|
+
Assertions pending — Source changes need impact assessment
|
|
120
|
+
```
|
|
121
|
+
|
|
122
|
+
A partial map can produce a useful numeric score. Counts show assertions with
|
|
123
|
+
and without flows, current/draft/stale/invalid flows, unobserved assertions,
|
|
124
|
+
questions and pending changes. There is no `mapped` or `complete` status. The
|
|
125
|
+
number of missing semantic flows is unknown. `current` means the claim has no
|
|
126
|
+
freshness/reference/question blockers; selector evidence is reported separately.
|
|
127
|
+
Diagnostic statement counts remain inspectable while change impact is pending;
|
|
128
|
+
the public percentage stays null.
|
|
129
|
+
|
|
130
|
+
JSON regular reports expose this under `data.assertionCoverage.summary`; detailed
|
|
131
|
+
assertion reports use `data.summary`. `assertionCoverage.available` indicates
|
|
132
|
+
whether the report could be read, while `summary.status` describes whether its
|
|
133
|
+
percentage is available. Structural filters do not change the assertion score:
|
|
134
|
+
it describes the whole run, as its label says. MC/DC remains a separate metric.
|
|
135
|
+
`revision` binds each response to the map, managed state and run evidence.
|
|
136
|
+
|
|
137
|
+
## Resource commands
|
|
138
|
+
|
|
139
|
+
| Command | Shows |
|
|
140
|
+
| --- | --- |
|
|
141
|
+
| `runs <run> assertions` | Recognized and authored assertions, including sites missing from the map. |
|
|
142
|
+
| `runs <run> assertions --needs-attention` | Assertions without flows, with open questions, or with flows needing investigation/evidence. |
|
|
143
|
+
| `runs <run> assertion <id>` | One exact assertion and its graph, freshness, selectors and evidence. |
|
|
144
|
+
| `runs <run> source <path>` | Matching current source, printed as code with line numbers. |
|
|
145
|
+
| `runs <run> assertions files` | Input paths, byte sizes and SHA-256 hashes; works even with a stale checkout. |
|
|
146
|
+
| `runs <run> assertions report --view <view>` | `summary`, `assertions`, `statements`, `tests`, `changes`, `creditedLines`, or `unassertedLines`. |
|
|
147
|
+
| `runs <run> assertions validate --json` | Shape/reference errors, changes and flow `expectedBasis` tokens. Read-only. |
|
|
148
|
+
| `runs <run> assertions check` | Read-only policy gates. |
|
|
149
|
+
|
|
150
|
+
`--file <path>` filters assertion/statement/line items, never the summary.
|
|
151
|
+
List views support `--offset <n> --limit <1..1000>` and `--json`. Follow
|
|
152
|
+
`pagination.nextOffset`; restart a multi-page read if `revision` changes.
|
|
153
|
+
Source defaults to 20 lines and prints `34 │ code here`. Source JSON uses
|
|
154
|
+
`{line, text}` items for integrations. It is usable with malformed map JSON but
|
|
155
|
+
requires current files matching the run. There is no separate inventory command.
|
|
156
|
+
|
|
157
|
+
Large validation responses can be paged with `validate --view flows|changes|errors`
|
|
158
|
+
and `--offset`/`--limit`. Entries appear under `items`; overall `valid` and
|
|
159
|
+
`errorCount` still cover the whole map. Default validation returns all entries.
|
|
160
|
+
|
|
161
|
+
`validate` returns exit 2 for syntax/reference errors, not merely null/stale
|
|
162
|
+
acknowledgements. `check` returns exit 2 for invalid references, failed runs,
|
|
163
|
+
unacknowledged/stale/invalid flows, open questions, pending impact assessments,
|
|
164
|
+
or an unverifiable checkout. An incremental map with untouched empty sites is
|
|
165
|
+
allowed by basic `check`. Optional gates:
|
|
166
|
+
|
|
167
|
+
- `--require-mappings`: every recognized assertion observed passing has at least
|
|
168
|
+
one eligible current explanation. A zero-credit explanation qualifies. This
|
|
169
|
+
is not a claim that every flow is known.
|
|
170
|
+
- `--require-observed`: every listed assertion and every explicit selector has
|
|
171
|
+
matching passing evidence. Empty selectors fail this gate.
|
|
172
|
+
- `--min <0..100>`: the public statement percentage must be numeric and meet the target.
|
|
173
|
+
|
|
174
|
+
## Reuse after source or test changes
|
|
175
|
+
|
|
176
|
+
Run the same test command again. Before publication, Supercov selects the newest
|
|
177
|
+
usable map for that exact command and language. A malformed newer candidate is
|
|
178
|
+
reported under `inheritance.skipped`; fallback claims require fresh acknowledgement.
|
|
179
|
+
Publication writes evidence, map and managed state together atomically. Previous
|
|
180
|
+
runs stay untouched. Finish editing the newest map before starting its successor.
|
|
181
|
+
|
|
182
|
+
The run stores a hash manifest and assertion identities, not complete source
|
|
183
|
+
files. Current project files supply the source for investigation. Managed
|
|
184
|
+
`assertions.state.json` (version 3) binds evidence and input identities and stores
|
|
185
|
+
invalidation generations, outstanding change records and inheritance metadata.
|
|
186
|
+
It contains no second semantic graph. Only `assertions.json` is agent-editable.
|
|
187
|
+
No query changes either file.
|
|
188
|
+
|
|
189
|
+
A flow depends on the assertion file, each selected test file, every node file,
|
|
190
|
+
and extra `watch` files. Any byte change in those files, including comments or
|
|
191
|
+
blank lines, invalidates that flow. Other sibling flows can remain current.
|
|
192
|
+
Changes to the assertion/observation affect all its flows. Context/dependency/
|
|
193
|
+
instrumenter changes invalidate inherited claims conservatively. Flow tokens
|
|
194
|
+
bind the claim (excluding their own token), context, sorted dependency hashes,
|
|
195
|
+
and effective invalidation generation; they exclude run IDs and runtime events.
|
|
196
|
+
|
|
197
|
+
Exact identities retain IDs. Unique snippets or unique identical-file renames
|
|
198
|
+
can relocate as suggestions. A sole unmatched old/new assertion in the same file
|
|
199
|
+
can retain its ID as a changed candidate. Ambiguous/removed assertions remain
|
|
200
|
+
under `retiredAssertions` with explanations. Unresolved changes stay stale across
|
|
201
|
+
reruns and reverts until acknowledged. Every new run supplies fresh evidence.
|
|
202
|
+
|
|
203
|
+
Every added, edited or removed file in the captured analysis scope enters the
|
|
204
|
+
change queue, even if a flow already watches it. Known dependencies are a starting
|
|
205
|
+
point, not proof that other flows are unaffected. Read `--view changes` and edit:
|
|
206
|
+
|
|
207
|
+
```json
|
|
208
|
+
{
|
|
209
|
+
"changeAssessments": [{
|
|
210
|
+
"id": "c_copy_from_changes_view",
|
|
211
|
+
"basis": null,
|
|
212
|
+
"affectedFlows": ["a_example/return-value"],
|
|
213
|
+
"explanation": "The edit affects the returned value; the independent sibling computation is unchanged."
|
|
214
|
+
}]
|
|
215
|
+
}
|
|
216
|
+
```
|
|
217
|
+
|
|
218
|
+
This is an excerpt. Include all `knownFlows` that still exist, plus any additional
|
|
219
|
+
claims affected by the change. Repair missing watches when appropriate. An empty
|
|
220
|
+
list needs an explanation of why existing claims are unaffected; it does not
|
|
221
|
+
mean the changed code is tested. Save responses, validate, copy examined change
|
|
222
|
+
tokens, save, then validate again for final flow tokens. Impact assignment can
|
|
223
|
+
invalidate another flow, so this order matters. Tokens have no circular dependency.
|
|
224
|
+
Deleting a response does not clear its managed change record.
|
|
225
|
+
|
|
226
|
+
On the next run, resolved changes are folded into per-flow generation baselines
|
|
227
|
+
and retired. Unchanged equivalent runs preserve current tokens. Pending records
|
|
228
|
+
survive and their responses must match the current input manifest. The scope is
|
|
229
|
+
finite: uncaptured external state is not automatically monitored.
|
|
230
|
+
|
|
231
|
+
Version-1 maps remain importable during carry; graphs and IDs are preserved,
|
|
232
|
+
span watches become file watches, and all flow tokens start null. Former test
|
|
233
|
+
names are retained in investigation questions until file-qualified selectors are
|
|
234
|
+
provided. Missing sink edges are not invented. Legacy evidence may contain full
|
|
235
|
+
source, used only to recover hashes. No migration writes into the old run.
|
|
236
|
+
|
|
237
|
+
## Limits and validation
|
|
238
|
+
|
|
239
|
+
The JS/TS verification matrix covers Node ESM/CommonJS/native TypeScript, Vitest
|
|
240
|
+
TypeScript, Jest CommonJS and Playwright's Node-side TypeScript assertions,
|
|
241
|
+
including aliases, shared parameterized sites and async rejection matchers.
|
|
242
|
+
Unsupported/custom sites can be documented, but JS/TS credit requires the exact
|
|
243
|
+
recognized expression and operation in runtime evidence. This does not infer
|
|
244
|
+
browser assertion identity or data flows through external services.
|
|
245
|
+
|
|
246
|
+
The map is an agent assessment. A credited statement can be changed without
|
|
247
|
+
failing a test when the change preserves the observed property. Neither 100%
|
|
248
|
+
assertion coverage nor 100% MC/DC proves mutation resistance or safety for every
|
|
249
|
+
change. Use the observation and graph to select tests and identify missing
|
|
250
|
+
checks; sample mutation audits can independently evaluate the authored claims.
|
|
251
|
+
See [the agent workflow](assertion-agent.md), [assertion evidence](assertion-evidence.md),
|
|
252
|
+
and [verification](verification.md).
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
# Understanding assertion coverage
|
|
2
|
+
|
|
3
|
+
After a normal test run, a coding agent can map what each JavaScript or
|
|
4
|
+
TypeScript assertion checks into the run's `assertions.json`. Supercov reads
|
|
5
|
+
that map alongside the recorded execution evidence and displays an assertion
|
|
6
|
+
percentage in the regular coverage report.
|
|
7
|
+
|
|
8
|
+
```sh supercov
|
|
9
|
+
npx supercov -- npm test
|
|
10
|
+
npx supercov runs latest assertions --json
|
|
11
|
+
```
|
|
12
|
+
|
|
13
|
+
The run creates the map automatically, reusing the newest available map for the
|
|
14
|
+
same test command and language. Pin the returned run ID and edit `data.map`.
|
|
15
|
+
Ask your agent to follow
|
|
16
|
+
`supercov docs assertion-agent`, read the current project source, and complete the map.
|
|
17
|
+
Then validate and save the acknowledgement tokens for examined flows:
|
|
18
|
+
|
|
19
|
+
```sh supercov-example
|
|
20
|
+
npx supercov runs <run> assertions validate --json
|
|
21
|
+
# Copy examined expectedBasis tokens into assertions.json; save again.
|
|
22
|
+
npx supercov runs <run> assertions check --require-mappings --require-observed --json
|
|
23
|
+
npx supercov runs <run>
|
|
24
|
+
```
|
|
25
|
+
|
|
26
|
+
The **Assertions** row appears beside Lines, Branches and MC/DC. It is the
|
|
27
|
+
percentage of measured statements credited by current agent-authored flows,
|
|
28
|
+
with exact passing assertion identity and execution in the same test. It uses
|
|
29
|
+
the whole run with matching current source even when structural coverage is filtered. Incomplete
|
|
30
|
+
maps remain useful; unmapped or dirty flows earn no credit.
|
|
31
|
+
|
|
32
|
+
## Check what the test asserts
|
|
33
|
+
|
|
34
|
+
Suppose `src/shipping.js` contains:
|
|
35
|
+
|
|
36
|
+
```js
|
|
37
|
+
export function shippingCost() {
|
|
38
|
+
return 4;
|
|
39
|
+
}
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
These checks all call the function, but they test different things. Here,
|
|
43
|
+
`assert` comes from `node:assert/strict`:
|
|
44
|
+
|
|
45
|
+
| Assertion | What it checks |
|
|
46
|
+
| --- | --- |
|
|
47
|
+
| `assert.equal(shippingCost(), 4)` | The cost is exactly `4`. Returning `5` would fail. |
|
|
48
|
+
| `assert.ok(shippingCost())` | The cost is truthy. Returning `5` would still pass. |
|
|
49
|
+
| `shippingCost(); assert.equal(7, 7)` | Nothing about the returned cost. The assertion compares two constants. |
|
|
50
|
+
|
|
51
|
+
Line coverage can be the same in all three cases. The useful test is the one
|
|
52
|
+
that checks the behavior you need to preserve.
|
|
53
|
+
|
|
54
|
+
The same applies to other results: checking the number of log calls does not
|
|
55
|
+
check their messages, and checking one substring does not check the whole response.
|
|
56
|
+
|
|
57
|
+
## Inspect and improve the map
|
|
58
|
+
|
|
59
|
+
```sh supercov-example
|
|
60
|
+
npx supercov runs <run> assertions --limit 5
|
|
61
|
+
npx supercov runs <run> assertion <assertion-id>
|
|
62
|
+
npx supercov runs <run> source src/shipping.js
|
|
63
|
+
npx supercov runs <run> assertions --view statements --file src/shipping.js --limit 20
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
Each assertion keeps its exact test source anchor, the behavior it observes,
|
|
67
|
+
and flows to explicit production statements. Follow those links when deciding
|
|
68
|
+
whether a test needs another scenario or a stronger assertion. The model owns
|
|
69
|
+
the semantic assessment; Supercov validates syntax, source references, evidence
|
|
70
|
+
and freshness without reconstructing the reasoning.
|
|
71
|
+
|
|
72
|
+
After changing code or tests, run the same suite command again. Its new map
|
|
73
|
+
automatically carries forward unchanged entries;
|
|
74
|
+
changed or ambiguous references stay dirty until reviewed. Source files are
|
|
75
|
+
read from the project; each run retains file hashes and the map, without storing
|
|
76
|
+
complete source copies. A stale checkout makes assertion coverage unavailable
|
|
77
|
+
until tests are rerun. See the
|
|
78
|
+
[map reference](assertion-maps.md) for the format, commands and limitations.
|
|
79
|
+
|
|
80
|
+
MC/DC measures independent condition effects. Assertion coverage measures the
|
|
81
|
+
statements linked to checks by the reviewed map. Maximizing either number does
|
|
82
|
+
not prove every edit will fail a test: the exact property in `observes` matters.
|
package/docs/cli.md
CHANGED
|
@@ -4,7 +4,7 @@ Supercov has one command for measuring a suite and a small set of commands for
|
|
|
4
4
|
reading the result. Text output is designed for people and coding agents. Add
|
|
5
5
|
`--json` only when an integration needs a stable machine-readable response.
|
|
6
6
|
|
|
7
|
-
```sh
|
|
7
|
+
```sh supercov
|
|
8
8
|
npx supercov --help
|
|
9
9
|
```
|
|
10
10
|
|
|
@@ -17,7 +17,9 @@ npx supercov --help
|
|
|
17
17
|
| Read the newest run | `npx supercov runs latest` |
|
|
18
18
|
| Find useful gaps | `npx supercov runs latest gaps` |
|
|
19
19
|
| Inspect one file | `npx supercov runs latest file <path>` |
|
|
20
|
-
|
|
|
20
|
+
| List assertions and their status | `npx supercov runs latest assertions` |
|
|
21
|
+
| Inspect one assertion and its flows | `npx supercov runs latest assertion <id>` |
|
|
22
|
+
| Read matching current source code | `npx supercov runs latest source <path>` |
|
|
21
23
|
| Compare two runs | `npx supercov diff <older> <newer>` |
|
|
22
24
|
| Combine shards | `npx supercov merge <id> <id> [...]` |
|
|
23
25
|
| Remove local data | `npx supercov clean` |
|
|
@@ -25,13 +27,13 @@ npx supercov --help
|
|
|
25
27
|
|
|
26
28
|
## Measure a test command
|
|
27
29
|
|
|
28
|
-
```sh
|
|
30
|
+
```sh supercov
|
|
29
31
|
npx supercov -- <test command>
|
|
30
32
|
```
|
|
31
33
|
|
|
32
34
|
Everything after `--` is passed to the test command:
|
|
33
35
|
|
|
34
|
-
```sh
|
|
36
|
+
```sh supercov
|
|
35
37
|
npx supercov -- npm test
|
|
36
38
|
npx supercov -- npx playwright test --project=chromium
|
|
37
39
|
npx supercov -- cargo test
|
|
@@ -43,7 +45,7 @@ preserves the wrapped command's exit status, so it can remain a CI gate.
|
|
|
43
45
|
|
|
44
46
|
## List and select runs
|
|
45
47
|
|
|
46
|
-
```sh
|
|
48
|
+
```sh supercov
|
|
47
49
|
npx supercov runs
|
|
48
50
|
npx supercov runs --limit 5
|
|
49
51
|
npx supercov runs latest
|
|
@@ -56,7 +58,7 @@ sessions.
|
|
|
56
58
|
|
|
57
59
|
## Query a run
|
|
58
60
|
|
|
59
|
-
```sh
|
|
61
|
+
```sh supercov
|
|
60
62
|
npx supercov runs <run-id> [query] [options]
|
|
61
63
|
```
|
|
62
64
|
|
|
@@ -72,12 +74,14 @@ npx supercov runs <run-id> [query] [options]
|
|
|
72
74
|
| `kinds` | Group coverage by test level, such as unit or E2E |
|
|
73
75
|
| `runners` | Group coverage by test runner |
|
|
74
76
|
| `scope` | Review included, excluded, and ambiguous source files |
|
|
75
|
-
| `assertions` |
|
|
77
|
+
| `assertions` | List assertions, including sites without flows, with freshness and execution status |
|
|
78
|
+
| `assertion <id>` | Inspect one assertion and its authored flows |
|
|
79
|
+
| `source <path>` | Read matching current project source with line numbers |
|
|
76
80
|
| `minimize` | Find a small test subset that preserves a coverage target |
|
|
77
81
|
|
|
78
82
|
Common examples:
|
|
79
83
|
|
|
80
|
-
```sh
|
|
84
|
+
```sh supercov
|
|
81
85
|
npx supercov runs latest gaps --limit 10
|
|
82
86
|
npx supercov runs latest file app/routes/checkout.ts
|
|
83
87
|
npx supercov runs latest decision app/routes/checkout.ts:42
|
|
@@ -87,18 +91,29 @@ npx supercov runs latest test "checkout retry"
|
|
|
87
91
|
|
|
88
92
|
Run any query with `--help` to see only the options valid for that query:
|
|
89
93
|
|
|
90
|
-
```sh
|
|
94
|
+
```sh supercov
|
|
91
95
|
npx supercov runs latest --help
|
|
92
96
|
npx supercov runs latest file --help
|
|
93
97
|
npx supercov runs latest assertions --help
|
|
94
98
|
```
|
|
95
99
|
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
`
|
|
100
|
+
Assertion queries read the run-owned map. `source <path>` reads the matching current
|
|
101
|
+
file directly. It prints source code with line numbers, preserving indentation;
|
|
102
|
+
add `--json` only when you want structured `{line, text}` items. `--offset` is
|
|
103
|
+
zero-based and `--limit` controls the number of source lines. Source and assertion analysis require the current
|
|
104
|
+
checkout to match the run. Changed source makes assertion coverage unavailable;
|
|
105
|
+
rerun tests to inherit mappings. The old `assertions inventory` and nested
|
|
106
|
+
`assertions source --file` commands are replaced by these resource queries.
|
|
107
|
+
Use read-only `assertions validate --json` to obtain expected acknowledgement tokens.
|
|
108
|
+
After investigating, copy those tokens into `assertions.json`, save, and run
|
|
109
|
+
`assertions check`. The `review` command is removed. `--require-mappings` requires
|
|
110
|
+
a current explanation per recognized passing site, without implying completeness.
|
|
111
|
+
Use `assertions --needs-attention` and `assertions report --view changes` to resume work.
|
|
112
|
+
The regular run summary automatically includes the assertion percentage when
|
|
113
|
+
that map exists, in text and `data.assertionCoverage` JSON. Read
|
|
114
|
+
[Understanding assertion coverage](assertions.md) for the workflow or run
|
|
115
|
+
`npx supercov docs assertion-agent` for instructions to give a coding agent.
|
|
116
|
+
The [map reference](assertion-maps.md) covers syntax and verification gates.
|
|
102
117
|
|
|
103
118
|
## Narrow a view
|
|
104
119
|
|
|
@@ -115,14 +130,14 @@ Collection output includes a copyable command for the next page.
|
|
|
115
130
|
|
|
116
131
|
For a large file, group and rank its decisions:
|
|
117
132
|
|
|
118
|
-
```sh
|
|
133
|
+
```sh supercov
|
|
119
134
|
npx supercov runs latest file app/routes/checkout.ts \
|
|
120
135
|
--group decision --sort missing
|
|
121
136
|
```
|
|
122
137
|
|
|
123
138
|
## Compare runs
|
|
124
139
|
|
|
125
|
-
```sh
|
|
140
|
+
```sh supercov
|
|
126
141
|
npx supercov diff <older-run> <newer-run>
|
|
127
142
|
```
|
|
128
143
|
|
|
@@ -132,13 +147,13 @@ Neither input run is changed.
|
|
|
132
147
|
|
|
133
148
|
The same filters can focus a comparison:
|
|
134
149
|
|
|
135
|
-
```sh
|
|
150
|
+
```sh supercov
|
|
136
151
|
npx supercov diff <older-run> <newer-run> --kind e2e
|
|
137
152
|
```
|
|
138
153
|
|
|
139
154
|
## Find a smaller test set
|
|
140
155
|
|
|
141
|
-
```sh
|
|
156
|
+
```sh supercov
|
|
142
157
|
npx supercov runs latest minimize
|
|
143
158
|
npx supercov runs latest minimize --metric branches --target 90
|
|
144
159
|
```
|
|
@@ -150,7 +165,7 @@ selected metric.
|
|
|
150
165
|
|
|
151
166
|
## Combine shards
|
|
152
167
|
|
|
153
|
-
```sh
|
|
168
|
+
```sh supercov
|
|
154
169
|
npx supercov merge <shard-a> <shard-b> <shard-c>
|
|
155
170
|
```
|
|
156
171
|
|
|
@@ -160,7 +175,7 @@ incompatible merge rather than publishing a misleading aggregate.
|
|
|
160
175
|
|
|
161
176
|
## Clean local data
|
|
162
177
|
|
|
163
|
-
```sh
|
|
178
|
+
```sh supercov
|
|
164
179
|
npx supercov clean --dry-run
|
|
165
180
|
npx supercov clean --keep 20
|
|
166
181
|
npx supercov clean
|
|
@@ -172,7 +187,7 @@ storage.
|
|
|
172
187
|
|
|
173
188
|
## Read bundled documentation
|
|
174
189
|
|
|
175
|
-
```sh
|
|
190
|
+
```sh supercov
|
|
176
191
|
npx supercov docs
|
|
177
192
|
npx supercov docs getting-started
|
|
178
193
|
npx supercov docs troubleshooting
|
|
@@ -190,7 +205,7 @@ terminal or offline environment after the package has been downloaded.
|
|
|
190
205
|
|
|
191
206
|
Examples:
|
|
192
207
|
|
|
193
|
-
```sh
|
|
208
|
+
```sh supercov
|
|
194
209
|
SUPERCOV_SOURCE_ROOTS=src,app npx supercov -- npm test
|
|
195
210
|
SUPERCOV_TEST_KIND=e2e npx supercov -- npx playwright test
|
|
196
211
|
```
|
package/docs/coverage-model.md
CHANGED
|
@@ -17,7 +17,7 @@ runner boundary.
|
|
|
17
17
|
Supercov keeps those states separate. It does not turn “unknown” into
|
|
18
18
|
“uncovered,” and it does not round either one away to produce a reassuring 100%.
|
|
19
19
|
|
|
20
|
-
```sh
|
|
20
|
+
```sh supercov
|
|
21
21
|
npx supercov runs latest
|
|
22
22
|
npx supercov runs latest gaps
|
|
23
23
|
npx supercov runs latest scope
|
|
@@ -39,7 +39,7 @@ The exact obligations depend on the language and source construct. You do not
|
|
|
39
39
|
need to reason about all of them at once. Start with a file, then open a decision
|
|
40
40
|
or line only when the missing behavior needs explanation:
|
|
41
41
|
|
|
42
|
-
```sh
|
|
42
|
+
```sh supercov
|
|
43
43
|
npx supercov runs latest file app/checkout/session.ts
|
|
44
44
|
npx supercov runs latest decision app/checkout/session.ts:64
|
|
45
45
|
npx supercov runs latest line app/checkout/session.ts:64
|
|
@@ -71,6 +71,9 @@ does not mean the product has no bugs, the assertions are meaningful, or every
|
|
|
71
71
|
possible input was tested. Review test quality and user-visible behavior, not
|
|
72
72
|
only the percentage.
|
|
73
73
|
|
|
74
|
+
To review what your JavaScript and TypeScript tests actually check, see
|
|
75
|
+
[Understanding assertions](assertions.md).
|
|
76
|
+
|
|
74
77
|
If source cannot be measured safely, Supercov reports a measurement limit
|
|
75
78
|
instead of claiming completeness.
|
|
76
79
|
|
|
@@ -92,7 +95,7 @@ each runner.
|
|
|
92
95
|
|
|
93
96
|
The same stored run can answer different questions:
|
|
94
97
|
|
|
95
|
-
```sh
|
|
98
|
+
```sh supercov
|
|
96
99
|
npx supercov runs latest --filter all
|
|
97
100
|
npx supercov runs latest --filter passed
|
|
98
101
|
npx supercov runs latest --filter failed
|
|
@@ -103,7 +106,7 @@ attempts. `failed` isolates failed attempts, including failed retries.
|
|
|
103
106
|
|
|
104
107
|
You can also focus on a test level or runner:
|
|
105
108
|
|
|
106
|
-
```sh
|
|
109
|
+
```sh supercov
|
|
107
110
|
npx supercov runs latest gaps --kind e2e
|
|
108
111
|
npx supercov runs latest gaps --runner playwright
|
|
109
112
|
```
|
|
@@ -115,13 +118,13 @@ percentage that was computed from a different set of tests.
|
|
|
115
118
|
|
|
116
119
|
If the summary reports ambiguous source scope, inspect it:
|
|
117
120
|
|
|
118
|
-
```sh
|
|
121
|
+
```sh supercov
|
|
119
122
|
npx supercov runs latest scope
|
|
120
123
|
```
|
|
121
124
|
|
|
122
125
|
When first-party source lives in unusual directories, declare it explicitly:
|
|
123
126
|
|
|
124
|
-
```sh
|
|
127
|
+
```sh supercov
|
|
125
128
|
SUPERCOV_SOURCE_ROOTS=src,app npx supercov -- npm test
|
|
126
129
|
```
|
|
127
130
|
|
|
@@ -138,3 +141,12 @@ the bundler consumes them at build time; nothing about them runs.
|
|
|
138
141
|
|
|
139
142
|
Choose roots that describe code the repository owns. Do not include dependencies
|
|
140
143
|
or generated output merely to make a warning disappear.
|
|
144
|
+
|
|
145
|
+
## Assertion percentage
|
|
146
|
+
|
|
147
|
+
When a run has an `assertions.json` map, its regular summary also shows
|
|
148
|
+
agent-assessed assertion coverage: credited measured statements divided by all
|
|
149
|
+
measured statements. This row always describes the whole archived run,
|
|
150
|
+
independently of `--filter`, `--kind` and `--runner`. It is separate from the
|
|
151
|
+
structural metrics above. See [assertion maps](assertion-maps.md) for credit,
|
|
152
|
+
freshness and incomplete-map rules.
|
package/docs/evidence.md
CHANGED
|
@@ -6,7 +6,7 @@ without rerunning the tests.
|
|
|
6
6
|
|
|
7
7
|
## Find the run you want
|
|
8
8
|
|
|
9
|
-
```sh
|
|
9
|
+
```sh supercov
|
|
10
10
|
npx supercov runs
|
|
11
11
|
npx supercov runs --limit 10
|
|
12
12
|
npx supercov runs latest
|
|
@@ -23,7 +23,7 @@ A run id is immutable. `latest` is only a convenient selector.
|
|
|
23
23
|
|
|
24
24
|
## Ask the same run different questions
|
|
25
25
|
|
|
26
|
-
```sh
|
|
26
|
+
```sh supercov
|
|
27
27
|
npx supercov runs latest gaps --limit 10
|
|
28
28
|
npx supercov runs latest file app/checkout/session.ts
|
|
29
29
|
npx supercov runs latest line app/checkout/session.ts:64
|
|
@@ -45,7 +45,7 @@ new work from it.
|
|
|
45
45
|
|
|
46
46
|
## Focus on passed or failed attempts
|
|
47
47
|
|
|
48
|
-
```sh
|
|
48
|
+
```sh supercov
|
|
49
49
|
npx supercov runs latest --filter all
|
|
50
50
|
npx supercov runs latest --filter passed
|
|
51
51
|
npx supercov runs latest --filter failed
|
|
@@ -60,7 +60,7 @@ reports that can drift apart.
|
|
|
60
60
|
|
|
61
61
|
## Compare before and after
|
|
62
62
|
|
|
63
|
-
```sh
|
|
63
|
+
```sh supercov
|
|
64
64
|
npx supercov diff <older-run> <newer-run>
|
|
65
65
|
```
|
|
66
66
|
|
|
@@ -76,7 +76,7 @@ need to reproduce the result.
|
|
|
76
76
|
|
|
77
77
|
## Combine distributed shards
|
|
78
78
|
|
|
79
|
-
```sh
|
|
79
|
+
```sh supercov
|
|
80
80
|
npx supercov merge <shard-a> <shard-b> <shard-c>
|
|
81
81
|
```
|
|
82
82
|
|
|
@@ -100,7 +100,7 @@ Completed runs live under `.supercov/runs/<run-id>/`. The isolated workspace and
|
|
|
100
100
|
instrumented build cache may use more space than the compressed run itself.
|
|
101
101
|
Nothing is pruned in the background.
|
|
102
102
|
|
|
103
|
-
```sh
|
|
103
|
+
```sh supercov
|
|
104
104
|
npx supercov clean --dry-run
|
|
105
105
|
npx supercov clean --keep 20
|
|
106
106
|
npx supercov clean
|