supercov 0.0.45 → 0.0.46
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/docs/agent-loop.md +17 -13
- package/docs/assertion-agent.md +134 -144
- package/docs/assertion-evidence.md +103 -9
- package/docs/assertion-maps.md +188 -204
- package/docs/assertions.md +99 -49
- package/docs/cli.md +31 -14
- package/docs/coverage-model.md +6 -0
- package/docs/evidence.md +28 -0
- package/docs/performance.md +16 -0
- package/docs/supported-suites.md +64 -147
- package/package.json +14 -12
- package/runtime/javascript/launchSupervisor.mjs +5 -0
- package/runtime/javascript/nodeTest.mjs +57 -41
- package/runtime/javascript/provenance.mjs +4 -1
- package/runtime/javascript/register.mjs +9 -5
- package/runtime/javascript/runnerEvidence.mjs +4 -1
- package/runtime/javascript/runtime.mjs +44 -3
package/docs/assertions.md
CHANGED
|
@@ -1,35 +1,71 @@
|
|
|
1
1
|
# Understanding assertion coverage
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
3
|
+
Assertion coverage helps you see which parts of your JavaScript or TypeScript
|
|
4
|
+
code are checked by tests. After a normal run, your coding agent records what
|
|
5
|
+
each assertion checks in `assertions.json`. Supercov combines that map with the
|
|
6
|
+
run's execution evidence and shows an **Assertions** percentage alongside Lines,
|
|
7
|
+
Branches and MC/DC.
|
|
8
|
+
|
|
9
|
+
## Start with your existing tests
|
|
7
10
|
|
|
8
11
|
```sh supercov
|
|
9
12
|
npx supercov -- npm test
|
|
10
|
-
npx supercov runs latest assertions
|
|
13
|
+
npx supercov runs latest assertions
|
|
14
|
+
```
|
|
15
|
+
|
|
16
|
+
Use your project's test command after `--`. Each run creates its own map and
|
|
17
|
+
reuses compatible work from earlier runs of the same command. The assertions
|
|
18
|
+
list shows the map's path and the assertions that still need attention.
|
|
19
|
+
|
|
20
|
+
Copy the printed run ID before asking an agent to edit the map. This keeps its
|
|
21
|
+
work on the same run even if another test run finishes. You can give it this
|
|
22
|
+
prompt, replacing `<run-id>` with that ID:
|
|
23
|
+
|
|
24
|
+
```text
|
|
25
|
+
Read the instructions from npx supercov docs assertion-agent. Build or update
|
|
26
|
+
assertion coverage for run <run-id>. Investigate the tests and current source,
|
|
27
|
+
then edit that run's assertions.json. Preserve reusable flows and explain
|
|
28
|
+
uncertainty. Do not change application code or tests. Validate the map and
|
|
29
|
+
show the assertion percentage, remaining gaps, and any missing evidence.
|
|
11
30
|
```
|
|
12
31
|
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
Then validate and save the acknowledgement tokens for examined flows:
|
|
32
|
+
You can use your usual coding agent. Supercov does not require a particular
|
|
33
|
+
model or make model calls itself.
|
|
34
|
+
|
|
35
|
+
## Read the percentage
|
|
18
36
|
|
|
19
37
|
```sh supercov-example
|
|
20
|
-
npx supercov runs <run>
|
|
21
|
-
# Copy examined expectedBasis tokens into assertions.json; save again.
|
|
22
|
-
npx supercov runs <run> assertions check --require-mappings --require-observed --json
|
|
23
|
-
npx supercov runs <run>
|
|
38
|
+
npx supercov runs <run-id>
|
|
24
39
|
```
|
|
25
40
|
|
|
26
|
-
|
|
27
|
-
percentage of measured statements credited by current agent-authored flows,
|
|
28
|
-
with exact passing assertion identity and execution in the same test. It uses
|
|
29
|
-
the whole run with matching current source even when structural coverage is filtered. Incomplete
|
|
30
|
-
maps remain useful; unmapped or dirty flows earn no credit.
|
|
41
|
+
An illustrative report might show:
|
|
31
42
|
|
|
32
|
-
|
|
43
|
+
```text
|
|
44
|
+
Assertions 62.50% (50/80) — agent-assessed statements, whole run
|
|
45
|
+
18 assertions with flows; 7 without; 29 current flows; 3 stale; 2 draft
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
Here, 50 of the run's 80 measured statements have a current explanation linking
|
|
49
|
+
them to a passing assertion, with execution recorded in the same test. Each
|
|
50
|
+
statement counts once, even when several assertions check it. Unexecuted code
|
|
51
|
+
stays in the denominator. TypeScript imports that are known to disappear during
|
|
52
|
+
compilation are excluded.
|
|
53
|
+
|
|
54
|
+
The score describes the whole run. Filtering Lines or MC/DC by test kind does
|
|
55
|
+
not change the assertion percentage. You do not need to rerun tests just to
|
|
56
|
+
recalculate a map: save it and query the run again.
|
|
57
|
+
|
|
58
|
+
| Report state | What to do |
|
|
59
|
+
| --- | --- |
|
|
60
|
+
| Not assessed | Ask your agent to start explaining the assertions. |
|
|
61
|
+
| Pending | Inspect changed inputs, draft flows or stale flows before using a percentage. |
|
|
62
|
+
| Unavailable | Check for a failed run, invalid map or source that no longer matches the run. |
|
|
63
|
+
| Not applicable | The run has no measured statements. |
|
|
64
|
+
|
|
65
|
+
A partially written map can have a useful percentage. Check the counts beside
|
|
66
|
+
it: having some flows does not mean every relevant flow has been found.
|
|
67
|
+
|
|
68
|
+
## Check what an assertion actually protects
|
|
33
69
|
|
|
34
70
|
Suppose `src/shipping.js` contains:
|
|
35
71
|
|
|
@@ -39,44 +75,58 @@ export function shippingCost() {
|
|
|
39
75
|
}
|
|
40
76
|
```
|
|
41
77
|
|
|
42
|
-
These
|
|
78
|
+
These tests all call the function, but they check different things. Here,
|
|
43
79
|
`assert` comes from `node:assert/strict`:
|
|
44
80
|
|
|
45
81
|
| Assertion | What it checks |
|
|
46
82
|
| --- | --- |
|
|
47
83
|
| `assert.equal(shippingCost(), 4)` | The cost is exactly `4`. Returning `5` would fail. |
|
|
48
84
|
| `assert.ok(shippingCost())` | The cost is truthy. Returning `5` would still pass. |
|
|
49
|
-
| `shippingCost(); assert.equal(7, 7)` | Nothing about the returned cost. The assertion compares
|
|
50
|
-
|
|
51
|
-
Line coverage can be the same in all three cases. The useful test is the one
|
|
52
|
-
that checks the behavior you need to preserve.
|
|
85
|
+
| `shippingCost(); assert.equal(7, 7)` | Nothing about the returned cost. The assertion compares constants. |
|
|
53
86
|
|
|
54
|
-
|
|
55
|
-
|
|
87
|
+
Line coverage can be the same in all three cases. Even a credited statement can
|
|
88
|
+
change without failing a test when the new value still satisfies the assertion.
|
|
89
|
+
Read the mapped observation when deciding whether a test protects your change.
|
|
56
90
|
|
|
57
|
-
## Inspect
|
|
91
|
+
## Inspect a gap
|
|
58
92
|
|
|
59
93
|
```sh supercov-example
|
|
60
|
-
npx supercov runs <run> assertions --limit 5
|
|
61
|
-
npx supercov runs <run> assertion <assertion-id>
|
|
62
|
-
npx supercov runs <run>
|
|
63
|
-
npx supercov runs <run>
|
|
94
|
+
npx supercov runs <run-id> assertions --needs-attention --limit 5
|
|
95
|
+
npx supercov runs <run-id> assertion <assertion-id>
|
|
96
|
+
npx supercov runs <run-id> assertions report --view statements --file src/shipping.js
|
|
97
|
+
npx supercov runs <run-id> source src/shipping.js
|
|
64
98
|
```
|
|
65
99
|
|
|
66
|
-
|
|
67
|
-
and
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
100
|
+
An assertion detail shows its exact location in the test, its recorded flows,
|
|
101
|
+
and why each source node is credited, not credited or context only. Use this to
|
|
102
|
+
find a missing scenario, strengthen a weak check or correct an explanation.
|
|
103
|
+
See [Investigating assertion evidence](assertion-evidence.md) when execution
|
|
104
|
+
or a passing assertion is missing.
|
|
105
|
+
|
|
106
|
+
An absence check, such as “no message was received,” can be useful without
|
|
107
|
+
crediting a message-sending statement that never ran. Its map should explain
|
|
108
|
+
what was observed, when observation ended and why that was sufficient.
|
|
109
|
+
|
|
110
|
+
## Keep the map when code changes
|
|
111
|
+
|
|
112
|
+
Run the same suite command again after editing source or tests. Supercov copies
|
|
113
|
+
reusable mappings into the new run and identifies flows that need another look.
|
|
114
|
+
One changed dependency can affect a single flow; changing the assertion itself
|
|
115
|
+
can affect all its flows. Your agent can update those parts instead of starting
|
|
116
|
+
over.
|
|
117
|
+
|
|
118
|
+
Investigation uses your current project files. A run retains the map and file
|
|
119
|
+
identities, but does not keep a complete source checkout. If the source no
|
|
120
|
+
longer matches, rerun tests before continuing the map.
|
|
121
|
+
|
|
122
|
+
## Use the score to guide changes
|
|
123
|
+
|
|
124
|
+
Use MC/DC to find missing condition cases and assertion coverage to investigate
|
|
125
|
+
what those tests check. Improve the weakest observations and the gaps relevant
|
|
126
|
+
to your change, then run the full suite again.
|
|
127
|
+
|
|
128
|
+
Neither percentage proves that every possible bug will fail a test. A high
|
|
129
|
+
score is most useful together with the recorded observations and a review of
|
|
130
|
+
the behavior you intend to preserve. See [Assertion map format](assertion-maps.md)
|
|
131
|
+
for editing and validation, or [Trusting results](verification.md) for the
|
|
132
|
+
broader test-review workflow.
|
package/docs/cli.md
CHANGED
|
@@ -100,20 +100,37 @@ npx supercov runs latest assertions --help
|
|
|
100
100
|
Assertion queries read the run-owned map. `source <path>` reads the matching current
|
|
101
101
|
file directly. It prints source code with line numbers, preserving indentation;
|
|
102
102
|
add `--json` only when you want structured `{line, text}` items. `--offset` is
|
|
103
|
-
zero-based and `--limit` controls the number of source lines. Source and assertion
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
103
|
+
zero-based and `--limit` controls the number of source lines. Source and assertion
|
|
104
|
+
investigation require current files that match the run. Rerun the suite after
|
|
105
|
+
source changes to inherit the map into a new run.
|
|
106
|
+
|
|
107
|
+
### Assertion coverage
|
|
108
|
+
|
|
109
|
+
The regular run summary includes assertion coverage when a map has been
|
|
110
|
+
assessed. JSON reports expose it under `data.assertionCoverage`. Start with
|
|
111
|
+
[Understanding assertion coverage](assertions.md), or use these commands to
|
|
112
|
+
inspect and check a map:
|
|
113
|
+
|
|
114
|
+
```sh supercov-example
|
|
115
|
+
npx supercov runs <run-id> assertions --needs-attention
|
|
116
|
+
npx supercov runs <run-id> assertion <assertion-id>
|
|
117
|
+
npx supercov runs <run-id> assertions report --view statements --file src/shipping.js
|
|
118
|
+
npx supercov runs <run-id> assertions report --view excludedStatements
|
|
119
|
+
npx supercov runs <run-id> assertions validate --json
|
|
120
|
+
npx supercov runs <run-id> assertions check --require-mappings
|
|
121
|
+
```
|
|
122
|
+
|
|
123
|
+
Edit the file shown by `assertions`. Validation returns `expectedBasis` tokens;
|
|
124
|
+
after examining a flow, save its token in the map before running `check`.
|
|
125
|
+
`--require-mappings` requires explanations for recognized assertions observed
|
|
126
|
+
passing. Add `--require-observed` when every mapped site and selector should have
|
|
127
|
+
passing evidence, or `--min <percentage>` for a chosen target.
|
|
128
|
+
|
|
129
|
+
To inspect one large flow, add `--flow <flow-id> --view nodes` or `--view edges`
|
|
130
|
+
to the assertion detail command. `--compact` omits repeated source text from the
|
|
131
|
+
report. Follow the printed next-page command or JSON `pagination.nextOffset`.
|
|
132
|
+
Validation supports `--view flows`, `--view changes` and `--view errors` for large
|
|
133
|
+
maps. The [map reference](assertion-maps.md) describes all fields and gates.
|
|
117
134
|
|
|
118
135
|
## Narrow a view
|
|
119
136
|
|
package/docs/coverage-model.md
CHANGED
|
@@ -33,6 +33,7 @@ npx supercov runs latest scope
|
|
|
33
33
|
| Branch | Did each alternative execute? |
|
|
34
34
|
| Decision vector | Which combinations of boolean conditions occurred? |
|
|
35
35
|
| MC/DC witness | Was each condition shown to affect the decision independently? |
|
|
36
|
+
| Assertion coverage | Which measured statements are linked to passing checks by the agent-authored map? |
|
|
36
37
|
| Value path | Did defaults, optional chains, logical assignments, and similar constructs take each meaningful path? |
|
|
37
38
|
|
|
38
39
|
The exact obligations depend on the language and source construct. You do not
|
|
@@ -71,6 +72,11 @@ does not mean the product has no bugs, the assertions are meaningful, or every
|
|
|
71
72
|
possible input was tested. Review test quality and user-visible behavior, not
|
|
72
73
|
only the percentage.
|
|
73
74
|
|
|
75
|
+
TypeScript imports known to disappear during compilation do not add runtime
|
|
76
|
+
statement obligations. The assertion report's `excludedStatements` view lists
|
|
77
|
+
these locations. Imports that still execute, including side-effect imports,
|
|
78
|
+
remain in the denominator.
|
|
79
|
+
|
|
74
80
|
To review what your JavaScript and TypeScript tests actually check, see
|
|
75
81
|
[Understanding assertions](assertions.md).
|
|
76
82
|
|
package/docs/evidence.md
CHANGED
|
@@ -109,3 +109,31 @@ npx supercov clean
|
|
|
109
109
|
Preview cleanup first. The final command removes all runs and the isolated build
|
|
110
110
|
cache; `--keep 20` preserves the 20 newest runs. Cleanup removes only
|
|
111
111
|
marker-owned Supercov data.
|
|
112
|
+
|
|
113
|
+
## Understand test kinds
|
|
114
|
+
|
|
115
|
+
Reports group tests by kind, such as unit, integration or E2E. Supercov uses
|
|
116
|
+
recognized file names and runner information; a kind is a classification, not
|
|
117
|
+
proof of what the test checks. A name such as `gatewayE2e.test.ts` identifies an
|
|
118
|
+
E2E test. Ambiguous names keep the runner's default, and the report tells you
|
|
119
|
+
how many tests use that default.
|
|
120
|
+
|
|
121
|
+
If your suite has a known kind, set it when running the command:
|
|
122
|
+
|
|
123
|
+
```sh supercov-example
|
|
124
|
+
SUPERCOV_TEST_KIND=integration npx supercov -- npm test
|
|
125
|
+
```
|
|
126
|
+
|
|
127
|
+
Supported values are `unit`, `component`, `integration` and `e2e`. Use separate
|
|
128
|
+
runs when different suites need different classifications.
|
|
129
|
+
|
|
130
|
+
## When assertion evidence is missing
|
|
131
|
+
|
|
132
|
+
A test can make assertions without executing measured application code. It
|
|
133
|
+
might check static data or a dependency, or use work performed in shared setup.
|
|
134
|
+
Missing attribution across an asynchronous boundary can also leave a gap.
|
|
135
|
+
|
|
136
|
+
The assertion detail explains why a mapped node did or did not receive credit.
|
|
137
|
+
Use [Investigating assertion evidence](assertion-evidence.md) to distinguish
|
|
138
|
+
these cases. The report's runtime action-phase counts are separate from the
|
|
139
|
+
assertion map; zero action-phase lines does not mean zero asserted statements.
|
package/docs/performance.md
CHANGED
|
@@ -82,3 +82,19 @@ npx supercov clean
|
|
|
82
82
|
Use `--dry-run` to preview cleanup. Keep enough run history for active reviews
|
|
83
83
|
and automation; remove the cache only when reclaiming space matters more than a
|
|
84
84
|
faster next run.
|
|
85
|
+
|
|
86
|
+
## Keep assertion investigation fast
|
|
87
|
+
|
|
88
|
+
Save `assertions.json` and query the run to see the updated score. Supercov does
|
|
89
|
+
not rerun tests or call a model to calculate the report. Repeated queries reuse
|
|
90
|
+
the previous assessment while checking that your source still matches the run.
|
|
91
|
+
Editing the map automatically refreshes that assessment.
|
|
92
|
+
|
|
93
|
+
For large maps, ask for a short page instead of the whole graph. Within one
|
|
94
|
+
flow, `--view nodes` or `--view edges` pages the details; `--compact` omits repeated
|
|
95
|
+
source text while keeping locations and credit reasons. See
|
|
96
|
+
[Investigating assertion evidence](assertion-evidence.md#read-a-large-flow).
|
|
97
|
+
|
|
98
|
+
The disposable `assertions.report.cache.json` file lives beside the map. A missing
|
|
99
|
+
or damaged cache is rebuilt. Removing it affects the next query's speed, not
|
|
100
|
+
your saved explanations.
|
package/docs/supported-suites.md
CHANGED
|
@@ -45,7 +45,7 @@ Supercov reports the level it actually observed. It does not guess.
|
|
|
45
45
|
| --- | --- |
|
|
46
46
|
| Playwright | Exact per test, worker, retry, outcome, action, and assertion phase |
|
|
47
47
|
| Vitest | Exact per test, with setup execution kept separate |
|
|
48
|
-
| Jest | Exact per test, including parameterized tests, with the user's own configuration, setup files and reporters kept; `expect`
|
|
48
|
+
| Jest | Exact per test, including parameterized tests, with the user's own configuration, setup files and reporters kept; passing `expect` occurrences are identified for assertion maps |
|
|
49
49
|
| `node:test` | Exact per test |
|
|
50
50
|
| AVA and Mocha | Aggregate structural coverage |
|
|
51
51
|
| Other Node-based runners | Aggregate when their processes remain visible to Supercov |
|
|
@@ -63,19 +63,15 @@ TypeScript, and TSX are supported.
|
|
|
63
63
|
Supercov instruments an isolated copy. It does not ask you to add an import,
|
|
64
64
|
reporter, plugin, or alternate build output.
|
|
65
65
|
|
|
66
|
-
|
|
67
|
-
Supercov
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
part of the run, and a `build` script is run inside the isolated copy first.
|
|
71
|
-
Without it the server under test would never exist. A launch that only names a
|
|
72
|
-
build subcommand, such as `vite build`, is not taken as consuming a build.
|
|
66
|
+
If tests import compiled output such as `dist/` or launch a script that uses it,
|
|
67
|
+
Supercov runs the project's build inside the isolated copy before testing.
|
|
68
|
+
Keep using your normal test and build commands. Instrumentation does not
|
|
69
|
+
require changing the project's TypeScript settings.
|
|
73
70
|
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
command builds under measurement exactly as it does without it.
|
|
71
|
+
For assertion maps, Node, Vitest, Jest and Playwright's Node-side assertions
|
|
72
|
+
supply supported passing-occurrence evidence. Custom assertion wrappers and
|
|
73
|
+
browser-side checks can have additional observation limits. See
|
|
74
|
+
[Assertion evidence](assertion-evidence.md) before interpreting a missing occurrence.
|
|
79
75
|
|
|
80
76
|
### Browsers, servers, and child processes
|
|
81
77
|
|
|
@@ -114,22 +110,16 @@ whatever was still buffered.
|
|
|
114
110
|
| cargo-nextest | Exact test, attempt, retry, and binary identity | cargo-nextest 0.9.138 or 0.9.140 |
|
|
115
111
|
|
|
116
112
|
Supercov preserves Cargo's test selection, scheduling, fail-fast behavior,
|
|
117
|
-
environment
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
included), match arms, `&&` and `||`, `for` and `while` loops and the `?`
|
|
128
|
-
operator each take a probe; what a thread recorded before it passed an
|
|
129
|
-
`assert!`, `assert_eq!` or `assert_ne!` is linked to that assertion, so the
|
|
130
|
-
report can separate code a test checked from code it merely ran; const contexts and macro expansions stay in the
|
|
131
|
-
denominator behind an explicit limitation. Use the repository's normal flags after the
|
|
132
|
-
wrapped command:
|
|
113
|
+
environment and exit status. Doctests run with their own identities; nextest
|
|
114
|
+
retries remain separate attempts.
|
|
115
|
+
|
|
116
|
+
Measured source follows the modules rustc compiles, including `#[path]` and
|
|
117
|
+
literal `include!` calls. Undeclared `.rs` files are not treated as application
|
|
118
|
+
modules. Statements, functions, branches, boolean decisions, loops and error
|
|
119
|
+
propagation are measured. Const contexts and macro expansions remain visible
|
|
120
|
+
with explicit measurement limitations.
|
|
121
|
+
|
|
122
|
+
Use the repository's normal flags after the wrapped command:
|
|
133
123
|
|
|
134
124
|
```sh
|
|
135
125
|
npx supercov -- cargo test --workspace
|
|
@@ -149,35 +139,22 @@ attribution.
|
|
|
149
139
|
| pytest-rerunfailures | Exact per attempt; flaky tests are reported as such | |
|
|
150
140
|
| `python -m unittest` | Exact test and setUp/test/tearDown phase identity | Serial in-process; skips and expected failures are recorded; subtest failures roll up to the parent test |
|
|
151
141
|
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
Each interpreter writes commit-framed evidence to a process-owned mmap. A hard
|
|
170
|
-
kill preserves completed observations and an incomplete tail is ignored; an
|
|
171
|
-
exhausted transport or corrupt committed frame fails the run closed.
|
|
172
|
-
|
|
173
|
-
Measured obligations are statements (including several on one line), function
|
|
174
|
-
entry, boolean decisions with MC/DC vectors, `for` and comprehension iteration,
|
|
175
|
-
`and`/`or` short-circuiting, `match` case selection, and `try` completion,
|
|
176
|
-
handler selection and exception propagation, all derived from CPython's own
|
|
177
|
-
instruction positions rather than from exception hooks.
|
|
178
|
-
|
|
179
|
-
Interpreters launched with `-I`, `-E`, or `-S` ignore `PYTHONPATH` and are not
|
|
180
|
-
measured. Code compiled from strings at runtime has no source obligations.
|
|
142
|
+
Your project runs in place with its own interpreter and virtual environment.
|
|
143
|
+
Supercov adds its monitoring and runner hooks through the process environment;
|
|
144
|
+
you do not need to rewrite tests or configure a different build.
|
|
145
|
+
|
|
146
|
+
Coverage includes statements, functions, boolean decisions, loops,
|
|
147
|
+
comprehensions, short-circuit operators, `match` cases and exception paths.
|
|
148
|
+
Child interpreters, threads and thread pools can retain the calling test's
|
|
149
|
+
identity. The report also distinguishes execution before a passing assertion
|
|
150
|
+
from later execution. These phase records alone do not prove which values the
|
|
151
|
+
assertion checks.
|
|
152
|
+
|
|
153
|
+
Interpreters launched with `-I`, `-E` or `-S` ignore the required startup hook
|
|
154
|
+
and are not measured. Code compiled from strings at runtime has no source
|
|
155
|
+
obligations. Completed observations can survive a hard kill, but a corrupt or
|
|
156
|
+
exhausted evidence channel fails the run rather than reporting partial data as
|
|
157
|
+
complete.
|
|
181
158
|
|
|
182
159
|
```sh
|
|
183
160
|
npx supercov -- pytest
|
|
@@ -197,95 +174,35 @@ npx supercov -- python -m unittest
|
|
|
197
174
|
| Thread-parallel Minitest (`parallelize_me!`, `parallelize(with: :threads)`) | Probe observations exact per test; line, method and simple-branch observations made while phases overlapped go to the run, declared | |
|
|
198
175
|
| Cucumber | Exact scenario identity (`features/x.feature:LINE`), hook steps as setup/teardown | `cucumber`, `bundle exec cucumber` |
|
|
199
176
|
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
On Ruby 3.4 and newer, Supercov asks the `Coverage` module for line events
|
|
230
|
-
alone. Each test phase is sampled from `Coverage`, and asking for its branch
|
|
231
|
-
and method tables too made every sample rebuild both for every loaded file,
|
|
232
|
-
gems included, which was most of what a Ruby test suite paid under Supercov.
|
|
233
|
-
Instead, the statement that starts a branch body or a method body proves the
|
|
234
|
-
branch, the method and the decision outcome it witnesses, and what has no such
|
|
235
|
-
statement is probed: an `if` without `else`, a modifier `if`, a ternary, `&.`,
|
|
236
|
-
a `case` without `else`, an empty body. Ruby 3.3 cannot apply probes and keeps
|
|
237
|
-
reading `Coverage`'s branch and method keys.
|
|
238
|
-
`if true`/`if false`/`if nil` and other literal predicates are folded the way
|
|
239
|
-
Ruby folds them: no branch, and the dead arm is not an obligation. Code inside
|
|
240
|
-
a `Ractor.new` block gets no probes: a non-main Ractor cannot read the probe
|
|
241
|
-
receiver, so a probe there would raise where the untouched program ran. Its
|
|
242
|
-
lines are still counted; what only a probe could have proven inside it is
|
|
243
|
-
declared unmeasured at the block. A Spring
|
|
244
|
-
preloader started before the run has no hook and fails closed; JRuby and
|
|
245
|
-
TruffleRuby are not supported.
|
|
246
|
-
|
|
247
|
-
The runtime loads through `RUBYOPT` before Bundler and requires only
|
|
248
|
-
`coverage`, so it never activates a gem an application's Gemfile pins
|
|
249
|
-
differently. Insertions are checked against Ruby itself by a sweep
|
|
250
|
-
(`scripts/ruby-corpus-sweep.sh`, which drives `scripts/ruby-position-sweep.rb`)
|
|
251
|
-
over Ruby's whole standard library and the Rails, Rack, RSpec, Minitest,
|
|
252
|
-
test-unit and Cucumber gems, about 4,300 files installed once into a stable
|
|
253
|
-
corpus directory: every file is transformed and compiled with its line count
|
|
254
|
-
intact, every branch key Ruby 3.3 reads is compared with what Ruby reports for
|
|
255
|
-
the untouched source, and each file is loaded twice, untouched and
|
|
256
|
-
transformed, so the probes are proven to preserve behaviour and define the
|
|
257
|
-
same methods.
|
|
258
|
-
|
|
259
|
-
A `begin` whose body ends in an expression that can `return` from inside
|
|
260
|
-
itself has its handlers and propagation measured as usual, but its normal
|
|
261
|
-
completion is declared instead of probed unless every branch of that
|
|
262
|
-
expression can carry the probe: Ruby cannot pass such an expression as an
|
|
263
|
-
argument, which is what a probe wrapper does.
|
|
264
|
-
|
|
265
|
-
Measuring never breaks the program being measured. If a file cannot be
|
|
266
|
-
compiled with its probes, it loads unmodified: Ruby's `Coverage` still
|
|
267
|
-
measures its lines, methods and own branches, and only the obligations a
|
|
268
|
-
probe would have proven are declared for that file. Setting
|
|
269
|
-
`SUPERCOV_RUBY_SKIP_PROBES` to a comma-separated list of path fragments puts
|
|
270
|
-
chosen files on that same path deliberately, which is the escape hatch if
|
|
271
|
-
instrumentation ever disagrees with one of yours.
|
|
272
|
-
|
|
273
|
-
Ruby 3.3 does not apply its `Coverage` module to code compiled by a load hook,
|
|
274
|
-
so on 3.3 Supercov measures through `Coverage` alone: lines, methods and the
|
|
275
|
-
branches Ruby reports itself. Everything that needs a probe (multi-condition
|
|
276
|
-
decisions, `||=`, loops, `rescue` flow, a second statement on a line) is
|
|
277
|
-
declared unmeasured on that interpreter rather than shown as a gap. Ruby 3.4
|
|
278
|
-
and newer measure everything.
|
|
279
|
-
|
|
280
|
-
Ruby reads its own coverage as the interpreter exits, and that shapes what a
|
|
281
|
-
stopped process keeps. A process ended by a signal it can catch—`SIGTERM`
|
|
282
|
-
from a test's teardown, `SIGINT`—unwinds through that exit and reports
|
|
283
|
-
everything it measured. A process killed with `SIGKILL`, or one that leaves
|
|
284
|
-
through `exit!`, never gets there and takes with it whatever it observed since
|
|
285
|
-
its last test boundary. Supercov cannot recover that or say which lines it
|
|
286
|
-
would have been, so the run declares that a process did not report, which
|
|
287
|
-
blocks completeness, rather than counting those lines against the code. Stop a
|
|
288
|
-
Ruby server with `SIGTERM`, or wait for it to exit, and it reports.
|
|
177
|
+
Your project runs in place with its own interpreter and bundle. Supercov loads
|
|
178
|
+
through `RUBYOPT`; application files on disk and their backtrace line numbers
|
|
179
|
+
stay unchanged. RSpec, Minitest and test-unit assertions can identify execution
|
|
180
|
+
before a passing assertion. That timing evidence alone does not show which
|
|
181
|
+
values the assertion checks.
|
|
182
|
+
|
|
183
|
+
Ruby 3.4 and newer support statement, method, branch and MC/DC measurement,
|
|
184
|
+
including loops, iterator blocks, short-circuit operators, pattern matching,
|
|
185
|
+
optional calls and exception paths. Ruby 3.3 supplies Ruby's own line, method
|
|
186
|
+
and branch coverage; obligations requiring additional instrumentation are
|
|
187
|
+
reported as measurement limits.
|
|
188
|
+
|
|
189
|
+
Some constructs have narrower coverage. Code in a non-main Ractor keeps line
|
|
190
|
+
coverage but may lack other observations. Certain nested-return expressions
|
|
191
|
+
limit normal-completion measurement. Constant predicates are folded as Ruby
|
|
192
|
+
folds them, so unreachable alternatives do not become obligations.
|
|
193
|
+
|
|
194
|
+
If a file cannot be instrumented safely, Supercov loads it unchanged and reports
|
|
195
|
+
the remaining limits. To apply that fallback to a known incompatible file, set
|
|
196
|
+
`SUPERCOV_RUBY_SKIP_PROBES` to a comma-separated list of path fragments. This
|
|
197
|
+
reduces measurement; it is not a way to claim that skipped obligations are covered.
|
|
198
|
+
|
|
199
|
+
A Spring preloader started before the run has no coverage hook. Restart it
|
|
200
|
+
within the measured command. JRuby and TruffleRuby are not supported.
|
|
201
|
+
|
|
202
|
+
Stop test-owned Ruby servers with `SIGTERM` or wait for normal exit so they can
|
|
203
|
+
report their evidence. `SIGKILL` and `exit!` can lose observations since the last
|
|
204
|
+
test boundary. A process that does not report leaves a measurement limit; its
|
|
205
|
+
missing evidence is not counted as uncovered application code.
|
|
289
206
|
|
|
290
207
|
```sh
|
|
291
208
|
npx supercov -- rspec
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "supercov",
|
|
3
|
-
"version": "0.0.
|
|
3
|
+
"version": "0.0.46",
|
|
4
4
|
"description": "Coverage for coding agents and software factories \ud83c\udf19",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"repository": {
|
|
@@ -87,19 +87,21 @@
|
|
|
87
87
|
"test:python-monitoring": "cargo build -p supercov && node scripts/python-monitoring-integration.mjs",
|
|
88
88
|
"test:ruby-coverage": "cargo build -p supercov && node scripts/ruby-coverage-integration.mjs",
|
|
89
89
|
"test:rust-public-cargo": "cargo build -p supercov && node scripts/rust-public-cargo-integration.mjs",
|
|
90
|
-
"test:assertion-maps": "cargo build -p supercov && node scripts/assertion-map-schema.mjs --check && node scripts/assertion-map-integration.mjs && node scripts/assertion-map-js-integration.mjs",
|
|
91
|
-
"test:assertion-maps:js": "cargo build -p supercov && node scripts/assertion-map-schema.mjs --check && node scripts/assertion-map-js-integration.mjs",
|
|
92
|
-
"sync:assertion-schema": "cargo build -p supercov && node scripts/assertion-map-schema.mjs"
|
|
90
|
+
"test:assertion-maps": "cargo build -p supercov && node scripts/assertion-map-schema.mjs --check && node scripts/assertion-map-integration.mjs && node scripts/assertion-map-js-integration.mjs && node scripts/assertion-dogfood-integration.mjs",
|
|
91
|
+
"test:assertion-maps:js": "cargo build -p supercov && node scripts/assertion-map-schema.mjs --check && node scripts/assertion-map-js-integration.mjs && node scripts/assertion-dogfood-integration.mjs",
|
|
92
|
+
"sync:assertion-schema": "cargo build -p supercov && node scripts/assertion-map-schema.mjs",
|
|
93
|
+
"docs:sync": "node scripts/sync-docs.mjs",
|
|
94
|
+
"docs:check": "node scripts/sync-docs.mjs --check"
|
|
93
95
|
},
|
|
94
96
|
"optionalDependencies": {
|
|
95
|
-
"@supercov/cli-darwin-arm64": "0.0.
|
|
96
|
-
"@supercov/cli-darwin-x64": "0.0.
|
|
97
|
-
"@supercov/cli-linux-arm64-gnu": "0.0.
|
|
98
|
-
"@supercov/cli-linux-arm64-musl": "0.0.
|
|
99
|
-
"@supercov/cli-linux-x64-gnu": "0.0.
|
|
100
|
-
"@supercov/cli-linux-x64-musl": "0.0.
|
|
101
|
-
"@supercov/cli-win32-arm64": "0.0.
|
|
102
|
-
"@supercov/cli-win32-x64": "0.0.
|
|
97
|
+
"@supercov/cli-darwin-arm64": "0.0.46",
|
|
98
|
+
"@supercov/cli-darwin-x64": "0.0.46",
|
|
99
|
+
"@supercov/cli-linux-arm64-gnu": "0.0.46",
|
|
100
|
+
"@supercov/cli-linux-arm64-musl": "0.0.46",
|
|
101
|
+
"@supercov/cli-linux-x64-gnu": "0.0.46",
|
|
102
|
+
"@supercov/cli-linux-x64-musl": "0.0.46",
|
|
103
|
+
"@supercov/cli-win32-arm64": "0.0.46",
|
|
104
|
+
"@supercov/cli-win32-x64": "0.0.46"
|
|
103
105
|
},
|
|
104
106
|
"peerDependencies": {
|
|
105
107
|
"@playwright/test": ">=1.55.0",
|
|
@@ -578,6 +578,11 @@ function injectChildEnvironment(method, args) {
|
|
|
578
578
|
const environment = existingEnvironment
|
|
579
579
|
? { ...existingEnvironment, ...inherited }
|
|
580
580
|
: { ...process.env, ...inherited };
|
|
581
|
+
// The runtime child wrapper has captured the current async test scope in
|
|
582
|
+
// these options. An inherited setup/unscoped carrier from the launcher
|
|
583
|
+
// must not overwrite it when we restore the other coverage variables.
|
|
584
|
+
if (existingEnvironment?.SUPERCOV_CONTEXT !== undefined)
|
|
585
|
+
environment.SUPERCOV_CONTEXT = existingEnvironment.SUPERCOV_CONTEXT;
|
|
581
586
|
// The register path must come from THIS module's own location, never from
|
|
582
587
|
// the environment or the working directory. Monorepo runners both defeat
|
|
583
588
|
// the old derivation at once: turbo's strict env strips SUPERCOV_PROJECT_ROOT
|