supercov 0.0.41 → 0.0.43
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +24 -4
- package/analyzers/typescript/README.md +55 -0
- package/analyzers/typescript/bin/compiler-identity.mjs +78 -0
- package/analyzers/typescript/bin/identity.mjs +67 -0
- package/analyzers/typescript/bin/query.mjs +29 -0
- package/analyzers/typescript/dist/analyze.js +3972 -0
- package/analyzers/typescript/dist/archive.js +309 -0
- package/analyzers/typescript/dist/build-identity.json +1 -0
- package/analyzers/typescript/dist/compiler.js +32 -0
- package/analyzers/typescript/dist/frontend.js +75 -0
- package/analyzers/typescript/dist/native-frontend.js +271 -0
- package/analyzers/typescript/dist/pragmas.js +143 -0
- package/analyzers/typescript/dist/types.js +1 -0
- package/analyzers/typescript/package.json +27 -0
- package/analyzers/typescript/src/analyze.ts +4538 -0
- package/analyzers/typescript/src/archive.ts +438 -0
- package/analyzers/typescript/src/compiler.ts +49 -0
- package/analyzers/typescript/src/frontend.ts +136 -0
- package/analyzers/typescript/src/native-frontend.ts +315 -0
- package/analyzers/typescript/src/pragmas.ts +218 -0
- package/analyzers/typescript/src/types.ts +45 -0
- package/analyzers/typescript/tsconfig.json +12 -0
- package/docs/agent-loop.md +13 -0
- package/docs/assertion-evidence.md +135 -0
- package/docs/cli.md +10 -0
- package/docs/code-verification.md +182 -0
- package/docs/supported-suites.md +44 -11
- package/docs/troubleshooting.md +13 -0
- package/docs/verification.md +12 -0
- package/package.json +33 -15
- package/runtime/javascript/jest.cjs +134 -0
- package/runtime/javascript/jest.config.mjs +39 -0
- package/runtime/javascript/jestReporter.mjs +77 -0
- package/runtime/javascript/register.mjs +22 -4
- package/runtime/javascript/runtime.mjs +51 -13
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
# Assertion evidence (JS/TS)
|
|
2
|
+
|
|
3
|
+
Supercov can analyze which source behaviors existing assertions appear to check,
|
|
4
|
+
using an ordinary run archive, its matching source, and the existing statement
|
|
5
|
+
and assertion-phase evidence. This work happens after tests. It adds no new
|
|
6
|
+
test-time probes, does not run mutants, and does not rewrite your tests.
|
|
7
|
+
|
|
8
|
+
This is candidate evidence, not a proof that arbitrary changes are safe.
|
|
9
|
+
`assertionScore` remains null. Execution-only links, test gaps and analysis limits
|
|
10
|
+
are different things; do not interpret a passing assertion nearby as protection.
|
|
11
|
+
|
|
12
|
+
## Requirements
|
|
13
|
+
|
|
14
|
+
Use the **npm Supercov launcher**, which supplies the installed analyzer assets.
|
|
15
|
+
The current JS/TS adapter is exercised with Node's test runner and Vitest.
|
|
16
|
+
Browser, background, merged, retried and ambiguously attributed records have
|
|
17
|
+
explicit limitations; other language assertion analyzers are not enabled here.
|
|
18
|
+
|
|
19
|
+
The analyzed project must provide a TypeScript compiler API, **even for a
|
|
20
|
+
JavaScript project**. For example, add TypeScript as a development dependency
|
|
21
|
+
before recording the run. The analyzer uses that project's compiler, not a
|
|
22
|
+
silently substituted global or bundled version. Installing it afterward changes
|
|
23
|
+
the dependency fingerprint, so rerun the suite. Missing/incompatible compiler APIs
|
|
24
|
+
produce an error. **TypeScript 5.8.3 and native 7.0.2 are tested.** Version 7.0.2
|
|
25
|
+
uses its own native parser/checker, not a fallback to TypeScript 5. Install its
|
|
26
|
+
platform-specific optional dependency too; the report hashes both the JS client
|
|
27
|
+
and the native compiler package, including its standard libraries. This backend
|
|
28
|
+
requires Node 22.12 or newer (Node 24 tested), and uses original-source evidence
|
|
29
|
+
from ordinary Supercov archives, not legacy ts-node/V8 generated-line coverage.
|
|
30
|
+
Native module resolution currently follows actual import/export references;
|
|
31
|
+
unresolved helper-only specifiers remain visible as compiler limitations.
|
|
32
|
+
Other native compiler versions are not enabled until separately calibrated.
|
|
33
|
+
Do not downgrade an application's compiler just to improve an assertion report.
|
|
34
|
+
|
|
35
|
+
Compiler compatibility and regression checks do not prove general correctness.
|
|
36
|
+
All candidates remain unverified and `assertionScore` remains null. Query-side
|
|
37
|
+
native compiler work adds no test-time instrumentation.
|
|
38
|
+
|
|
39
|
+
Standalone native/Python/Ruby distributions do not currently bundle this JS/TS
|
|
40
|
+
analyzer. Use the npm launcher, or explicitly set `SUPERCOV_PACKAGE_ROOT` to an
|
|
41
|
+
installed npm package directory. Normal coverage commands are unaffected.
|
|
42
|
+
|
|
43
|
+
## Run, inspect, follow evidence
|
|
44
|
+
|
|
45
|
+
`assertions` analyzes source behaviors and their assertion evidence; it does not
|
|
46
|
+
just count or list assertion calls. Use `npx supercov docs assertion-evidence`
|
|
47
|
+
to read this guide from the installed package.
|
|
48
|
+
|
|
49
|
+
```sh
|
|
50
|
+
npx supercov -- npm test
|
|
51
|
+
npx supercov runs latest assertions --limit 5
|
|
52
|
+
npx supercov runs latest assertions --file src/core.ts --json
|
|
53
|
+
npx supercov runs latest assertions --site '<site-id>' --json
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
The summary reports candidate counts and the site denominator, without claiming
|
|
57
|
+
a verified assertion percentage. Each ordinary site row contains its source,
|
|
58
|
+
candidate classification, available facts, and an `evidence.pointer` for full
|
|
59
|
+
details. Global tests, attempts, execution links, diagnostics and source-scope
|
|
60
|
+
limits are referenced under `evidence`, not repeated inside every site page.
|
|
61
|
+
|
|
62
|
+
```sh
|
|
63
|
+
npx supercov runs latest assertions --evidence /tests --limit 5 --json
|
|
64
|
+
npx supercov runs latest assertions --evidence /diagnostics --json
|
|
65
|
+
npx supercov runs latest assertions --evidence /sites/0/facts --json
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
Pointers use JSON Pointer syntax: escape `/` in a property name as `~1` and `~`
|
|
69
|
+
as `~0`. Use an empty pointer (`--evidence ''`) for the document root. Follow
|
|
70
|
+
returned pointers rather than constructing indices from filtered page offsets:
|
|
71
|
+
site pointers index the complete, stable inventory.
|
|
72
|
+
|
|
73
|
+
## Pagination and oversized records
|
|
74
|
+
|
|
75
|
+
All JSON responses stay within the normal response budget. Pages may contain
|
|
76
|
+
fewer than `--limit` results: **follow `pagination.nextOffset`**, not offset plus
|
|
77
|
+
the requested limit. `hasMore: false` marks the end.
|
|
78
|
+
|
|
79
|
+
An oversized site/hint is returned as `detailOnly: true` with an
|
|
80
|
+
`evidence.pointer`. Nothing has been discarded. An evidence page returns immediate
|
|
81
|
+
object members or array entries as `items`. Small entries contain `value`; large
|
|
82
|
+
ones contain `detailOnly: true` and another `pointer` to inspect. String leaves
|
|
83
|
+
return `text` chunks, with offsets/counts measured in Unicode scalar values,
|
|
84
|
+
not bytes. This allows reading one large observation or diagnostic completely.
|
|
85
|
+
|
|
86
|
+
```sh
|
|
87
|
+
npx supercov runs '<run-id>' assertions --evidence '<returned-pointer>' \
|
|
88
|
+
--offset 0 --limit 4000 --analysis '<analysisId>' --json
|
|
89
|
+
```
|
|
90
|
+
|
|
91
|
+
Every page carries an `analysisId` hashing the complete derived document and its
|
|
92
|
+
provenance. Pass the first page's id as `--analysis` on follow-ups to reject mixed
|
|
93
|
+
analyses if the compiler, analyzer, or derived results change. Use a fixed run id
|
|
94
|
+
instead of `latest` while paging. Source/run freshness checks still run on every
|
|
95
|
+
query. Derived results are not cached or written into the archive.
|
|
96
|
+
|
|
97
|
+
JSON reports use `reportSchema: 2`. Shared evidence is accessed through the
|
|
98
|
+
evidence pointers above.
|
|
99
|
+
Success and error JSON envelopes identify this query as `coverage.assertions`.
|
|
100
|
+
|
|
101
|
+
## Optional assertion hints
|
|
102
|
+
|
|
103
|
+
```ts
|
|
104
|
+
// observes: src/core.ts#compute return value
|
|
105
|
+
assert.equal(compute(), 4);
|
|
106
|
+
```
|
|
107
|
+
|
|
108
|
+
The comment must precede a statement containing exactly one recognized assertion.
|
|
109
|
+
The project-relative file, exact function/owner name, and optional literal source
|
|
110
|
+
substring must select one inventory site. Optional `via ...` is explanation only.
|
|
111
|
+
|
|
112
|
+
```sh
|
|
113
|
+
npx supercov runs latest assertions --pragmas --json
|
|
114
|
+
```
|
|
115
|
+
|
|
116
|
+
Origin is `user-suggested`; validation is separately `analyzer-supported`,
|
|
117
|
+
`unresolved`, or `invalid`. Support requires the named assertion's exact passing
|
|
118
|
+
witness and a connection supported by the ordinary effect rules. Strength remains
|
|
119
|
+
presence/value/total; presence does not mean value protection. Unsupported
|
|
120
|
+
decision, absence and internal-state paths remain unresolved. A missing inventory
|
|
121
|
+
match may reflect unsupported source rather than a bad declaration.
|
|
122
|
+
|
|
123
|
+
Hints cannot inject observations, borrow another assertion's evidence, inflate
|
|
124
|
+
coverage or remove sites from the denominator. Editing comments changes test
|
|
125
|
+
source and requires a new run. Supported hints retain the analyzer's limitations;
|
|
126
|
+
they are not formal proofs.
|
|
127
|
+
|
|
128
|
+
## Agent workflow
|
|
129
|
+
|
|
130
|
+
Read coverage first, then inspect assertion candidates and their evidence before
|
|
131
|
+
writing another test. An execution gap may need a reachable scenario; an assertion
|
|
132
|
+
gap may need a stronger check. An analysis limit needs investigation—not a test
|
|
133
|
+
written just to satisfy the analyzer. Keep the original denominator and unresolved
|
|
134
|
+
work visible. Never automatically treat an `evident` candidate as permission to
|
|
135
|
+
change application behavior.
|
package/docs/cli.md
CHANGED
|
@@ -17,6 +17,7 @@ npx supercov --help
|
|
|
17
17
|
| Read the newest run | `npx supercov runs latest` |
|
|
18
18
|
| Find useful gaps | `npx supercov runs latest gaps` |
|
|
19
19
|
| Inspect one file | `npx supercov runs latest file <path>` |
|
|
20
|
+
| Inspect assertion evidence (JS/TS) | `npx supercov runs latest assertions` |
|
|
20
21
|
| Compare two runs | `npx supercov diff <older> <newer>` |
|
|
21
22
|
| Combine shards | `npx supercov merge <id> <id> [...]` |
|
|
22
23
|
| Remove local data | `npx supercov clean` |
|
|
@@ -71,6 +72,7 @@ npx supercov runs <run-id> [query] [options]
|
|
|
71
72
|
| `kinds` | Group coverage by test level, such as unit or E2E |
|
|
72
73
|
| `runners` | Group coverage by test runner |
|
|
73
74
|
| `scope` | Review included, excluded, and ambiguous source files |
|
|
75
|
+
| `assertions` | Analyze source-to-assertion evidence, gaps, and limits (JS/TS) |
|
|
74
76
|
| `minimize` | Find a small test subset that preserves a coverage target |
|
|
75
77
|
|
|
76
78
|
Common examples:
|
|
@@ -88,8 +90,16 @@ Run any query with `--help` to see only the options valid for that query:
|
|
|
88
90
|
```sh
|
|
89
91
|
npx supercov runs latest --help
|
|
90
92
|
npx supercov runs latest file --help
|
|
93
|
+
npx supercov runs latest assertions --help
|
|
91
94
|
```
|
|
92
95
|
|
|
96
|
+
The npm-only `assertions` query runs after tests against matching source. It
|
|
97
|
+
reports unverified candidates, not a proven assertion score or permission to
|
|
98
|
+
change behavior. Its filters differ from the structural queries below; see
|
|
99
|
+
[assertion evidence](assertion-evidence.md) for requirements, pagination, and
|
|
100
|
+
optional assertion hints. The guide is also available through
|
|
101
|
+
`npx supercov docs assertion-evidence`.
|
|
102
|
+
|
|
93
103
|
## Narrow a view
|
|
94
104
|
|
|
95
105
|
| Option | Meaning |
|
|
@@ -0,0 +1,182 @@
|
|
|
1
|
+
# Example
|
|
2
|
+
|
|
3
|
+
This example measures coverage for a small checkout function. It includes the
|
|
4
|
+
original tests, an additional test for an expired session, and commands for
|
|
5
|
+
comparing the results.
|
|
6
|
+
|
|
7
|
+
## Before you start
|
|
8
|
+
|
|
9
|
+
You need Node.js 22 or newer and npm. Clone the repository and install the
|
|
10
|
+
example's dependencies:
|
|
11
|
+
|
|
12
|
+
```sh
|
|
13
|
+
git clone --depth 1 https://github.com/supercorp-ai/supercov.git
|
|
14
|
+
cd supercov/examples/checkout-verification
|
|
15
|
+
npm ci
|
|
16
|
+
```
|
|
17
|
+
|
|
18
|
+
Run the commands below from this directory. The example uses Supercov 0.0.42
|
|
19
|
+
and Node's built-in test runner. Both the original tests and the additional
|
|
20
|
+
test are included. The first three steps do not require any file edits.
|
|
21
|
+
|
|
22
|
+
## 1. Run the original tests
|
|
23
|
+
|
|
24
|
+
In `src/session.js`, checkout is allowed only if the customer is signed in and
|
|
25
|
+
their session has not expired:
|
|
26
|
+
|
|
27
|
+
```js
|
|
28
|
+
export function canCheckout(signedIn, expired) {
|
|
29
|
+
if (signedIn && !expired) return true;
|
|
30
|
+
return false;
|
|
31
|
+
}
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
The two tests in `tests/session.test.js` check a valid session and a signed-out
|
|
35
|
+
visitor:
|
|
36
|
+
|
|
37
|
+
```js
|
|
38
|
+
assert.equal(canCheckout(true, false), true);
|
|
39
|
+
assert.equal(canCheckout(false, false), false);
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
Run those tests through Supercov, then open the summary:
|
|
43
|
+
|
|
44
|
+
```sh
|
|
45
|
+
npx supercov -- node --test tests/session.test.js
|
|
46
|
+
npx supercov runs latest
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
Everything after `--` is the test command Supercov runs. In your own project,
|
|
50
|
+
use your existing test command there.
|
|
51
|
+
|
|
52
|
+
Both tests pass. The coverage section shows:
|
|
53
|
+
|
|
54
|
+
```text
|
|
55
|
+
Coverage
|
|
56
|
+
Lines 100.00% (3/3)
|
|
57
|
+
Branches 100.00% (2/2)
|
|
58
|
+
MC/DC 50.00% (1/2)
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
Line and branch coverage are 100% because the tests reach both `return true`
|
|
62
|
+
and `return false`. The MC/DC result shows there is still a condition to test.
|
|
63
|
+
|
|
64
|
+
Keep the run ID printed at the top of the summary. You'll use it to compare
|
|
65
|
+
this run with the next one.
|
|
66
|
+
|
|
67
|
+
## 2. Inspect the missing condition
|
|
68
|
+
|
|
69
|
+
Ask about the decision on line 2:
|
|
70
|
+
|
|
71
|
+
```sh
|
|
72
|
+
npx supercov runs latest decision src/session.js:2
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
```text
|
|
76
|
+
signedIn && !expired
|
|
77
|
+
C1 covered + asserted: signedIn
|
|
78
|
+
C2 MISSING: !expired
|
|
79
|
+
confidence asserted; asserted MC/DC 1/2
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
MC/DC stands for Modified Condition/Decision Coverage. It checks whether each
|
|
83
|
+
condition has independently affected the decision. The original tests show
|
|
84
|
+
that changing `signedIn` changes the result, but neither test changes `expired`.
|
|
85
|
+
That leaves one of two conditions covered: 50%.
|
|
86
|
+
|
|
87
|
+
`C2 MISSING: !expired` points to the case to test: a customer who is still
|
|
88
|
+
signed in, but whose session has expired. Checkout should be denied.
|
|
89
|
+
|
|
90
|
+
## 3. Include the expired-session test
|
|
91
|
+
|
|
92
|
+
`tests/expired-session.test.js` contains that test:
|
|
93
|
+
|
|
94
|
+
```js
|
|
95
|
+
test('an expired session cannot check out', () => {
|
|
96
|
+
assert.equal(canCheckout(true, true), false);
|
|
97
|
+
});
|
|
98
|
+
```
|
|
99
|
+
|
|
100
|
+
Run both test files and open the new summary:
|
|
101
|
+
|
|
102
|
+
```sh
|
|
103
|
+
npx supercov -- node --test tests/session.test.js tests/expired-session.test.js
|
|
104
|
+
npx supercov runs latest
|
|
105
|
+
```
|
|
106
|
+
|
|
107
|
+
All three tests pass:
|
|
108
|
+
|
|
109
|
+
```text
|
|
110
|
+
Coverage
|
|
111
|
+
Lines 100.00% (3/3)
|
|
112
|
+
Branches 100.00% (2/2)
|
|
113
|
+
MC/DC 100.00% (2/2)
|
|
114
|
+
```
|
|
115
|
+
|
|
116
|
+
Query the same decision again:
|
|
117
|
+
|
|
118
|
+
```sh
|
|
119
|
+
npx supercov runs latest decision src/session.js:2
|
|
120
|
+
```
|
|
121
|
+
|
|
122
|
+
The expiry condition is now covered. `asserted MC/DC 2/2` means both conditions
|
|
123
|
+
have coverage evidence linked to passing assertions:
|
|
124
|
+
|
|
125
|
+
```text
|
|
126
|
+
C2 covered + asserted: !expired
|
|
127
|
+
confidence asserted; asserted MC/DC 2/2
|
|
128
|
+
```
|
|
129
|
+
|
|
130
|
+
Compare the runs, replacing `<before-run-id>` with the ID you saved in step 1:
|
|
131
|
+
|
|
132
|
+
```sh
|
|
133
|
+
npx supercov diff <before-run-id> latest
|
|
134
|
+
```
|
|
135
|
+
|
|
136
|
+
```text
|
|
137
|
+
lines +0pp, branches +0pp, MC/DC +50pp
|
|
138
|
+
gained: 0 lines, 0 branches, 1 MC/DC conditions
|
|
139
|
+
lost: 0 lines, 0 branches, 0 MC/DC conditions
|
|
140
|
+
+ MC/DC src/session.js:2 C2 !expired
|
|
141
|
+
```
|
|
142
|
+
|
|
143
|
+
Line and branch coverage have not changed. The new test covers the missing
|
|
144
|
+
expiry condition without changing application code.
|
|
145
|
+
|
|
146
|
+
## 4. Check that the test catches a regression
|
|
147
|
+
|
|
148
|
+
In this example only, temporarily remove the expiry check from `src/session.js`:
|
|
149
|
+
|
|
150
|
+
```diff
|
|
151
|
+
- if (signedIn && !expired) return true;
|
|
152
|
+
+ if (signedIn) return true;
|
|
153
|
+
```
|
|
154
|
+
|
|
155
|
+
Run the original two tests against the changed function:
|
|
156
|
+
|
|
157
|
+
```sh
|
|
158
|
+
npx supercov -- node --test tests/session.test.js
|
|
159
|
+
```
|
|
160
|
+
|
|
161
|
+
Then include the expired-session test:
|
|
162
|
+
|
|
163
|
+
```sh
|
|
164
|
+
npx supercov -- node --test tests/session.test.js tests/expired-session.test.js
|
|
165
|
+
```
|
|
166
|
+
|
|
167
|
+
| Tests run against the changed function | Result |
|
|
168
|
+
| --- | --- |
|
|
169
|
+
| Original two tests | Both pass. |
|
|
170
|
+
| All three tests | The expired-session test fails; the other two pass. |
|
|
171
|
+
|
|
172
|
+
The new test expects `false`, but the changed function returns `true`. The
|
|
173
|
+
second command should fail: that is the test catching the removed expiry check.
|
|
174
|
+
|
|
175
|
+
Restore `&& !expired` in `src/session.js` when you finish, then rerun all three
|
|
176
|
+
tests with the same command. They should pass again.
|
|
177
|
+
|
|
178
|
+
## Next
|
|
179
|
+
|
|
180
|
+
- [Full example and recorded output](https://github.com/supercorp-ai/supercov/tree/main/examples/checkout-verification) — source, tests, and the complete output excerpted above.
|
|
181
|
+
- [Understanding coverage](coverage-model.md) — what each metric measures and what 100% means.
|
|
182
|
+
- [Agent workflow](agent-loop.md) — use the same run, inspect, test, and compare steps with a coding agent.
|
package/docs/supported-suites.md
CHANGED
|
@@ -45,7 +45,7 @@ Supercov reports the level it actually observed. It does not guess.
|
|
|
45
45
|
| --- | --- |
|
|
46
46
|
| Playwright | Exact per test, worker, retry, outcome, action, and assertion phase |
|
|
47
47
|
| Vitest | Exact per test, with setup execution kept separate |
|
|
48
|
-
| Jest | Exact per test, including
|
|
48
|
+
| Jest | Exact per test, including parameterized tests, with the user's own configuration, setup files and reporters kept; `expect` assertions link the evidence they check |
|
|
49
49
|
| `node:test` | Exact per test |
|
|
50
50
|
| AVA and Mocha | Aggregate structural coverage |
|
|
51
51
|
| Other Node-based runners | Aggregate when their processes remain visible to Supercov |
|
|
@@ -109,7 +109,7 @@ whatever was still buffered.
|
|
|
109
109
|
|
|
110
110
|
| Runner | Attribution | Current requirement |
|
|
111
111
|
| --- | --- | --- |
|
|
112
|
-
| Cargo's standard libtest runner | Exact test and
|
|
112
|
+
| Cargo's standard libtest runner | Exact test, attempt, and passing-assertion identity | Rust 1.95; run with `npx supercov -- cargo test` |
|
|
113
113
|
| rustdoc doctests | Exact doctest identity; every doctest runs in a process of its own | Rust 1.95; part of `npx supercov -- cargo test` |
|
|
114
114
|
| cargo-nextest | Exact test, attempt, retry, and binary identity | cargo-nextest 0.9.138 or 0.9.140 |
|
|
115
115
|
|
|
@@ -125,7 +125,9 @@ literal `include!` calls; a `.rs` file nothing declares as a module, such as
|
|
|
125
125
|
one embedded with `include_str!`, is left untouched. Statements, functions,
|
|
126
126
|
`if`/`while`/match-guard decisions with their conditions (let chains
|
|
127
127
|
included), match arms, `&&` and `||`, `for` and `while` loops and the `?`
|
|
128
|
-
operator each take a probe;
|
|
128
|
+
operator each take a probe; what a thread recorded before it passed an
|
|
129
|
+
`assert!`, `assert_eq!` or `assert_ne!` is linked to that assertion, so the
|
|
130
|
+
report can separate code a test checked from code it merely ran; const contexts and macro expansions stay in the
|
|
129
131
|
denominator behind an explicit limitation. Use the repository's normal flags after the
|
|
130
132
|
wrapped command:
|
|
131
133
|
|
|
@@ -147,10 +149,19 @@ attribution.
|
|
|
147
149
|
| pytest-rerunfailures | Exact per attempt; flaky tests are reported as such | |
|
|
148
150
|
| `python -m unittest` | Exact test and setUp/test/tearDown phase identity | Serial in-process; skips and expected failures are recorded; subtest failures roll up to the parent test |
|
|
149
151
|
|
|
152
|
+
Evidence a test records before its first assertion is linked to that assertion
|
|
153
|
+
when the test passes, so a line reads "linked to a passing assertion" rather
|
|
154
|
+
than "execution only". Under pytest this covers plain `assert` statements,
|
|
155
|
+
through pytest's assertion-pass hook, which Supercov turns on for a rewrite
|
|
156
|
+
cache of its own so plain runs keep theirs, and `pytest.raises` and
|
|
157
|
+
`pytest.warns` blocks; every `unittest` `assert*` method counts under both
|
|
158
|
+
runners. What a test runs after its first assertion is execution only.
|
|
159
|
+
|
|
150
160
|
Supercov measures Python through CPython's own monitoring interface. Nothing is
|
|
151
161
|
copied, rewritten, or compiled differently: the project runs in place with its
|
|
152
162
|
own interpreter and virtual environment, and Supercov only adds a start-up hook
|
|
153
|
-
through `PYTHONPATH`, a pytest plugin through `PYTEST_PLUGINS`,
|
|
163
|
+
through `PYTHONPATH`, a pytest plugin through `PYTEST_PLUGINS`, one pytest
|
|
164
|
+
option (`enable_assertion_pass_hook`) through `PYTEST_ADDOPTS`, and a few
|
|
154
165
|
`SUPERCOV_*` variables. Child interpreters started with `subprocess` or
|
|
155
166
|
`multiprocessing` inherit the exact test identity; threads and thread pools
|
|
156
167
|
carry it through `contextvars`.
|
|
@@ -186,6 +197,12 @@ npx supercov -- python -m unittest
|
|
|
186
197
|
| Thread-parallel Minitest (`parallelize_me!`, `parallelize(with: :threads)`) | Probe observations exact per test; line, method and simple-branch observations made while phases overlapped go to the run, declared | |
|
|
187
198
|
| Cucumber | Exact scenario identity (`features/x.feature:LINE`), hook steps as setup/teardown | `cucumber`, `bundle exec cucumber` |
|
|
188
199
|
|
|
200
|
+
Evidence a test records before its first assertion is linked to that assertion
|
|
201
|
+
when the test passes: Minitest's `assert`/`refute` family, RSpec's
|
|
202
|
+
`expect(...).to` and `not_to` (in Cucumber steps too) and test-unit's
|
|
203
|
+
assertions all count. What a test runs after its first assertion is execution
|
|
204
|
+
only.
|
|
205
|
+
|
|
189
206
|
Supercov measures Ruby with Ruby's own `Coverage` module plus probe calls it
|
|
190
207
|
splices into application files in memory as they load. Nothing on disk is
|
|
191
208
|
rewritten or copied; the project runs with its own interpreter and bundle, and
|
|
@@ -208,20 +225,36 @@ instead of as a definition.
|
|
|
208
225
|
A statement on a line Ruby's own line table never counts (`x = case`, a
|
|
209
226
|
multi-line literal assignment, a bare `begin`, `if false`) gets a probe at load
|
|
210
227
|
time instead.
|
|
228
|
+
|
|
229
|
+
On Ruby 3.4 and newer, Supercov asks the `Coverage` module for line events
|
|
230
|
+
alone. Each test phase is sampled from `Coverage`, and asking for its branch
|
|
231
|
+
and method tables too made every sample rebuild both for every loaded file,
|
|
232
|
+
gems included, which was most of what a Ruby test suite paid under Supercov.
|
|
233
|
+
Instead, the statement that starts a branch body or a method body proves the
|
|
234
|
+
branch, the method and the decision outcome it witnesses, and what has no such
|
|
235
|
+
statement is probed: an `if` without `else`, a modifier `if`, a ternary, `&.`,
|
|
236
|
+
a `case` without `else`, an empty body. Ruby 3.3 cannot apply probes and keeps
|
|
237
|
+
reading `Coverage`'s branch and method keys.
|
|
211
238
|
`if true`/`if false`/`if nil` and other literal predicates are folded the way
|
|
212
|
-
Ruby folds them: no branch, and the dead arm is not an obligation.
|
|
239
|
+
Ruby folds them: no branch, and the dead arm is not an obligation. Code inside
|
|
240
|
+
a `Ractor.new` block gets no probes: a non-main Ractor cannot read the probe
|
|
241
|
+
receiver, so a probe there would raise where the untouched program ran. Its
|
|
242
|
+
lines are still counted; what only a probe could have proven inside it is
|
|
243
|
+
declared unmeasured at the block. A Spring
|
|
213
244
|
preloader started before the run has no hook and fails closed; JRuby and
|
|
214
245
|
TruffleRuby are not supported.
|
|
215
246
|
|
|
216
247
|
The runtime loads through `RUBYOPT` before Bundler and requires only
|
|
217
248
|
`coverage`, so it never activates a gem an application's Gemfile pins
|
|
218
249
|
differently. Insertions are checked against Ruby itself by a sweep
|
|
219
|
-
(`scripts/ruby-position-sweep.rb`)
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
250
|
+
(`scripts/ruby-corpus-sweep.sh`, which drives `scripts/ruby-position-sweep.rb`)
|
|
251
|
+
over Ruby's whole standard library and the Rails, Rack, RSpec, Minitest,
|
|
252
|
+
test-unit and Cucumber gems, about 4,300 files installed once into a stable
|
|
253
|
+
corpus directory: every file is transformed and compiled with its line count
|
|
254
|
+
intact, every branch key Ruby 3.3 reads is compared with what Ruby reports for
|
|
255
|
+
the untouched source, and each file is loaded twice, untouched and
|
|
256
|
+
transformed, so the probes are proven to preserve behaviour and define the
|
|
257
|
+
same methods.
|
|
225
258
|
|
|
226
259
|
A `begin` whose body ends in an expression that can `return` from inside
|
|
227
260
|
itself has its handlers and propagation measured as usual, but its normal
|
package/docs/troubleshooting.md
CHANGED
|
@@ -176,3 +176,16 @@ never looks complete. Since 0.0.29 the background and execution-trace writers al
|
|
|
176
176
|
clone sharing their file and move to a fresh one, so this should be rare; if it
|
|
177
177
|
persists, check whether something outside Supercov appends to
|
|
178
178
|
`.supercov/…/server/background/`.
|
|
179
|
+
|
|
180
|
+
## "Complete ... — N declared boundary(ies)"
|
|
181
|
+
|
|
182
|
+
A limitation is either blocking or declared. A blocking one means Supercov
|
|
183
|
+
could not measure something inside the denominator it claims — corrupt
|
|
184
|
+
evidence, a transport that never reported — and the run reads as
|
|
185
|
+
"Incomplete". A declared one marks a boundary of the denominator itself: a
|
|
186
|
+
Rust macro the compiler expands, a `const fn` body no runtime probe can
|
|
187
|
+
enter, a proc-macro crate whose code runs inside the compiler. Nothing inside
|
|
188
|
+
the measured denominator went unmeasured, so the run reads as complete within
|
|
189
|
+
those boundaries, and each boundary is reported at the line it covers with a
|
|
190
|
+
reason. `runs <run> file <path>` lists them per file; the `measurement`
|
|
191
|
+
object in `--json` output counts them under `declared`.
|
package/docs/verification.md
CHANGED
|
@@ -78,6 +78,18 @@ self-consistent calculation error does not pass unnoticed. Python and Ruby
|
|
|
78
78
|
gates run real suites through their supported runners and assert the resulting
|
|
79
79
|
coverage totals, test identity, and measurement limits.
|
|
80
80
|
|
|
81
|
+
Rust numbers are also checked against an independent LLVM line-coverage
|
|
82
|
+
oracle over real crates. `npm run oracle:rust` runs the same tests plain, under
|
|
83
|
+
the oracle, and under Supercov on twenty-six crates from bytes to tokio,
|
|
84
|
+
compares line coverage file by file, and fails if Supercov ran any suite
|
|
85
|
+
differently from plain Cargo. The two tools count differently — the oracle
|
|
86
|
+
counts every executable region after monomorphisation, Supercov the lines its
|
|
87
|
+
own obligations sit on — so the per-file comparison is read by a person: a file
|
|
88
|
+
at 0% in one and 70% in the other means lost evidence or a wrong denominator,
|
|
89
|
+
and this check found five such defects before 0.0.42. It runs locally and
|
|
90
|
+
costs no hosted minutes; `scripts/rust-coverage-oracle.mjs` documents what
|
|
91
|
+
each flag means.
|
|
92
|
+
|
|
81
93
|
These checks reduce risk; they do not replace reviewing the assertions and
|
|
82
94
|
behavior protected by a new test.
|
|
83
95
|
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "supercov",
|
|
3
|
-
"version": "0.0.
|
|
4
|
-
"description": "
|
|
3
|
+
"version": "0.0.43",
|
|
4
|
+
"description": "Coverage for coding agents and software factories 🌙",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"repository": {
|
|
7
7
|
"type": "git",
|
|
@@ -16,6 +16,19 @@
|
|
|
16
16
|
"bin",
|
|
17
17
|
"runtime/javascript",
|
|
18
18
|
"docs",
|
|
19
|
+
"analyzers/typescript/bin",
|
|
20
|
+
"analyzers/typescript/dist/analyze.js",
|
|
21
|
+
"analyzers/typescript/dist/archive.js",
|
|
22
|
+
"analyzers/typescript/dist/compiler.js",
|
|
23
|
+
"analyzers/typescript/dist/frontend.js",
|
|
24
|
+
"analyzers/typescript/dist/native-frontend.js",
|
|
25
|
+
"analyzers/typescript/dist/pragmas.js",
|
|
26
|
+
"analyzers/typescript/dist/types.js",
|
|
27
|
+
"analyzers/typescript/dist/build-identity.json",
|
|
28
|
+
"analyzers/typescript/src",
|
|
29
|
+
"analyzers/typescript/package.json",
|
|
30
|
+
"analyzers/typescript/tsconfig.json",
|
|
31
|
+
"analyzers/typescript/README.md",
|
|
19
32
|
"README.md"
|
|
20
33
|
],
|
|
21
34
|
"engines": {
|
|
@@ -34,6 +47,7 @@
|
|
|
34
47
|
},
|
|
35
48
|
"scripts": {
|
|
36
49
|
"build": "cargo build -p supercov",
|
|
50
|
+
"build:asserted-typescript": "npm --prefix analyzers/typescript run build",
|
|
37
51
|
"sync:rust-assets": "node scripts/sync-rust-package-assets.mjs",
|
|
38
52
|
"test:rust-assets": "node scripts/sync-rust-package-assets.mjs --check",
|
|
39
53
|
"check:rustc-backend-spike": "(cd spikes/rustc-backend && RUSTC_BOOTSTRAP=1 RUSTUP_TOOLCHAIN=1.95.0 cargo fmt --check && RUSTC_BOOTSTRAP=1 RUSTUP_TOOLCHAIN=1.95.0 cargo clippy --all-targets -- -D warnings)",
|
|
@@ -41,16 +55,19 @@
|
|
|
41
55
|
"test:rust-compiler-spikes": "cargo build -p supercov && node scripts/rust-libtest-companion-spike.mjs && node scripts/rust-async-attribution-spike.mjs && node scripts/rust-subprocess-attribution-spike.mjs && node scripts/rust-custom-harness-spike.mjs && node scripts/rust-libtest-builder-lifecycle-spike.mjs",
|
|
42
56
|
"test": "cargo test --workspace",
|
|
43
57
|
"test:runtime": "node --test tests/runtime/*.test.mjs",
|
|
58
|
+
"test:asserted-typescript": "npm --prefix analyzers/typescript test",
|
|
59
|
+
"test:asserted-integration": "cargo build -p supercov && npm --prefix analyzers/typescript run build && SUPERCOV_ASSERTED_INTEGRATION=1 node --test analyzers/typescript/tests/archive.integration.test.mjs",
|
|
60
|
+
"test:asserted-package": "cargo build -p supercov && npm run build:asserted-typescript && node scripts/asserted-packed-integration.mjs",
|
|
44
61
|
"test:fixture": "cargo build -p supercov && node scripts/rust-fixture-matrix.mjs",
|
|
45
|
-
"test:packed-npx": "cargo build --release -p supercov && node scripts/packed-npx-integration.mjs",
|
|
62
|
+
"test:packed-npx": "cargo build --release -p supercov && npm run build:asserted-typescript && node scripts/packed-npx-integration.mjs",
|
|
46
63
|
"test:isolation": "cargo build -p supercov && node scripts/isolation-integration.mjs",
|
|
47
64
|
"test:filesystem": "cargo test --workspace && cargo build -p supercov && node scripts/workspace-crash-integration.mjs",
|
|
48
65
|
"test:watchdog": "cargo build -p supercov && node scripts/watchdog-integration.mjs",
|
|
49
66
|
"test:engine-contract": "cargo build -p supercov && node scripts/engine-contract.mjs",
|
|
50
67
|
"test:agent": "cargo build -p supercov && node scripts/agent-query-eval.mjs",
|
|
51
|
-
"test:engine": "cargo fmt --all -- --check && cargo clippy --workspace --all-targets -- -D warnings && cargo test --workspace && cargo build -p supercov && npm run test:runtime && npm run test:rust-assets && node scripts/rust-process-supervision.mjs && node scripts/rust-direct-node-integration.mjs && node scripts/rust-public-run-integration.mjs && node scripts/rust-embedded-runtime-integration.mjs && node scripts/rust-direct-vitest-integration.mjs && node scripts/rust-direct-playwright-integration.mjs && node scripts/rust-custom-browser-playwright-integration.mjs && node scripts/rust-generic-esbuild-integration.mjs && node scripts/rust-generic-tsc-integration.mjs && node scripts/rust-generic-build-matrix.mjs && node scripts/rust-vite-playwright-integration.mjs",
|
|
68
|
+
"test:engine": "cargo fmt --all -- --check && cargo clippy --workspace --all-targets -- -D warnings && cargo test --workspace && cargo build -p supercov && npm run test:runtime && npm run test:rust-assets && node scripts/rust-process-supervision.mjs && node scripts/rust-direct-node-integration.mjs && node scripts/rust-public-run-integration.mjs && node scripts/rust-embedded-runtime-integration.mjs && node scripts/rust-direct-vitest-integration.mjs && node scripts/rust-direct-jest-integration.mjs && node scripts/rust-direct-playwright-integration.mjs && node scripts/rust-custom-browser-playwright-integration.mjs && node scripts/rust-generic-esbuild-integration.mjs && node scripts/rust-generic-tsc-integration.mjs && node scripts/rust-generic-build-matrix.mjs && node scripts/rust-vite-playwright-integration.mjs",
|
|
52
69
|
"test:platform": "cargo test --workspace && cargo build -p supercov && node scripts/rust-process-supervision.mjs && node scripts/workspace-crash-integration.mjs",
|
|
53
|
-
"test:native-package": "cargo build --release -p supercov && node scripts/native-package-integration.mjs && node scripts/native-release-set-integration.mjs",
|
|
70
|
+
"test:native-package": "cargo build --release -p supercov && npm run build:asserted-typescript && node scripts/native-package-integration.mjs && node scripts/native-release-set-integration.mjs",
|
|
54
71
|
"test:pypi-wheel": "node scripts/pypi-wheel-integration.mjs",
|
|
55
72
|
"test:pypi-registry": "node scripts/pypi-registry-integration.mjs",
|
|
56
73
|
"build:rubygem": "cargo build --release -p supercov && node scripts/build-rubygem.mjs",
|
|
@@ -61,11 +78,12 @@
|
|
|
61
78
|
"test:test262": "cargo build --release -p supercov && node scripts/test262-equivalence.mjs",
|
|
62
79
|
"benchmark:check": "cargo build --release -p supercov && node scripts/rust-transform-benchmark.mjs",
|
|
63
80
|
"benchmark:python-monitoring": "cargo build -p supercov && node scripts/python-monitoring-benchmark.mjs",
|
|
64
|
-
"check": "cargo fmt --all -- --check && cargo clippy --workspace --all-targets -- -D warnings && npm run test && npm run test:runtime && npm run test:rust-assets && node scripts/package-preflight.mjs && node scripts/verify-binstall-metadata.mjs --offline",
|
|
81
|
+
"check": "cargo fmt --all -- --check && cargo clippy --workspace --all-targets -- -D warnings && npm run test && npm run test:runtime && npm run test:rust-assets && npm run test:asserted-typescript && npm run test:asserted-integration && npm run test:asserted-package && node scripts/package-preflight.mjs && node scripts/verify-binstall-metadata.mjs --offline",
|
|
65
82
|
"release:check": "cargo clean && node scripts/sweep-target.mjs && npm run check && npm run test:engine && npm run test:fixture && npm run test:watchdog && npm run test:engine-contract && npm run test:agent && npm run test:child-attribution && npm run test:host-loader && npm run test:python-monitoring && npm run test:ruby-coverage && npm run test:rust-public-cargo && npm run test:packed-npx && npm run test:clang-mcdc && npm run benchmark:check",
|
|
66
83
|
"release:bump": "node scripts/bump-version.mjs",
|
|
67
84
|
"sweep": "node scripts/sweep-target.mjs",
|
|
68
|
-
"
|
|
85
|
+
"oracle:rust": "cargo build --release -p supercov && node scripts/rust-coverage-oracle.mjs",
|
|
86
|
+
"prepack": "npm run build:asserted-typescript && node scripts/package-preflight.mjs",
|
|
69
87
|
"prepublishOnly": "npm run release:check",
|
|
70
88
|
"test:child-attribution": "cargo build -p supercov && node scripts/rust-child-attribution-integration.mjs",
|
|
71
89
|
"test:host-loader": "cargo build -p supercov && node scripts/rust-host-loader-integration.mjs",
|
|
@@ -74,14 +92,14 @@
|
|
|
74
92
|
"test:rust-public-cargo": "cargo build -p supercov && node scripts/rust-public-cargo-integration.mjs"
|
|
75
93
|
},
|
|
76
94
|
"optionalDependencies": {
|
|
77
|
-
"@supercov/cli-darwin-arm64": "0.0.
|
|
78
|
-
"@supercov/cli-darwin-x64": "0.0.
|
|
79
|
-
"@supercov/cli-linux-arm64-gnu": "0.0.
|
|
80
|
-
"@supercov/cli-linux-arm64-musl": "0.0.
|
|
81
|
-
"@supercov/cli-linux-x64-gnu": "0.0.
|
|
82
|
-
"@supercov/cli-linux-x64-musl": "0.0.
|
|
83
|
-
"@supercov/cli-win32-arm64": "0.0.
|
|
84
|
-
"@supercov/cli-win32-x64": "0.0.
|
|
95
|
+
"@supercov/cli-darwin-arm64": "0.0.43",
|
|
96
|
+
"@supercov/cli-darwin-x64": "0.0.43",
|
|
97
|
+
"@supercov/cli-linux-arm64-gnu": "0.0.43",
|
|
98
|
+
"@supercov/cli-linux-arm64-musl": "0.0.43",
|
|
99
|
+
"@supercov/cli-linux-x64-gnu": "0.0.43",
|
|
100
|
+
"@supercov/cli-linux-x64-musl": "0.0.43",
|
|
101
|
+
"@supercov/cli-win32-arm64": "0.0.43",
|
|
102
|
+
"@supercov/cli-win32-x64": "0.0.43"
|
|
85
103
|
},
|
|
86
104
|
"peerDependencies": {
|
|
87
105
|
"@playwright/test": ">=1.55.0",
|