supercov 0.0.54 โ 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +62 -5
- package/docs/cli.md +62 -0
- package/docs/getting-started.md +10 -0
- package/docs/quality.md +206 -0
- package/docs/supported-suites.md +24 -3
- package/package.json +13 -10
- package/runtime/javascript/jest.cjs +37 -7
- package/runtime/javascript/jest.config.mjs +10 -1
- package/runtime/javascript/jestReporter.mjs +13 -2
- package/runtime/javascript/jestRuntime.cjs +12 -0
- package/runtime/javascript/register.mjs +9 -3
- package/runtime/javascript/vitestBrowser.mjs +22 -0
- package/runtime/javascript/vitestReporter.mjs +9 -0
package/README.md
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
|
-

|
|
2
2
|
|
|
3
|
-
**
|
|
3
|
+
**Code quality and coverage for coding agents**
|
|
4
4
|
|
|
5
|
-
**Supercov
|
|
5
|
+
**Supercov tells your coding agent what to fix and what to test.** It scores your code quality with [Jev](https://typesafe.ai), runs the test command you already use, and turns uncovered paths into small, actionable queries. Your agent picks a target, writes a focused test or a focused refactor, proves what improved, and keeps going.
|
|
6
6
|
|
|
7
|
-
|
|
7
|
+
Coverage needs no account, config file, import, custom reporter, or hosted service. Supercov is local, free, open source, and MIT licensed. Quality uses [Jev](https://typesafe.ai) and needs a key.
|
|
8
8
|
|
|
9
9
|
[Website](https://supercov.com) ยท [Documentation](https://supercov.com/docs) ยท [npm](https://www.npmjs.com/package/supercov) ยท [GitHub](https://github.com/supercorp-ai/supercov)
|
|
10
10
|
|
|
@@ -13,6 +13,41 @@ Supported by [Supercorp](https://supercorp.ai).
|
|
|
13
13
|
[Agent workflow](https://supercov.com/docs/agent-loop): ask your coding agent
|
|
14
14
|
to add a test in your own project, with a recorded example to follow along.
|
|
15
15
|
|
|
16
|
+
## Score your code
|
|
17
|
+
|
|
18
|
+
```bash
|
|
19
|
+
export TYPESAFE_API_KEY=... # get one at https://typesafe.ai
|
|
20
|
+
npx supercov quality
|
|
21
|
+
```
|
|
22
|
+
|
|
23
|
+
No arguments and no configuration. It finds your source, asks Jev a set of
|
|
24
|
+
yes/no questions about each file, and does the arithmetic itself, so every part
|
|
25
|
+
of a score is a claim you can check against the file.
|
|
26
|
+
|
|
27
|
+
```
|
|
28
|
+
Quality weak (3.8/10) over 163 files.
|
|
29
|
+
11 good, 52 fair, 100 weak.
|
|
30
|
+
|
|
31
|
+
Weakest:
|
|
32
|
+
weak runtime/python/supercov_runtime.py
|
|
33
|
+
long_method 0.96, deep_nesting 0.92, complex_conditional 0.89, +9 more
|
|
34
|
+
weak crates/supercov-engine/src/assertion_store.rs
|
|
35
|
+
long_method 0.96, deep_nesting 0.91, complex_conditional 0.86, +8 more
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
Narrow to what fired, read one file in full, or review a change:
|
|
39
|
+
|
|
40
|
+
```bash
|
|
41
|
+
npx supercov quality gaps
|
|
42
|
+
npx supercov quality file src/server.ts
|
|
43
|
+
npx supercov quality patch # your uncommitted work, or your branch
|
|
44
|
+
npx supercov quality patch --annotate github # workflow annotations, no token
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
Jev charges for what it reads and nothing for what it writes, so a megabyte of
|
|
48
|
+
source costs a little over a cent. Answers are cached by content, so a second
|
|
49
|
+
run pays only for what changed. See [Understanding quality](docs/quality.md).
|
|
50
|
+
|
|
16
51
|
## Start with the suite you already have
|
|
17
52
|
|
|
18
53
|
```bash
|
|
@@ -25,12 +60,33 @@ github.com/supercorp-ai/supercov/cmd/supercov@latest`.
|
|
|
25
60
|
|
|
26
61
|
Everything after `--` is your test command. Supercov runs it without changing your source, tests, runner configuration, or normal build output.
|
|
27
62
|
|
|
28
|
-
Then ask what is still uncovered:
|
|
63
|
+
Then read the result and ask what is still uncovered:
|
|
64
|
+
|
|
65
|
+
```bash
|
|
66
|
+
npx supercov runs latest
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
```
|
|
70
|
+
run run_7fc676ba671d42be
|
|
71
|
+
command: npm test
|
|
72
|
+
|
|
73
|
+
Coverage
|
|
74
|
+
Lines 100.00% (5/5)
|
|
75
|
+
Branches 66.67% (4/6)
|
|
76
|
+
MC/DC 33.33% (1/3)
|
|
77
|
+
```
|
|
29
78
|
|
|
30
79
|
```bash
|
|
31
80
|
npx supercov runs latest gaps --limit 10
|
|
32
81
|
```
|
|
33
82
|
|
|
83
|
+
```
|
|
84
|
+
Coverage gaps โ only files with unresolved obligations
|
|
85
|
+
|
|
86
|
+
src/pricing.js
|
|
87
|
+
uncovered: lines 0 statements 1 functions 0 branch outcomes 0 MC/DC conditions 2
|
|
88
|
+
```
|
|
89
|
+
|
|
34
90
|
After your agent adds a test, rerun the complete suite and prove the gain:
|
|
35
91
|
|
|
36
92
|
```bash
|
|
@@ -243,6 +299,7 @@ npx supercov clean # remove all runs and the build cache
|
|
|
243
299
|
## Documentation
|
|
244
300
|
|
|
245
301
|
- [Getting started](https://supercov.com/docs/getting-started)
|
|
302
|
+
- [Understanding quality](https://supercov.com/docs/quality)
|
|
246
303
|
- [Agent workflow](https://supercov.com/docs/agent-loop)
|
|
247
304
|
- [Understanding assertions](docs/assertions.md)
|
|
248
305
|
- [Troubleshooting](https://supercov.com/docs/troubleshooting)
|
package/docs/cli.md
CHANGED
|
@@ -24,8 +24,70 @@ npx supercov --help
|
|
|
24
24
|
| Find the tests a change affects | `npx supercov runs latest tests affected` |
|
|
25
25
|
| Combine shards | `npx supercov merge <id> <id> [...]` |
|
|
26
26
|
| Remove local data | `npx supercov clean` |
|
|
27
|
+
| Assess code quality with Jev | `npx supercov quality` |
|
|
28
|
+
| See only files with findings | `npx supercov quality gaps` |
|
|
29
|
+
| Review what a change introduced | `npx supercov quality patch` |
|
|
27
30
|
| Read bundled guides | `npx supercov docs` |
|
|
28
31
|
|
|
32
|
+
## Assess source quality
|
|
33
|
+
|
|
34
|
+
```sh supercov-example
|
|
35
|
+
supercov quality # this repository
|
|
36
|
+
supercov quality src/ # one directory
|
|
37
|
+
supercov quality gaps # only files something fired on
|
|
38
|
+
supercov quality file src/a.ts # one file, every check
|
|
39
|
+
supercov quality scope # which files are assessed, and why
|
|
40
|
+
supercov quality snapshots # saved assessments
|
|
41
|
+
supercov quality diff <older> <newer>
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
With no argument the subject is the repository you are standing in. Every
|
|
45
|
+
assessment saves a snapshot, so the reading commands work afterwards with no key
|
|
46
|
+
and no network.
|
|
47
|
+
|
|
48
|
+
Twelve yes/no questions about named code properties go to [Jev](https://typesafe.ai);
|
|
49
|
+
the score is arithmetic this command does over the answers. Text reports a band,
|
|
50
|
+
`good`, `fair` or `weak`; `--json` carries the number and every check with what
|
|
51
|
+
is known about it. `--all` includes test files, generated output and anything
|
|
52
|
+
outside a source root, all of which are left out by default. `--dry-run` prints
|
|
53
|
+
the exact requests and contacts nothing.
|
|
54
|
+
|
|
55
|
+
`quality diff` reports what declined between two assessments: which files lost
|
|
56
|
+
health, which properties appeared, and which files entered or left the scope.
|
|
57
|
+
|
|
58
|
+
Assessing needs a TypeSafe API key in `TYPESAFE_API_KEY`; reading a saved
|
|
59
|
+
assessment does not. The command prints a cost estimate before sending anything
|
|
60
|
+
and caches answers by content, so a second run pays only for what changed.
|
|
61
|
+
|
|
62
|
+
See [Understanding quality](https://supercov.com/docs/quality) for what the
|
|
63
|
+
number is worth and which files get assessed.
|
|
64
|
+
|
|
65
|
+
## Review what a change introduced
|
|
66
|
+
|
|
67
|
+
```sh supercov-example
|
|
68
|
+
supercov quality patch
|
|
69
|
+
supercov quality patch --base origin/main
|
|
70
|
+
supercov quality patch --base origin/main --annotate github --run latest
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
The same twelve properties asked of a change, plus six risk checks that only
|
|
74
|
+
apply to one: a credential in source, untrusted input in a query, a change to
|
|
75
|
+
who may do what, a test that now checks less, a schema migration, and debugging
|
|
76
|
+
left behind.
|
|
77
|
+
|
|
78
|
+
With no range it reviews uncommitted work when the tree is dirty and everything
|
|
79
|
+
since this branch left its default branch when it is clean. `--unstaged`,
|
|
80
|
+
`--staged` and `--base <ref>` say so explicitly; `--base` uses the merge base,
|
|
81
|
+
like `runs patch`, so commits other people landed after you branched are not
|
|
82
|
+
your change.
|
|
83
|
+
|
|
84
|
+
`--annotate github` prints workflow annotations on stdout, needing no token and
|
|
85
|
+
posting no comment. `--run <id>` reads a saved coverage run and marks any file
|
|
86
|
+
where a property appeared and the run left lines uncovered.
|
|
87
|
+
|
|
88
|
+
Output lists only files where something appeared. A change that introduces
|
|
89
|
+
nothing prints one line saying so. About $0.0005 per changed file.
|
|
90
|
+
|
|
29
91
|
## Measure a test command
|
|
30
92
|
|
|
31
93
|
```sh supercov
|
package/docs/getting-started.md
CHANGED
|
@@ -12,6 +12,16 @@ added and the before-and-after coverage.
|
|
|
12
12
|
Your agent can install Supercov if needed. It runs the commands and edits the
|
|
13
13
|
tests; you don't need to do those steps yourself.
|
|
14
14
|
|
|
15
|
+
To score the code itself rather than its tests, set a [Jev](https://typesafe.ai)
|
|
16
|
+
key and ask:
|
|
17
|
+
|
|
18
|
+
```sh supercov-example
|
|
19
|
+
export TYPESAFE_API_KEY=...
|
|
20
|
+
npx supercov quality
|
|
21
|
+
```
|
|
22
|
+
|
|
23
|
+
See [Understanding quality](https://supercov.com/docs/quality).
|
|
24
|
+
|
|
15
25
|
When it finishes, review the test change and coverage comparison in your
|
|
16
26
|
conversation. Ask separately if you want a commit or pull request.
|
|
17
27
|
|
package/docs/quality.md
ADDED
|
@@ -0,0 +1,206 @@
|
|
|
1
|
+
# Understanding quality
|
|
2
|
+
|
|
3
|
+
`supercov quality` tells you what is in your code: twelve named properties,
|
|
4
|
+
checked file by file, so a score is never a number you have to take on faith.
|
|
5
|
+
|
|
6
|
+
Judgments come from [Jev](https://typesafe.ai), which answers typed questions
|
|
7
|
+
rather than generating prose. Supercov asks the questions and does the
|
|
8
|
+
arithmetic, so every part of a score is a claim you can check against the file.
|
|
9
|
+
|
|
10
|
+
## Set your key
|
|
11
|
+
|
|
12
|
+
Assessing needs a TypeSafe API key, read from `TYPESAFE_API_KEY`. Get one at
|
|
13
|
+
[typesafe.ai](https://typesafe.ai).
|
|
14
|
+
|
|
15
|
+
```bash
|
|
16
|
+
export TYPESAFE_API_KEY=... # for this shell
|
|
17
|
+
TYPESAFE_API_KEY=... npx supercov quality # for one command
|
|
18
|
+
```
|
|
19
|
+
|
|
20
|
+
The environment variable is the only way to pass it. A key given on the command
|
|
21
|
+
line ends up in your shell history and in the process list, where anyone on the
|
|
22
|
+
machine can read it.
|
|
23
|
+
|
|
24
|
+
Set it the way your environment already sets secrets:
|
|
25
|
+
|
|
26
|
+
| Where | How |
|
|
27
|
+
| --- | --- |
|
|
28
|
+
| GitHub Actions | `env: { TYPESAFE_API_KEY: ${{ secrets.TYPESAFE_API_KEY }} }` |
|
|
29
|
+
| GitLab CI | a masked CI/CD variable named `TYPESAFE_API_KEY` |
|
|
30
|
+
| Docker | `docker run -e TYPESAFE_API_KEY ...` |
|
|
31
|
+
| Local development | a `.env` loaded by `direnv`, `dotenv` or your shell profile |
|
|
32
|
+
|
|
33
|
+
Reading a saved assessment never needs a key, and `--dry-run` prints the exact
|
|
34
|
+
requests without sending them.
|
|
35
|
+
|
|
36
|
+
## Start with the repository
|
|
37
|
+
|
|
38
|
+
```bash
|
|
39
|
+
npx supercov quality
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
No arguments, no configuration. It finds your source, asks twelve questions of
|
|
43
|
+
each file, and saves a snapshot you can read afterwards without a key or a
|
|
44
|
+
network.
|
|
45
|
+
|
|
46
|
+
```
|
|
47
|
+
Quality fair (5.0/10) over 163 files.
|
|
48
|
+
11 good, 53 fair, 99 weak.
|
|
49
|
+
|
|
50
|
+
Weakest:
|
|
51
|
+
weak src/lib/modernHttp.ts
|
|
52
|
+
long_method 0.93, duplicated_logic 0.88, complex_conditional 0.87, +7 more
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
`good` is 8 and above, `fair` is 5 to 8, `weak` is below 5. `--json` carries the
|
|
56
|
+
number when something needs to sort.
|
|
57
|
+
|
|
58
|
+
## The twelve properties
|
|
59
|
+
|
|
60
|
+
god class, long method, deep nesting, complex conditional, long parameter list,
|
|
61
|
+
duplicated logic, primitive obsession, dead code, feature envy, temporary field,
|
|
62
|
+
message chains, magic values.
|
|
63
|
+
|
|
64
|
+
They come from Fowler and Beck's refactoring smells and the class-scope smells
|
|
65
|
+
CodeScene's Code Health is built from. Each is a yes/no question with a
|
|
66
|
+
definition and a stated exception, so two careful readers would agree on the
|
|
67
|
+
answer.
|
|
68
|
+
|
|
69
|
+
Three of them fire on more than half the files in a typical repository, so the
|
|
70
|
+
summary shows the three strongest per file. To see all twelve with what is known
|
|
71
|
+
about each:
|
|
72
|
+
|
|
73
|
+
```bash
|
|
74
|
+
npx supercov quality file src/lib/modernHttp.ts
|
|
75
|
+
npx supercov quality gaps # only files something fired on
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
## Reading the score
|
|
79
|
+
|
|
80
|
+
Health is the mean of the twelve answers, done by this command rather than by
|
|
81
|
+
the model. A directory or a whole repository counts its larger files for more,
|
|
82
|
+
so a folder of one-line re-exports cannot outvote the file everything depends
|
|
83
|
+
on.
|
|
84
|
+
|
|
85
|
+
Use it to find the code that is hardest to change, and `quality gaps` to jump
|
|
86
|
+
straight to the files something fired on.
|
|
87
|
+
|
|
88
|
+
## What a change introduced
|
|
89
|
+
|
|
90
|
+
```bash
|
|
91
|
+
npx supercov quality patch
|
|
92
|
+
```
|
|
93
|
+
|
|
94
|
+
With no range it reviews your uncommitted work when the tree is dirty, and
|
|
95
|
+
everything since your branch left its default branch when it is clean. Say so
|
|
96
|
+
explicitly with `--unstaged`, `--staged`, or `--base origin/main`, which uses
|
|
97
|
+
the merge base so commits other people landed after you branched are not
|
|
98
|
+
counted as yours.
|
|
99
|
+
|
|
100
|
+
It asks the twelve properties differentially, whether the new version shows
|
|
101
|
+
something the old one did not, and adds six checks that only make sense for a
|
|
102
|
+
change:
|
|
103
|
+
|
|
104
|
+
- a credential written into source
|
|
105
|
+
- untrusted input interpolated into a query
|
|
106
|
+
- a change to how the system decides who may do what
|
|
107
|
+
- a test that now checks less than it did
|
|
108
|
+
- a database schema or data migration
|
|
109
|
+
- debugging left behind
|
|
110
|
+
|
|
111
|
+
Output lists only files where something appeared. A change that introduces
|
|
112
|
+
nothing says so in one line.
|
|
113
|
+
|
|
114
|
+
```bash
|
|
115
|
+
npx supercov quality patch --base origin/main --annotate github
|
|
116
|
+
```
|
|
117
|
+
|
|
118
|
+
`--annotate github` prints workflow annotations on stdout. It needs no token and
|
|
119
|
+
posts no comment.
|
|
120
|
+
|
|
121
|
+
## Which files get looked at
|
|
122
|
+
|
|
123
|
+
```bash
|
|
124
|
+
npx supercov quality scope
|
|
125
|
+
```
|
|
126
|
+
|
|
127
|
+
Source roots come from your package manifests and from what they declare, so a
|
|
128
|
+
`bin` or `exports` entry counts even when it is not in a conventional directory.
|
|
129
|
+
Test files, generated output, tool scripts, examples, benchmarks and
|
|
130
|
+
documentation are left out, each with its reason. `--all` includes everything.
|
|
131
|
+
|
|
132
|
+
If some of your code sits somewhere none of that recognises, Supercov asks Jev
|
|
133
|
+
about those paths with your whole tree as context, and says how many it decided
|
|
134
|
+
that way. To decide yourself:
|
|
135
|
+
|
|
136
|
+
```bash
|
|
137
|
+
SUPERCOV_SOURCE_ROOTS=src,packages npx supercov quality
|
|
138
|
+
```
|
|
139
|
+
|
|
140
|
+
A declaration is never second-guessed.
|
|
141
|
+
|
|
142
|
+
## Tracking it over time
|
|
143
|
+
|
|
144
|
+
```bash
|
|
145
|
+
npx supercov quality snapshots
|
|
146
|
+
npx supercov quality diff <older> <newer>
|
|
147
|
+
```
|
|
148
|
+
|
|
149
|
+
Which files lost health, which gained, which properties appeared, and which
|
|
150
|
+
files entered or left the scope. A file whose contents did not change is marked,
|
|
151
|
+
so a small movement does not send you looking for an edit that was never made.
|
|
152
|
+
|
|
153
|
+
## What it costs
|
|
154
|
+
|
|
155
|
+
Jev charges for what it reads and nothing for what it writes, at $42 per billion
|
|
156
|
+
input tokens. What that means in practice, measured rather than estimated:
|
|
157
|
+
|
|
158
|
+
| | source | cost |
|
|
159
|
+
| --- | ---: | ---: |
|
|
160
|
+
| Supercov's CLI crate | 0.5 MB | $0.007 |
|
|
161
|
+
| a TypeScript gateway, 197 files | 0.9 MB | $0.02 |
|
|
162
|
+
| one changed file in a review | โ | $0.0005 |
|
|
163
|
+
|
|
164
|
+
About a cent per megabyte of source, a little more when the files are small,
|
|
165
|
+
because each one carries its own questions.
|
|
166
|
+
|
|
167
|
+
The command prints its estimate before sending anything, so a number that looks
|
|
168
|
+
wrong can be stopped rather than discovered on an invoice:
|
|
169
|
+
|
|
170
|
+
```
|
|
171
|
+
[supercov] quality: 35 requests, about 111840 input tokens ($0.0047) if none is cached
|
|
172
|
+
```
|
|
173
|
+
|
|
174
|
+
**Most runs cost far less than that estimate.** Answers are cached by content
|
|
175
|
+
under `.supercov/quality/requests/`, so a second run pays only for files that
|
|
176
|
+
actually changed. The cache follows content rather than paths, which means
|
|
177
|
+
switching branches, rebasing or checking out an old commit reuses everything
|
|
178
|
+
unchanged: assessing the same directory 400 commits back in a real repository
|
|
179
|
+
answered every file from cache and sent nothing.
|
|
180
|
+
|
|
181
|
+
Three ways to spend less:
|
|
182
|
+
|
|
183
|
+
- **Assess a directory, not the tree**, while you are iterating:
|
|
184
|
+
`npx supercov quality src/api`.
|
|
185
|
+
- **Review the change, not the repository**, in CI: `npx supercov quality patch`
|
|
186
|
+
costs about $0.0005 per changed file, so a typical pull request is a fraction
|
|
187
|
+
of a cent.
|
|
188
|
+
- **Keep `.supercov/` between CI runs** if your runner supports a cache. An
|
|
189
|
+
unchanged file then costs nothing on every run after the first.
|
|
190
|
+
|
|
191
|
+
`--refresh` asks again and bypasses the cache. `--dry-run` prints the exact
|
|
192
|
+
requests, sends nothing and costs nothing.
|
|
193
|
+
|
|
194
|
+
## Reference
|
|
195
|
+
|
|
196
|
+
```bash
|
|
197
|
+
npx supercov quality # this repository
|
|
198
|
+
npx supercov quality gaps # only files something fired on
|
|
199
|
+
npx supercov quality file src/server.ts # one file, every check
|
|
200
|
+
npx supercov quality scope # which files, and why
|
|
201
|
+
npx supercov quality snapshots # saved assessments
|
|
202
|
+
npx supercov quality diff <older> <newer> # what declined
|
|
203
|
+
npx supercov quality patch # what a change introduced
|
|
204
|
+
```
|
|
205
|
+
|
|
206
|
+
Full options are in the [CLI reference](https://supercov.com/docs/cli).
|
package/docs/supported-suites.md
CHANGED
|
@@ -63,9 +63,10 @@ one run and preserves runner identity wherever the runner exposes it.
|
|
|
63
63
|
|
|
64
64
|
Vitest Browser Mode runs the test file in a real browser. Supercov measures it
|
|
65
65
|
per test like any other Vitest run: lines, branches, MC/DC and assertion
|
|
66
|
-
coverage
|
|
67
|
-
|
|
68
|
-
|
|
66
|
+
coverage. Combined Node/browser projects are configured after Vitest resolves
|
|
67
|
+
inline definitions, config-file references, globs and project selection.
|
|
68
|
+
Evidence travels over Vitest's own browser command channel; the compatibility
|
|
69
|
+
suite exercises Chromium through the Playwright provider.
|
|
69
70
|
|
|
70
71
|
Component code is instrumented the same way as any other source, with one
|
|
71
72
|
addition that matters most here. A JSX tree is a single statement, so an
|
|
@@ -83,6 +84,26 @@ where it is created.
|
|
|
83
84
|
`expect.element(...)`, `expect.soft(...)` and `expect.poll(...)` are recognised
|
|
84
85
|
as assertions, so their passing occurrences are available to assertion maps.
|
|
85
86
|
|
|
87
|
+
### React and React Native
|
|
88
|
+
|
|
89
|
+
React components can use Testing Library with Vitest/jsdom, Vitest Browser Mode,
|
|
90
|
+
or Babel/Jest. React Native and Expo component tests run through their existing
|
|
91
|
+
Jest presets, including native mocks. This is JavaScript component-test coverage:
|
|
92
|
+
it does not measure Hermes, native modules, simulator/device execution, Detox or
|
|
93
|
+
Maestro. React SSR hydration is exercised in jsdom and Chromium; this does not
|
|
94
|
+
establish Next.js Server Components, streaming SSR or server actions.
|
|
95
|
+
|
|
96
|
+
An assertion map can explain which displayed value, accessible name, disabled
|
|
97
|
+
state or error message a test checks. JSX expression coverage and a passing
|
|
98
|
+
assertion are evidence for reviewing that explanation, not automatic semantic
|
|
99
|
+
proof. A button being rendered does not establish that its disabled state was
|
|
100
|
+
checked. Keep your existing runner and matchers.
|
|
101
|
+
|
|
102
|
+
The [React verification example](https://github.com/supercorp-ai/supercov/tree/main/examples/react-verification)
|
|
103
|
+
shows a fully executed checkout whose weak tests accept four UI regressions,
|
|
104
|
+
then adds four precise assertions and independently checks the broken copies.
|
|
105
|
+
The agent-authored maps credit only those four UI expressions.
|
|
106
|
+
|
|
86
107
|
### Builds and source formats
|
|
87
108
|
|
|
88
109
|
JavaScript and TypeScript projects may use Vite, Next, Turbopack, Webpack,
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "supercov",
|
|
3
|
-
"version": "0.0
|
|
4
|
-
"description": "
|
|
3
|
+
"version": "1.0.0",
|
|
4
|
+
"description": "Code quality and coverage for coding agents",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"repository": {
|
|
7
7
|
"type": "git",
|
|
@@ -18,6 +18,7 @@
|
|
|
18
18
|
"docs/getting-started.md",
|
|
19
19
|
"docs/agent-loop.md",
|
|
20
20
|
"docs/assertions.md",
|
|
21
|
+
"docs/quality.md",
|
|
21
22
|
"docs/assertion-evidence.md",
|
|
22
23
|
"docs/assertion-maps.md",
|
|
23
24
|
"docs/assertion-agent.md",
|
|
@@ -54,6 +55,7 @@
|
|
|
54
55
|
"test:rustc-backend-spike": "node scripts/rustc-backend-spike.mjs",
|
|
55
56
|
"test:rust-compiler-spikes": "cargo build -p supercov && node scripts/rust-libtest-companion-spike.mjs && node scripts/rust-async-attribution-spike.mjs && node scripts/rust-subprocess-attribution-spike.mjs && node scripts/rust-custom-harness-spike.mjs && node scripts/rust-libtest-builder-lifecycle-spike.mjs",
|
|
56
57
|
"test": "cargo test --workspace",
|
|
58
|
+
"test:react": "cargo build -p supercov && node scripts/rust-vitest-projects-integration.mjs && npm --prefix examples/react-verification ci && npm --prefix examples/react-verification run demo && node scripts/react-hydration-compatibility.mjs",
|
|
57
59
|
"test:runtime": "node --test tests/runtime/*.test.mjs",
|
|
58
60
|
"test:fixture": "cargo build -p supercov && node scripts/rust-fixture-matrix.mjs",
|
|
59
61
|
"test:packed-npx": "cargo build --release -p supercov && node scripts/packed-npx-integration.mjs",
|
|
@@ -97,14 +99,14 @@
|
|
|
97
99
|
"test:launcher": "go vet ./cmd/... && go test ./cmd/..."
|
|
98
100
|
},
|
|
99
101
|
"optionalDependencies": {
|
|
100
|
-
"@supercov/cli-darwin-arm64": "0.0
|
|
101
|
-
"@supercov/cli-darwin-x64": "0.0
|
|
102
|
-
"@supercov/cli-linux-arm64-gnu": "0.0
|
|
103
|
-
"@supercov/cli-linux-arm64-musl": "0.0
|
|
104
|
-
"@supercov/cli-linux-x64-gnu": "0.0
|
|
105
|
-
"@supercov/cli-linux-x64-musl": "0.0
|
|
106
|
-
"@supercov/cli-win32-arm64": "0.0
|
|
107
|
-
"@supercov/cli-win32-x64": "0.0
|
|
102
|
+
"@supercov/cli-darwin-arm64": "1.0.0",
|
|
103
|
+
"@supercov/cli-darwin-x64": "1.0.0",
|
|
104
|
+
"@supercov/cli-linux-arm64-gnu": "1.0.0",
|
|
105
|
+
"@supercov/cli-linux-arm64-musl": "1.0.0",
|
|
106
|
+
"@supercov/cli-linux-x64-gnu": "1.0.0",
|
|
107
|
+
"@supercov/cli-linux-x64-musl": "1.0.0",
|
|
108
|
+
"@supercov/cli-win32-arm64": "1.0.0",
|
|
109
|
+
"@supercov/cli-win32-x64": "1.0.0"
|
|
108
110
|
},
|
|
109
111
|
"peerDependencies": {
|
|
110
112
|
"@playwright/test": ">=1.55.0",
|
|
@@ -126,6 +128,7 @@
|
|
|
126
128
|
"@playwright/test": "1.62.1",
|
|
127
129
|
"@swc/core": "^1.16.1",
|
|
128
130
|
"@types/node": "^24.0.0",
|
|
131
|
+
"@vitest/browser-playwright": "4.1.11",
|
|
129
132
|
"esbuild": "^0.28.2",
|
|
130
133
|
"expect": "30.4.1",
|
|
131
134
|
"jest": "30.5.1",
|
|
@@ -28,8 +28,32 @@ function localFile(path) {
|
|
|
28
28
|
return relative(process.cwd(), path).split(sep).join("/");
|
|
29
29
|
}
|
|
30
30
|
// Must match jestReporter.mjs: the reporter has the same file and full name.
|
|
31
|
-
function testIdentity(testFile, fullName) {
|
|
32
|
-
return `jest:${digest(`${testFile}\0${fullName}`)}`;
|
|
31
|
+
function testIdentity(testFile, fullName, occurrence = 0) {
|
|
32
|
+
return `jest:${digest(`${testFile}\0${fullName}${occurrence ? `\0${occurrence}` : ""}`)}`;
|
|
33
|
+
}
|
|
34
|
+
// Circus keeps its declaration tree on the environment global. It distinguishes
|
|
35
|
+
// identical titles and survives retries; expect.currentTestName alone cannot.
|
|
36
|
+
function currentDeclaration() {
|
|
37
|
+
const symbol = Object.getOwnPropertySymbols(globalThis)
|
|
38
|
+
.find(key => key.description === "JEST_STATE_SYMBOL");
|
|
39
|
+
const state = symbol && globalThis[symbol];
|
|
40
|
+
if (!state?.currentlyRunningTest || !state.rootDescribeBlock) return undefined;
|
|
41
|
+
const names = new Map();
|
|
42
|
+
let found;
|
|
43
|
+
function visit(block, parents) {
|
|
44
|
+
for (const child of block.children ?? []) {
|
|
45
|
+
if (child.type === "describeBlock") visit(child, [...parents, child.name]);
|
|
46
|
+
else if (child.type === "test") {
|
|
47
|
+
const name = [...parents, child.name].join(" ");
|
|
48
|
+
const occurrence = names.get(name) ?? 0;
|
|
49
|
+
names.set(name, occurrence + 1);
|
|
50
|
+
if (child === state.currentlyRunningTest)
|
|
51
|
+
found = {occurrence, retry: Math.max((child.invocations ?? 1) - 1, 0)};
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
visit(state.rootDescribeBlock, []);
|
|
56
|
+
return found;
|
|
33
57
|
}
|
|
34
58
|
// Mirrors provenance.mjs, which is ESM and out of reach here.
|
|
35
59
|
const KINDS = [
|
|
@@ -56,7 +80,7 @@ function writeEvidence(suffix, payload) {
|
|
|
56
80
|
renameSync(temporary, target);
|
|
57
81
|
}
|
|
58
82
|
|
|
59
|
-
if (runtime && evidenceDirectory && typeof beforeEach === "function" && typeof
|
|
83
|
+
if (runtime && evidenceDirectory && typeof beforeEach === "function" && typeof afterAll === "function") {
|
|
60
84
|
// One in-memory snapshot per test; the server JSONL transport would be
|
|
61
85
|
// redundant and unattributed, as under Vitest.
|
|
62
86
|
runtime.enableRuntimeSnapshotEvidence();
|
|
@@ -64,6 +88,7 @@ if (runtime && evidenceDirectory && typeof beforeEach === "function" && typeof a
|
|
|
64
88
|
const emittedSetupFiles = new Set();
|
|
65
89
|
let active;
|
|
66
90
|
beforeEach(() => {
|
|
91
|
+
flushActive();
|
|
67
92
|
const state = expect.getState();
|
|
68
93
|
const testFile = localFile(state.testPath ?? "unknown");
|
|
69
94
|
if (!emittedSetupFiles.has(testFile)) {
|
|
@@ -87,8 +112,9 @@ if (runtime && evidenceDirectory && typeof beforeEach === "function" && typeof a
|
|
|
87
112
|
}
|
|
88
113
|
}
|
|
89
114
|
const fullName = state.currentTestName ?? "test";
|
|
90
|
-
const
|
|
91
|
-
const
|
|
115
|
+
const declaration = currentDeclaration();
|
|
116
|
+
const testId = testIdentity(testFile, fullName, declaration?.occurrence);
|
|
117
|
+
const retry = declaration?.retry ?? attempts.get(testId) ?? 0;
|
|
92
118
|
attempts.set(testId, retry + 1);
|
|
93
119
|
const testKey = digest(testId);
|
|
94
120
|
const scope = {
|
|
@@ -104,7 +130,10 @@ if (runtime && evidenceDirectory && typeof beforeEach === "function" && typeof a
|
|
|
104
130
|
runtime.activateCoverageScope(scope);
|
|
105
131
|
runtime.resetCoverage(testId);
|
|
106
132
|
});
|
|
107
|
-
afterEach(
|
|
133
|
+
// Flush after all user afterEach hooks (including Testing Library cleanup).
|
|
134
|
+
// Our setup registers first, so an afterEach snapshot would run too early.
|
|
135
|
+
// The next beforeEach or this file's afterAll runs after test teardown.
|
|
136
|
+
function flushActive() {
|
|
108
137
|
const current = active;
|
|
109
138
|
active = undefined;
|
|
110
139
|
if (!current)
|
|
@@ -130,5 +159,6 @@ if (runtime && evidenceDirectory && typeof beforeEach === "function" && typeof a
|
|
|
130
159
|
// snapshot (Vitest isolates files; Jest does not).
|
|
131
160
|
runtime.resetCoverage();
|
|
132
161
|
runtime.activateCoverageScope();
|
|
133
|
-
}
|
|
162
|
+
}
|
|
163
|
+
afterAll(flushActive);
|
|
134
164
|
}
|
|
@@ -30,7 +30,16 @@ export default async function supercovJestConfig() {
|
|
|
30
30
|
...config,
|
|
31
31
|
// Concurrent tests in one file would share the worker's scope.
|
|
32
32
|
maxConcurrency: 1,
|
|
33
|
-
|
|
33
|
+
setupFiles: [here("jestRuntime.cjs"), ...asList(config.setupFiles)],
|
|
34
|
+
// Babel-based presets (including React Native) lower the runtime
|
|
35
|
+
// side-effect import in instrumented tests to require(). Jest cannot
|
|
36
|
+
// require our ESM shim. Its environment already owns the runtime, so
|
|
37
|
+
// resolve only our generated import to a CommonJS bridge instead.
|
|
38
|
+
moduleNameMapper: {
|
|
39
|
+
"(?:^|/)\\.supercov/node_modules/runtime\\.mjs$": here("jestRuntime.cjs"),
|
|
40
|
+
...config.moduleNameMapper,
|
|
41
|
+
},
|
|
42
|
+
setupFilesAfterEnv: [here("jest.cjs"), ...asList(config.setupFilesAfterEnv)],
|
|
34
43
|
reporters: [
|
|
35
44
|
...(config.reporters === undefined ? ["default"] : asList(config.reporters)),
|
|
36
45
|
here("jestReporter.mjs"),
|
|
@@ -30,12 +30,23 @@ function attemptStatus(status) {
|
|
|
30
30
|
* record carries the attempt's coverage; the report joins the two.
|
|
31
31
|
*/
|
|
32
32
|
export default class SupercovJestReporter {
|
|
33
|
-
|
|
33
|
+
onTestResult(test, fileResult) {
|
|
34
|
+
// Jest's final array follows declaration order, including skipped cases.
|
|
35
|
+
// Completion order does not: retries can finish after a namesake test.
|
|
36
|
+
const occurrences = new Map();
|
|
37
|
+
for (const result of fileResult.testResults ?? []) {
|
|
38
|
+
const occurrence = occurrences.get(result.fullName) ?? 0;
|
|
39
|
+
occurrences.set(result.fullName, occurrence + 1);
|
|
40
|
+
this.recordResult(test, result, occurrence);
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
recordResult(test, result, occurrence) {
|
|
34
44
|
const evidenceDirectory = process.env["SUPERCOV_EVIDENCE_DIR"];
|
|
35
45
|
if (!evidenceDirectory)
|
|
36
46
|
return;
|
|
37
47
|
const testFile = localFile(test.path);
|
|
38
|
-
const
|
|
48
|
+
const identity = `${testFile}\0${result.fullName}${occurrence ? `\0${occurrence}` : ""}`;
|
|
49
|
+
const testId = `jest:${digest(identity)}`;
|
|
39
50
|
const retry = Math.max((result.invocations ?? 1) - 1, 0);
|
|
40
51
|
const status = attemptStatus(result.status);
|
|
41
52
|
const provenance = inferTestProvenance({
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
|
|
3
|
+
// Keep Babel-transformed test modules on the same runtime as the Jest adapter.
|
|
4
|
+
// Loading a second runtime here would split assertion and coverage evidence.
|
|
5
|
+
// jsdom owns a separate global and copies process without the preload's
|
|
6
|
+
// private property. Retrieve the host runtime through Node's VM API, then
|
|
7
|
+
// expose it before user setup files can import instrumented application code.
|
|
8
|
+
const runtime = globalThis.__SUPERCOV_DIRECT_RUNTIME__ ?? process.__SUPERCOV_DIRECT_RUNTIME__ ??
|
|
9
|
+
require("node:vm").runInThisContext("globalThis.__SUPERCOV_DIRECT_RUNTIME__");
|
|
10
|
+
if (!runtime) throw new Error("[supercov] Jest runtime was not initialized by the preload");
|
|
11
|
+
globalThis.__SUPERCOV_DIRECT_RUNTIME__ = runtime;
|
|
12
|
+
module.exports = runtime;
|
|
@@ -190,22 +190,28 @@ if (isJestEntrypoint && process.env.SUPERCOV_EVIDENCE_DIR) {
|
|
|
190
190
|
// Jest reads one configuration. Ours (jest.config.mjs) reads the user's
|
|
191
191
|
// the way Jest would and adds the adapter and reporter; an explicit
|
|
192
192
|
// --config on the command line reaches it through the environment.
|
|
193
|
+
const generatedJestConfig = fileURLToPath(new URL("./jest.config.mjs", import.meta.url));
|
|
193
194
|
for (let index = 2; index < process.argv.length; index += 1) {
|
|
194
195
|
const argument = process.argv[index];
|
|
195
196
|
if (argument === "--config" || argument === "-c") {
|
|
196
197
|
const value = process.argv[index + 1];
|
|
197
|
-
|
|
198
|
+
// Expo's Jest executable forwards argv to the real Jest process.
|
|
199
|
+
// Preserve the original config across that second preload; reading
|
|
200
|
+
// our own config as the user's would recurse until the heap fills.
|
|
201
|
+
if (value && resolve(value) !== resolve(generatedJestConfig))
|
|
198
202
|
process.env.SUPERCOV_ORIGINAL_JEST_CONFIG = resolve(value);
|
|
199
203
|
process.argv.splice(index, value ? 2 : 1);
|
|
200
204
|
index -= 1;
|
|
201
205
|
}
|
|
202
206
|
else if (argument?.startsWith("--config=")) {
|
|
203
|
-
|
|
207
|
+
const value = resolve(argument.slice("--config=".length));
|
|
208
|
+
if (value !== resolve(generatedJestConfig))
|
|
209
|
+
process.env.SUPERCOV_ORIGINAL_JEST_CONFIG = value;
|
|
204
210
|
process.argv.splice(index, 1);
|
|
205
211
|
index -= 1;
|
|
206
212
|
}
|
|
207
213
|
}
|
|
208
|
-
process.argv.push("--config",
|
|
214
|
+
process.argv.push("--config", generatedJestConfig);
|
|
209
215
|
}
|
|
210
216
|
if (generatedPlaywrightConfig &&
|
|
211
217
|
isPlaywrightEntrypoint) {
|
|
@@ -17,6 +17,7 @@ const browserContext = await import("vitest/browser").catch(() =>
|
|
|
17
17
|
const { commands } = browserContext;
|
|
18
18
|
const attempts = new Map();
|
|
19
19
|
const activeScopes = new Map();
|
|
20
|
+
const emittedSetupFiles = new Set();
|
|
20
21
|
enableRuntimeSnapshotEvidence();
|
|
21
22
|
function attemptStatus(state) {
|
|
22
23
|
if (state === "pass")
|
|
@@ -62,6 +63,27 @@ async function sendEvidence(payload, suffix) {
|
|
|
62
63
|
}
|
|
63
64
|
beforeEach(async (context) => {
|
|
64
65
|
const task = context.task;
|
|
66
|
+
// Module imports and shared setup execute before the first test. Save
|
|
67
|
+
// them before resetCoverage clears the snapshot, with a setup identity
|
|
68
|
+
// rather than crediting that execution to the first test.
|
|
69
|
+
if (!emittedSetupFiles.has(task.file.id)) {
|
|
70
|
+
const setupSnapshot = coverageSnapshot();
|
|
71
|
+
if (setupSnapshot.hits.length || setupSnapshot.decisions.length) {
|
|
72
|
+
await sendEvidence({
|
|
73
|
+
testId: `vitest:${task.file.id}:setup`,
|
|
74
|
+
test: `${task.file.name} > module setup`,
|
|
75
|
+
projectName: task.file.projectName,
|
|
76
|
+
title: "module setup",
|
|
77
|
+
retry: 0,
|
|
78
|
+
status: "passed",
|
|
79
|
+
role: "setup",
|
|
80
|
+
runtime: [setupSnapshot],
|
|
81
|
+
browser: [],
|
|
82
|
+
server: [],
|
|
83
|
+
}, `vitest-${task.file.id}-setup`);
|
|
84
|
+
}
|
|
85
|
+
emittedSetupFiles.add(task.file.id);
|
|
86
|
+
}
|
|
65
87
|
const testId = `vitest:${task.id}`;
|
|
66
88
|
const retry = attempts.get(testId) ?? 0;
|
|
67
89
|
attempts.set(testId, retry + 1);
|
|
@@ -25,6 +25,15 @@ function rawAttemptStatus(state, expectedFailure) {
|
|
|
25
25
|
/** Records final runner outcomes, including tests that never execute hooks. */
|
|
26
26
|
export default class SupercovVitestReporter {
|
|
27
27
|
reportedAttempts = new Set();
|
|
28
|
+
constructor(configureProjects) {
|
|
29
|
+
this.configureProjects = configureProjects;
|
|
30
|
+
}
|
|
31
|
+
onInit(vitest) {
|
|
32
|
+
this.configureProjects?.(vitest.projects);
|
|
33
|
+
}
|
|
34
|
+
onBrowserInit(project) {
|
|
35
|
+
this.configureProjects?.([project]);
|
|
36
|
+
}
|
|
28
37
|
onTestCaseResult(testCase) {
|
|
29
38
|
const evidenceDirectory = process.env["SUPERCOV_EVIDENCE_DIR"];
|
|
30
39
|
if (!evidenceDirectory)
|