verifaied 0.1.0.dev29__tar.gz → 0.2.0.dev31__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {verifaied-0.1.0.dev29 → verifaied-0.2.0.dev31}/PKG-INFO +75 -10
- {verifaied-0.1.0.dev29 → verifaied-0.2.0.dev31}/README.md +74 -9
- {verifaied-0.1.0.dev29 → verifaied-0.2.0.dev31}/pyproject.toml +1 -1
- {verifaied-0.1.0.dev29 → verifaied-0.2.0.dev31}/src/verifaied/cli.py +142 -1
- {verifaied-0.1.0.dev29 → verifaied-0.2.0.dev31}/src/verifaied/client.py +45 -1
- verifaied-0.2.0.dev31/tests/test_check_done.py +196 -0
- {verifaied-0.1.0.dev29 → verifaied-0.2.0.dev31}/tests/test_client.py +25 -0
- {verifaied-0.1.0.dev29 → verifaied-0.2.0.dev31}/.gitignore +0 -0
- {verifaied-0.1.0.dev29 → verifaied-0.2.0.dev31}/LICENSE +0 -0
- {verifaied-0.1.0.dev29 → verifaied-0.2.0.dev31}/src/verifaied/__init__.py +0 -0
- {verifaied-0.1.0.dev29 → verifaied-0.2.0.dev31}/src/verifaied/__main__.py +0 -0
- {verifaied-0.1.0.dev29 → verifaied-0.2.0.dev31}/src/verifaied/analyzer.py +0 -0
- {verifaied-0.1.0.dev29 → verifaied-0.2.0.dev31}/src/verifaied/config.py +0 -0
- {verifaied-0.1.0.dev29 → verifaied-0.2.0.dev31}/src/verifaied/prompts.py +0 -0
- {verifaied-0.1.0.dev29 → verifaied-0.2.0.dev31}/src/verifaied/uploader.py +0 -0
- {verifaied-0.1.0.dev29 → verifaied-0.2.0.dev31}/tests/__init__.py +0 -0
- {verifaied-0.1.0.dev29 → verifaied-0.2.0.dev31}/tests/test_check.py +0 -0
- {verifaied-0.1.0.dev29 → verifaied-0.2.0.dev31}/tests/test_cli.py +0 -0
- {verifaied-0.1.0.dev29 → verifaied-0.2.0.dev31}/tests/test_config.py +0 -0
- {verifaied-0.1.0.dev29 → verifaied-0.2.0.dev31}/tests/test_prompt_parity.py +0 -0
- {verifaied-0.1.0.dev29 → verifaied-0.2.0.dev31}/tests/test_uploader.py +0 -0
- {verifaied-0.1.0.dev29 → verifaied-0.2.0.dev31}/uv.lock +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: verifaied
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.2.0.dev31
|
|
4
4
|
Summary: Find what's untested in your Python code — locally, no account required
|
|
5
5
|
Project-URL: Homepage, https://pypi.org/project/verifaied/
|
|
6
6
|
Author: Kyle Richards
|
|
@@ -185,6 +185,68 @@ keeps the data around for accidental-deletion recovery.
|
|
|
185
185
|
uploaded, then exit without contacting the backend. Skips the token
|
|
186
186
|
requirement so you can audit without configuring auth.
|
|
187
187
|
|
|
188
|
+
## `verifaied check-done` — the "am I done?" gate
|
|
189
|
+
|
|
190
|
+
`check-done` is the **stop condition** for an agent's test-writing loop.
|
|
191
|
+
It fetches a verdict for a branch — failing tests plus the coverage of the
|
|
192
|
+
functions you changed — and exits accordingly, so a CI job or an agent can
|
|
193
|
+
gate on it:
|
|
194
|
+
|
|
195
|
+
```bash
|
|
196
|
+
pytest --cov --cov-report=json --junitxml=junit.xml
|
|
197
|
+
verifaied upload # push the fresh coverage first
|
|
198
|
+
verifaied check-done # then ask: am I done?
|
|
199
|
+
```
|
|
200
|
+
|
|
201
|
+
The verdict composes into `{done, status, gaps}`:
|
|
202
|
+
|
|
203
|
+
- `done` (bool) — `true` only when there are **no gaps at all**. The
|
|
204
|
+
strict signal to stop.
|
|
205
|
+
- `status` — `done` | `done-with-warnings` | `not-done`.
|
|
206
|
+
`done-with-warnings` means the only gaps left are ones the repo owner
|
|
207
|
+
marked non-blocking (per-repo policy, set in the web UI).
|
|
208
|
+
- `gaps[]` — each remaining problem with a `ref`, whether it's `blocking`,
|
|
209
|
+
and a `next_action` pointing at the free (`get_baseline_prompt`) or paid
|
|
210
|
+
(`fix_branch`) remedy.
|
|
211
|
+
- `not_checked[]` — signals that couldn't be evaluated (e.g. failing tests
|
|
212
|
+
when no `junit.xml` was uploaded), surfaced so a green verdict is never
|
|
213
|
+
silently green.
|
|
214
|
+
|
|
215
|
+
### Exit codes
|
|
216
|
+
|
|
217
|
+
- `0` — done (or `done-with-warnings` with `--allow-warnings`)
|
|
218
|
+
- `3` — not done: blocking gaps remain (the CI-gate failure code)
|
|
219
|
+
- `2` — HTTP/API error (auth, 404, network)
|
|
220
|
+
- `1` — usage/config error (no token, bad `--repo`)
|
|
221
|
+
|
|
222
|
+
### `check-done` flags
|
|
223
|
+
|
|
224
|
+
- `--repo / -r <UUID|owner/name>` — repository to check (default:
|
|
225
|
+
auto-detect from the GitHub `origin` remote)
|
|
226
|
+
- `--branch / -b <name>` — branch to judge (default:
|
|
227
|
+
`git branch --show-current`)
|
|
228
|
+
- `--json` — print the raw verdict JSON instead of the human summary
|
|
229
|
+
- `--allow-warnings` — treat `done-with-warnings` as passing (exit 0);
|
|
230
|
+
default is strict (only a clean `done` exits 0)
|
|
231
|
+
- `--api-url <url>` — backend base URL (overrides `VERIFAIED_API_URL`)
|
|
232
|
+
- `--token <token>` — API token (overrides `VERIFAIED_API_TOKEN`)
|
|
233
|
+
|
|
234
|
+
### As an agent stop condition
|
|
235
|
+
|
|
236
|
+
Drop this loop into your agent's instructions file so it self-corrects
|
|
237
|
+
and knows when to stop:
|
|
238
|
+
|
|
239
|
+
```text
|
|
240
|
+
Test-coverage loop (run until done):
|
|
241
|
+
1. Run tests with coverage: pytest --cov --cov-report=json --junitxml=junit.xml
|
|
242
|
+
2. Push the results: verifaied upload
|
|
243
|
+
3. Ask if you're done: verifaied check-done
|
|
244
|
+
4. If exit code is 0, stop — you're done.
|
|
245
|
+
If exit code is 3, fix each gap in the output (use get_baseline_prompt
|
|
246
|
+
for a free test prompt, or fix_branch for an LLM-written one), then
|
|
247
|
+
go back to step 1.
|
|
248
|
+
```
|
|
249
|
+
|
|
188
250
|
## Agent loop
|
|
189
251
|
|
|
190
252
|
We've found the most fruitful way to use verifAIed is to put a short
|
|
@@ -201,7 +263,7 @@ into the matching rules file, tailored to your repo:
|
|
|
201
263
|
```markdown
|
|
202
264
|
We've found that adding a short test-coverage loop to your agent's instructions is the most fruitful way to use verifAIed. I want you to write that loop into this repo's agent-instructions file, tailored to how this repo actually runs its tests.
|
|
203
265
|
|
|
204
|
-
verifAIed is a coverage analysis tool. Its CLI (`verifaied upload`) pushes a local `coverage.json` + `junit.xml` to the server; its MCP server exposes a `fix_branch` tool that returns a single prompt describing every untested function, partial branch, and failing test
|
|
266
|
+
verifAIed is a coverage analysis tool. Its CLI (`verifaied upload`) pushes a local `coverage.json` + `junit.xml` to the server; its MCP server exposes a `check_done` tool that returns a `{done, status, gaps}` verdict for the branch (the loop's stop condition), and a `fix_branch` tool that returns a single prompt describing every untested function, partial branch, and failing test. The agent loop is: run tests → `verifaied upload` → call `check_done` → if `done` is true stop, otherwise fix the gaps (`get_baseline_prompt` per function, or `fix_branch` for one prompt covering everything) → repeat.
|
|
205
267
|
|
|
206
268
|
Do the following, in order:
|
|
207
269
|
|
|
@@ -233,8 +295,9 @@ Append (or create) a section titled `## Test coverage loop with verifAIed` conta
|
|
|
233
295
|
|
|
234
296
|
- The exact test command(s) you built in step 1 — so future runs of the agent don't have to re-derive them.
|
|
235
297
|
- The exact upload command (`verifaied upload`, plus any flags this repo needs — e.g. `--coverage path/to/coverage.json` if your test command lands the JSON outside the repo root).
|
|
236
|
-
- An instruction to call the `
|
|
237
|
-
- An instruction
|
|
298
|
+
- An instruction to call the `check_done` tool on the verifAIed MCP server after each upload, and to stop only when it returns `done: true`.
|
|
299
|
+
- An instruction that while `check_done` reports gaps, fix them: call `get_baseline_prompt(function_id)` for a free per-function test prompt, or `fix_branch` with `scope: everything` for a single prompt covering every gap, then apply it.
|
|
300
|
+
- An instruction to repeat from step 1 until `check_done` reports `done: true`.
|
|
238
301
|
- One line: "If `VERIFAIED_API_TOKEN` is unset, stop and ask the user — do not skip this loop."
|
|
239
302
|
|
|
240
303
|
# 4. Confirm with the user
|
|
@@ -246,7 +309,8 @@ Show the user the diff of the file you edited (or created) and the test command
|
|
|
246
309
|
- Do NOT change the test command's behavior beyond adding coverage / junit flags.
|
|
247
310
|
- Do NOT add new dependencies if pytest-cov and coverage are already transitive deps.
|
|
248
311
|
- Do NOT write a fallback that skips the loop when the token is missing — the agent must stop and ask.
|
|
249
|
-
- Do NOT enumerate functions yourself or write tests; `fix_branch`
|
|
312
|
+
- Do NOT enumerate functions yourself or write tests; `check_done` lists the gaps and `fix_branch` / `get_baseline_prompt` return the prompts that close them.
|
|
313
|
+
- Do NOT decide "done" yourself; `check_done` is the stop condition.
|
|
250
314
|
- Do NOT replace an existing agent-instructions file — append a new section.
|
|
251
315
|
```
|
|
252
316
|
|
|
@@ -259,7 +323,7 @@ it differs from the pytest default:
|
|
|
259
323
|
```markdown
|
|
260
324
|
## Test coverage loop with verifAIed
|
|
261
325
|
|
|
262
|
-
After implementing or modifying any code in this repo, run this loop until verifAIed reports
|
|
326
|
+
After implementing or modifying any code in this repo, run this loop until verifAIed reports done:
|
|
263
327
|
|
|
264
328
|
1. Run the test suite with branch coverage and per-test contexts:
|
|
265
329
|
```
|
|
@@ -270,9 +334,10 @@ After implementing or modifying any code in this repo, run this loop until verif
|
|
|
270
334
|
```
|
|
271
335
|
verifaied upload
|
|
272
336
|
```
|
|
273
|
-
3.
|
|
274
|
-
4.
|
|
275
|
-
5.
|
|
337
|
+
3. Ask if you're done: call the `check_done` tool on the verifAIed MCP server (or run `verifaied check-done`). It returns `{done, status, gaps}` for this branch — failing tests plus untested/partially-covered functions among the ones you changed.
|
|
338
|
+
4. If `done` is true, stop — the branch is covered.
|
|
339
|
+
5. Otherwise fix the gaps it lists: call `get_baseline_prompt(function_id)` for a free test prompt per function, or `fix_branch` with `scope: everything` for a single LLM-written prompt covering every gap on the branch.
|
|
340
|
+
6. Go back to step 1.
|
|
276
341
|
|
|
277
|
-
Stop when `
|
|
342
|
+
Stop only when `check_done` reports `done: true`. If `VERIFAIED_API_TOKEN` is not set, stop and ask the user — do not skip this loop.
|
|
278
343
|
```
|
|
@@ -145,6 +145,68 @@ keeps the data around for accidental-deletion recovery.
|
|
|
145
145
|
uploaded, then exit without contacting the backend. Skips the token
|
|
146
146
|
requirement so you can audit without configuring auth.
|
|
147
147
|
|
|
148
|
+
## `verifaied check-done` — the "am I done?" gate
|
|
149
|
+
|
|
150
|
+
`check-done` is the **stop condition** for an agent's test-writing loop.
|
|
151
|
+
It fetches a verdict for a branch — failing tests plus the coverage of the
|
|
152
|
+
functions you changed — and exits accordingly, so a CI job or an agent can
|
|
153
|
+
gate on it:
|
|
154
|
+
|
|
155
|
+
```bash
|
|
156
|
+
pytest --cov --cov-report=json --junitxml=junit.xml
|
|
157
|
+
verifaied upload # push the fresh coverage first
|
|
158
|
+
verifaied check-done # then ask: am I done?
|
|
159
|
+
```
|
|
160
|
+
|
|
161
|
+
The verdict composes into `{done, status, gaps}`:
|
|
162
|
+
|
|
163
|
+
- `done` (bool) — `true` only when there are **no gaps at all**. The
|
|
164
|
+
strict signal to stop.
|
|
165
|
+
- `status` — `done` | `done-with-warnings` | `not-done`.
|
|
166
|
+
`done-with-warnings` means the only gaps left are ones the repo owner
|
|
167
|
+
marked non-blocking (per-repo policy, set in the web UI).
|
|
168
|
+
- `gaps[]` — each remaining problem with a `ref`, whether it's `blocking`,
|
|
169
|
+
and a `next_action` pointing at the free (`get_baseline_prompt`) or paid
|
|
170
|
+
(`fix_branch`) remedy.
|
|
171
|
+
- `not_checked[]` — signals that couldn't be evaluated (e.g. failing tests
|
|
172
|
+
when no `junit.xml` was uploaded), surfaced so a green verdict is never
|
|
173
|
+
silently green.
|
|
174
|
+
|
|
175
|
+
### Exit codes
|
|
176
|
+
|
|
177
|
+
- `0` — done (or `done-with-warnings` with `--allow-warnings`)
|
|
178
|
+
- `3` — not done: blocking gaps remain (the CI-gate failure code)
|
|
179
|
+
- `2` — HTTP/API error (auth, 404, network)
|
|
180
|
+
- `1` — usage/config error (no token, bad `--repo`)
|
|
181
|
+
|
|
182
|
+
### `check-done` flags
|
|
183
|
+
|
|
184
|
+
- `--repo / -r <UUID|owner/name>` — repository to check (default:
|
|
185
|
+
auto-detect from the GitHub `origin` remote)
|
|
186
|
+
- `--branch / -b <name>` — branch to judge (default:
|
|
187
|
+
`git branch --show-current`)
|
|
188
|
+
- `--json` — print the raw verdict JSON instead of the human summary
|
|
189
|
+
- `--allow-warnings` — treat `done-with-warnings` as passing (exit 0);
|
|
190
|
+
default is strict (only a clean `done` exits 0)
|
|
191
|
+
- `--api-url <url>` — backend base URL (overrides `VERIFAIED_API_URL`)
|
|
192
|
+
- `--token <token>` — API token (overrides `VERIFAIED_API_TOKEN`)
|
|
193
|
+
|
|
194
|
+
### As an agent stop condition
|
|
195
|
+
|
|
196
|
+
Drop this loop into your agent's instructions file so it self-corrects
|
|
197
|
+
and knows when to stop:
|
|
198
|
+
|
|
199
|
+
```text
|
|
200
|
+
Test-coverage loop (run until done):
|
|
201
|
+
1. Run tests with coverage: pytest --cov --cov-report=json --junitxml=junit.xml
|
|
202
|
+
2. Push the results: verifaied upload
|
|
203
|
+
3. Ask if you're done: verifaied check-done
|
|
204
|
+
4. If exit code is 0, stop — you're done.
|
|
205
|
+
If exit code is 3, fix each gap in the output (use get_baseline_prompt
|
|
206
|
+
for a free test prompt, or fix_branch for an LLM-written one), then
|
|
207
|
+
go back to step 1.
|
|
208
|
+
```
|
|
209
|
+
|
|
148
210
|
## Agent loop
|
|
149
211
|
|
|
150
212
|
We've found the most fruitful way to use verifAIed is to put a short
|
|
@@ -161,7 +223,7 @@ into the matching rules file, tailored to your repo:
|
|
|
161
223
|
```markdown
|
|
162
224
|
We've found that adding a short test-coverage loop to your agent's instructions is the most fruitful way to use verifAIed. I want you to write that loop into this repo's agent-instructions file, tailored to how this repo actually runs its tests.
|
|
163
225
|
|
|
164
|
-
verifAIed is a coverage analysis tool. Its CLI (`verifaied upload`) pushes a local `coverage.json` + `junit.xml` to the server; its MCP server exposes a `fix_branch` tool that returns a single prompt describing every untested function, partial branch, and failing test
|
|
226
|
+
verifAIed is a coverage analysis tool. Its CLI (`verifaied upload`) pushes a local `coverage.json` + `junit.xml` to the server; its MCP server exposes a `check_done` tool that returns a `{done, status, gaps}` verdict for the branch (the loop's stop condition), and a `fix_branch` tool that returns a single prompt describing every untested function, partial branch, and failing test. The agent loop is: run tests → `verifaied upload` → call `check_done` → if `done` is true stop, otherwise fix the gaps (`get_baseline_prompt` per function, or `fix_branch` for one prompt covering everything) → repeat.
|
|
165
227
|
|
|
166
228
|
Do the following, in order:
|
|
167
229
|
|
|
@@ -193,8 +255,9 @@ Append (or create) a section titled `## Test coverage loop with verifAIed` conta
|
|
|
193
255
|
|
|
194
256
|
- The exact test command(s) you built in step 1 — so future runs of the agent don't have to re-derive them.
|
|
195
257
|
- The exact upload command (`verifaied upload`, plus any flags this repo needs — e.g. `--coverage path/to/coverage.json` if your test command lands the JSON outside the repo root).
|
|
196
|
-
- An instruction to call the `
|
|
197
|
-
- An instruction
|
|
258
|
+
- An instruction to call the `check_done` tool on the verifAIed MCP server after each upload, and to stop only when it returns `done: true`.
|
|
259
|
+
- An instruction that while `check_done` reports gaps, fix them: call `get_baseline_prompt(function_id)` for a free per-function test prompt, or `fix_branch` with `scope: everything` for a single prompt covering every gap, then apply it.
|
|
260
|
+
- An instruction to repeat from step 1 until `check_done` reports `done: true`.
|
|
198
261
|
- One line: "If `VERIFAIED_API_TOKEN` is unset, stop and ask the user — do not skip this loop."
|
|
199
262
|
|
|
200
263
|
# 4. Confirm with the user
|
|
@@ -206,7 +269,8 @@ Show the user the diff of the file you edited (or created) and the test command
|
|
|
206
269
|
- Do NOT change the test command's behavior beyond adding coverage / junit flags.
|
|
207
270
|
- Do NOT add new dependencies if pytest-cov and coverage are already transitive deps.
|
|
208
271
|
- Do NOT write a fallback that skips the loop when the token is missing — the agent must stop and ask.
|
|
209
|
-
- Do NOT enumerate functions yourself or write tests; `fix_branch`
|
|
272
|
+
- Do NOT enumerate functions yourself or write tests; `check_done` lists the gaps and `fix_branch` / `get_baseline_prompt` return the prompts that close them.
|
|
273
|
+
- Do NOT decide "done" yourself; `check_done` is the stop condition.
|
|
210
274
|
- Do NOT replace an existing agent-instructions file — append a new section.
|
|
211
275
|
```
|
|
212
276
|
|
|
@@ -219,7 +283,7 @@ it differs from the pytest default:
|
|
|
219
283
|
```markdown
|
|
220
284
|
## Test coverage loop with verifAIed
|
|
221
285
|
|
|
222
|
-
After implementing or modifying any code in this repo, run this loop until verifAIed reports
|
|
286
|
+
After implementing or modifying any code in this repo, run this loop until verifAIed reports done:
|
|
223
287
|
|
|
224
288
|
1. Run the test suite with branch coverage and per-test contexts:
|
|
225
289
|
```
|
|
@@ -230,9 +294,10 @@ After implementing or modifying any code in this repo, run this loop until verif
|
|
|
230
294
|
```
|
|
231
295
|
verifaied upload
|
|
232
296
|
```
|
|
233
|
-
3.
|
|
234
|
-
4.
|
|
235
|
-
5.
|
|
297
|
+
3. Ask if you're done: call the `check_done` tool on the verifAIed MCP server (or run `verifaied check-done`). It returns `{done, status, gaps}` for this branch — failing tests plus untested/partially-covered functions among the ones you changed.
|
|
298
|
+
4. If `done` is true, stop — the branch is covered.
|
|
299
|
+
5. Otherwise fix the gaps it lists: call `get_baseline_prompt(function_id)` for a free test prompt per function, or `fix_branch` with `scope: everything` for a single LLM-written prompt covering every gap on the branch.
|
|
300
|
+
6. Go back to step 1.
|
|
236
301
|
|
|
237
|
-
Stop when `
|
|
302
|
+
Stop only when `check_done` reports `done: true`. If `VERIFAIED_API_TOKEN` is not set, stop and ask the user — do not skip this loop.
|
|
238
303
|
```
|
|
@@ -25,6 +25,7 @@ from verifaied.client import (
|
|
|
25
25
|
ApiError,
|
|
26
26
|
RepoNotFoundError,
|
|
27
27
|
UploadResult,
|
|
28
|
+
check_done,
|
|
28
29
|
resolve_repo_id,
|
|
29
30
|
upload,
|
|
30
31
|
)
|
|
@@ -36,7 +37,12 @@ from verifaied.config import (
|
|
|
36
37
|
resolved_api_url,
|
|
37
38
|
)
|
|
38
39
|
from verifaied.prompts import build_prompt, format_line_ranges
|
|
39
|
-
from verifaied.uploader import
|
|
40
|
+
from verifaied.uploader import (
|
|
41
|
+
UploaderError,
|
|
42
|
+
build_payload,
|
|
43
|
+
detect_branch,
|
|
44
|
+
detect_repo_slug,
|
|
45
|
+
)
|
|
40
46
|
|
|
41
47
|
app = typer.Typer(
|
|
42
48
|
add_completion=False,
|
|
@@ -293,6 +299,141 @@ def check_command(
|
|
|
293
299
|
raise typer.Exit(code=3)
|
|
294
300
|
|
|
295
301
|
|
|
302
|
+
@app.command("check-done")
|
|
303
|
+
def check_done_command(
|
|
304
|
+
repo: str | None = typer.Option(
|
|
305
|
+
None,
|
|
306
|
+
"--repo",
|
|
307
|
+
"-r",
|
|
308
|
+
help=(
|
|
309
|
+
"Repository to check. Accepts a UUID, an `owner/name` slug, or "
|
|
310
|
+
"omit to auto-detect from `git remote get-url origin`."
|
|
311
|
+
),
|
|
312
|
+
),
|
|
313
|
+
branch: str | None = typer.Option(
|
|
314
|
+
None,
|
|
315
|
+
"--branch",
|
|
316
|
+
"-b",
|
|
317
|
+
help="Branch to judge. Defaults to `git branch --show-current`.",
|
|
318
|
+
),
|
|
319
|
+
api_url: str | None = typer.Option(
|
|
320
|
+
None,
|
|
321
|
+
"--api-url",
|
|
322
|
+
help=f"Backend base URL. Defaults to ${ENV_API_URL} or {DEFAULT_API_URL}.",
|
|
323
|
+
),
|
|
324
|
+
token: str | None = typer.Option(
|
|
325
|
+
None,
|
|
326
|
+
"--token",
|
|
327
|
+
help=f"API token (vr_live_...). Defaults to ${ENV_API_TOKEN}.",
|
|
328
|
+
),
|
|
329
|
+
as_json: bool = typer.Option(
|
|
330
|
+
False,
|
|
331
|
+
"--json",
|
|
332
|
+
help="Print the raw verdict JSON instead of the human summary.",
|
|
333
|
+
),
|
|
334
|
+
allow_warnings: bool = typer.Option(
|
|
335
|
+
False,
|
|
336
|
+
"--allow-warnings",
|
|
337
|
+
help=(
|
|
338
|
+
"Treat `done-with-warnings` as passing (exit 0). Default is "
|
|
339
|
+
"strict: only a clean `done` exits 0."
|
|
340
|
+
),
|
|
341
|
+
),
|
|
342
|
+
) -> None:
|
|
343
|
+
"""Am I done? The definition-of-done gate for an agent loop.
|
|
344
|
+
|
|
345
|
+
Fetches the hosted verdict for a branch — failing tests plus the
|
|
346
|
+
coverage of the functions you changed — and exits accordingly so a
|
|
347
|
+
CI job or an agent can stop on it:
|
|
348
|
+
|
|
349
|
+
0 done (or `done-with-warnings` with --allow-warnings)
|
|
350
|
+
3 not done — blocking gaps remain
|
|
351
|
+
2 HTTP/API error (auth, 404, network)
|
|
352
|
+
1 usage / config error (no token, bad --repo)
|
|
353
|
+
|
|
354
|
+
Upload coverage first (`verifaied upload`), then gate on this.
|
|
355
|
+
"""
|
|
356
|
+
resolved_url = resolved_api_url(api_url)
|
|
357
|
+
resolved_token = resolved_api_token(token)
|
|
358
|
+
if not resolved_token:
|
|
359
|
+
err_console.print(
|
|
360
|
+
f"[red]error[/red]: no API token (set ${ENV_API_TOKEN} or pass --token)"
|
|
361
|
+
)
|
|
362
|
+
raise typer.Exit(code=1)
|
|
363
|
+
|
|
364
|
+
try:
|
|
365
|
+
repo_id = _resolve_repo(
|
|
366
|
+
repo_input=repo, api_url=resolved_url, token=resolved_token
|
|
367
|
+
)
|
|
368
|
+
except (UploaderError, ApiError, RepoNotFoundError) as e:
|
|
369
|
+
err_console.print(f"[red]error[/red]: {e}")
|
|
370
|
+
raise typer.Exit(code=2 if isinstance(e, ApiError) else 1) from e
|
|
371
|
+
|
|
372
|
+
resolved_branch = branch or detect_branch()
|
|
373
|
+
try:
|
|
374
|
+
verdict = check_done(
|
|
375
|
+
api_url=resolved_url,
|
|
376
|
+
token=resolved_token,
|
|
377
|
+
repo_id=repo_id,
|
|
378
|
+
branch=resolved_branch,
|
|
379
|
+
)
|
|
380
|
+
except ApiError as e:
|
|
381
|
+
err_console.print(f"[red]error[/red]: {e.message}")
|
|
382
|
+
raise typer.Exit(code=2) from e
|
|
383
|
+
|
|
384
|
+
if as_json:
|
|
385
|
+
print(json.dumps(verdict, indent=2))
|
|
386
|
+
else:
|
|
387
|
+
_render_check_done(verdict)
|
|
388
|
+
|
|
389
|
+
status = verdict.get("status")
|
|
390
|
+
if status == "done":
|
|
391
|
+
return
|
|
392
|
+
if status == "done-with-warnings" and allow_warnings:
|
|
393
|
+
return
|
|
394
|
+
raise typer.Exit(code=3)
|
|
395
|
+
|
|
396
|
+
|
|
397
|
+
def _render_check_done(verdict: dict) -> None:
|
|
398
|
+
"""Status line + a table of the remaining gaps."""
|
|
399
|
+
status = verdict.get("status", "unknown")
|
|
400
|
+
style = {
|
|
401
|
+
"done": "green",
|
|
402
|
+
"done-with-warnings": "yellow",
|
|
403
|
+
"not-done": "red",
|
|
404
|
+
}.get(status, "white")
|
|
405
|
+
console.print()
|
|
406
|
+
console.print(
|
|
407
|
+
f"[bold {style}]{status}[/bold {style}] "
|
|
408
|
+
f"[dim]{verdict.get('branch', '?')} · scope: "
|
|
409
|
+
f"{verdict.get('scope', '?')}[/dim]"
|
|
410
|
+
)
|
|
411
|
+
|
|
412
|
+
for signal in verdict.get("not_checked") or []:
|
|
413
|
+
console.print(
|
|
414
|
+
f"[yellow]not checked[/yellow]: {signal.get('signal')} "
|
|
415
|
+
f"— {signal.get('reason')}"
|
|
416
|
+
)
|
|
417
|
+
|
|
418
|
+
gaps = verdict.get("gaps") or []
|
|
419
|
+
if not gaps:
|
|
420
|
+
console.print("\n[green]No gaps — done.[/green]")
|
|
421
|
+
return
|
|
422
|
+
|
|
423
|
+
table = Table(show_header=True, header_style="bold", box=None, pad_edge=False)
|
|
424
|
+
table.add_column("")
|
|
425
|
+
table.add_column("kind")
|
|
426
|
+
table.add_column("ref")
|
|
427
|
+
for gap in gaps:
|
|
428
|
+
marker = "[red]●[/red]" if gap.get("blocking") else "[yellow]○[/yellow]"
|
|
429
|
+
table.add_row(marker, gap.get("kind", "?"), gap.get("ref", "?"))
|
|
430
|
+
console.print()
|
|
431
|
+
console.print(table)
|
|
432
|
+
console.print(
|
|
433
|
+
"\n[dim]● blocking ○ warning — fix each gap, re-upload, re-check.[/dim]"
|
|
434
|
+
)
|
|
435
|
+
|
|
436
|
+
|
|
296
437
|
def _render_check(
|
|
297
438
|
summary: CheckSummary, targets: list[FunctionResult], *, limit: int
|
|
298
439
|
) -> None:
|
|
@@ -95,6 +95,39 @@ def upload(
|
|
|
95
95
|
)
|
|
96
96
|
|
|
97
97
|
|
|
98
|
+
def check_done(
|
|
99
|
+
*,
|
|
100
|
+
api_url: str,
|
|
101
|
+
token: str,
|
|
102
|
+
repo_id: UUID,
|
|
103
|
+
branch: str | None = None,
|
|
104
|
+
timeout: float = 30.0,
|
|
105
|
+
) -> dict[str, Any]:
|
|
106
|
+
"""GET the ``check-done`` verdict for a (repo, branch).
|
|
107
|
+
|
|
108
|
+
Returns the parsed verdict dict (``{done, status, gaps, ...}``).
|
|
109
|
+
Raises ``ApiError`` on any non-2xx — the CLI maps a 404 to "no
|
|
110
|
+
analysis / repo not found" the same way the upload path does.
|
|
111
|
+
"""
|
|
112
|
+
url = f"{api_url.rstrip('/')}/repositories/{repo_id}/check-done"
|
|
113
|
+
params = {"branch": branch} if branch is not None else None
|
|
114
|
+
try:
|
|
115
|
+
response = httpx.get(
|
|
116
|
+
url,
|
|
117
|
+
params=params,
|
|
118
|
+
headers={"Authorization": f"Bearer {token}"},
|
|
119
|
+
timeout=timeout,
|
|
120
|
+
)
|
|
121
|
+
except httpx.HTTPError as e:
|
|
122
|
+
raise ApiError(0, f"could not reach {url}: {e}") from e
|
|
123
|
+
if response.status_code >= 400:
|
|
124
|
+
raise ApiError(response.status_code, _detail(response))
|
|
125
|
+
data = response.json()
|
|
126
|
+
if not isinstance(data, dict):
|
|
127
|
+
raise ApiError(0, f"unexpected response from {url}: not an object")
|
|
128
|
+
return data
|
|
129
|
+
|
|
130
|
+
|
|
98
131
|
def list_installations(
|
|
99
132
|
*, api_url: str, token: str, timeout: float = 15.0
|
|
100
133
|
) -> list[dict[str, Any]]:
|
|
@@ -148,7 +181,14 @@ def resolve_repo_id(*, api_url: str, token: str, owner: str, name: str) -> UUID:
|
|
|
148
181
|
def _detail(response: httpx.Response) -> str:
|
|
149
182
|
"""Pull a human-readable error message out of the response. Falls
|
|
150
183
|
back to status reason if the body isn't JSON or doesn't carry a
|
|
151
|
-
``detail`` field.
|
|
184
|
+
usable ``detail`` field.
|
|
185
|
+
|
|
186
|
+
``detail`` is usually a plain string, but structured errors return an
|
|
187
|
+
object instead — a plan-limit 403 carries
|
|
188
|
+
``{"code", "message", "limit"}`` so the UI and CLI can branch on the
|
|
189
|
+
code. We surface its ``message`` so the user sees "Your plan covers 10
|
|
190
|
+
active branches..." rather than a bare "403 Forbidden".
|
|
191
|
+
"""
|
|
152
192
|
try:
|
|
153
193
|
body = response.json()
|
|
154
194
|
except ValueError:
|
|
@@ -156,4 +196,8 @@ def _detail(response: httpx.Response) -> str:
|
|
|
156
196
|
detail = body.get("detail") if isinstance(body, dict) else None
|
|
157
197
|
if isinstance(detail, str) and detail:
|
|
158
198
|
return detail
|
|
199
|
+
if isinstance(detail, dict):
|
|
200
|
+
message = detail.get("message")
|
|
201
|
+
if isinstance(message, str) and message:
|
|
202
|
+
return message
|
|
159
203
|
return f"{response.status_code} {response.reason_phrase}".strip()
|
|
@@ -0,0 +1,196 @@
|
|
|
1
|
+
"""Tests for `verifaied check-done` — the definition-of-done CI gate.
|
|
2
|
+
|
|
3
|
+
Invoked through the Typer runner so config resolution, error mapping, and
|
|
4
|
+
the exit-code contract (0 done / 3 not-done / 2 HTTP / 1 config) all run
|
|
5
|
+
the way a CI job or an agent hits them. The HTTP layer is mocked with
|
|
6
|
+
pytest-httpx; ``--repo <UUID>`` is passed so repo resolution never needs
|
|
7
|
+
the ``/installations/me`` round-trip.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import json
|
|
13
|
+
from uuid import uuid4
|
|
14
|
+
|
|
15
|
+
from pytest_httpx import HTTPXMock
|
|
16
|
+
from typer.testing import CliRunner
|
|
17
|
+
|
|
18
|
+
from verifaied.cli import app
|
|
19
|
+
from verifaied.config import ENV_API_TOKEN
|
|
20
|
+
|
|
21
|
+
runner = CliRunner()
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def _verdict(status: str, gaps: list[dict] | None = None, **extra) -> dict:
|
|
25
|
+
verdict = {
|
|
26
|
+
"done": status == "done",
|
|
27
|
+
"status": status,
|
|
28
|
+
"scope": "changed-functions",
|
|
29
|
+
"repository_id": "r",
|
|
30
|
+
"branch": "feature",
|
|
31
|
+
"analysis_id": "a",
|
|
32
|
+
"commit_sha": "sha",
|
|
33
|
+
"origin": "local",
|
|
34
|
+
"gaps": gaps or [],
|
|
35
|
+
"checked": ["untested_functions", "partial_functions"],
|
|
36
|
+
"not_checked": [],
|
|
37
|
+
}
|
|
38
|
+
verdict.update(extra)
|
|
39
|
+
return verdict
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def _invoke(repo_id, *, branch: str = "feature", extra_args: list[str] | None = None):
|
|
43
|
+
return runner.invoke(
|
|
44
|
+
app,
|
|
45
|
+
[
|
|
46
|
+
"check-done",
|
|
47
|
+
"--repo",
|
|
48
|
+
str(repo_id),
|
|
49
|
+
"--branch",
|
|
50
|
+
branch,
|
|
51
|
+
"--api-url",
|
|
52
|
+
"http://api.test",
|
|
53
|
+
*(extra_args or []),
|
|
54
|
+
],
|
|
55
|
+
)
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def test_check_done_exits_0_when_done(monkeypatch, httpx_mock: HTTPXMock):
|
|
59
|
+
monkeypatch.setenv(ENV_API_TOKEN, "vr_live_tok")
|
|
60
|
+
repo_id = uuid4()
|
|
61
|
+
httpx_mock.add_response(
|
|
62
|
+
url=f"http://api.test/repositories/{repo_id}/check-done?branch=feature",
|
|
63
|
+
json=_verdict("done"),
|
|
64
|
+
)
|
|
65
|
+
|
|
66
|
+
result = _invoke(repo_id)
|
|
67
|
+
|
|
68
|
+
assert result.exit_code == 0
|
|
69
|
+
assert "done" in result.stdout
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def test_check_done_exits_3_when_not_done(monkeypatch, httpx_mock: HTTPXMock):
|
|
73
|
+
monkeypatch.setenv(ENV_API_TOKEN, "vr_live_tok")
|
|
74
|
+
repo_id = uuid4()
|
|
75
|
+
httpx_mock.add_response(
|
|
76
|
+
url=f"http://api.test/repositories/{repo_id}/check-done?branch=feature",
|
|
77
|
+
json=_verdict(
|
|
78
|
+
"not-done",
|
|
79
|
+
gaps=[
|
|
80
|
+
{
|
|
81
|
+
"kind": "untested_function",
|
|
82
|
+
"ref": "app/billing.py::apply_topup",
|
|
83
|
+
"blocking": True,
|
|
84
|
+
}
|
|
85
|
+
],
|
|
86
|
+
),
|
|
87
|
+
)
|
|
88
|
+
|
|
89
|
+
result = _invoke(repo_id)
|
|
90
|
+
|
|
91
|
+
assert result.exit_code == 3
|
|
92
|
+
assert "not-done" in result.stdout
|
|
93
|
+
assert "app/billing.py::apply_topup" in result.stdout
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def test_check_done_with_warnings_exits_3_by_default(
|
|
97
|
+
monkeypatch, httpx_mock: HTTPXMock
|
|
98
|
+
):
|
|
99
|
+
monkeypatch.setenv(ENV_API_TOKEN, "vr_live_tok")
|
|
100
|
+
repo_id = uuid4()
|
|
101
|
+
httpx_mock.add_response(
|
|
102
|
+
url=f"http://api.test/repositories/{repo_id}/check-done?branch=feature",
|
|
103
|
+
json=_verdict(
|
|
104
|
+
"done-with-warnings",
|
|
105
|
+
gaps=[{"kind": "partial_function", "ref": "m.py::f", "blocking": False}],
|
|
106
|
+
),
|
|
107
|
+
)
|
|
108
|
+
|
|
109
|
+
result = _invoke(repo_id)
|
|
110
|
+
|
|
111
|
+
assert result.exit_code == 3
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def test_check_done_with_warnings_exits_0_when_allowed(
|
|
115
|
+
monkeypatch, httpx_mock: HTTPXMock
|
|
116
|
+
):
|
|
117
|
+
monkeypatch.setenv(ENV_API_TOKEN, "vr_live_tok")
|
|
118
|
+
repo_id = uuid4()
|
|
119
|
+
httpx_mock.add_response(
|
|
120
|
+
url=f"http://api.test/repositories/{repo_id}/check-done?branch=feature",
|
|
121
|
+
json=_verdict(
|
|
122
|
+
"done-with-warnings",
|
|
123
|
+
gaps=[{"kind": "partial_function", "ref": "m.py::f", "blocking": False}],
|
|
124
|
+
),
|
|
125
|
+
)
|
|
126
|
+
|
|
127
|
+
result = _invoke(repo_id, extra_args=["--allow-warnings"])
|
|
128
|
+
|
|
129
|
+
assert result.exit_code == 0
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def test_check_done_json_output(monkeypatch, httpx_mock: HTTPXMock):
|
|
133
|
+
monkeypatch.setenv(ENV_API_TOKEN, "vr_live_tok")
|
|
134
|
+
repo_id = uuid4()
|
|
135
|
+
verdict = _verdict(
|
|
136
|
+
"not-done", gaps=[{"kind": "failing_test", "ref": "t", "blocking": True}]
|
|
137
|
+
)
|
|
138
|
+
httpx_mock.add_response(
|
|
139
|
+
url=f"http://api.test/repositories/{repo_id}/check-done?branch=feature",
|
|
140
|
+
json=verdict,
|
|
141
|
+
)
|
|
142
|
+
|
|
143
|
+
result = _invoke(repo_id, extra_args=["--json"])
|
|
144
|
+
|
|
145
|
+
assert result.exit_code == 3
|
|
146
|
+
# The raw verdict is machine-parseable on stdout.
|
|
147
|
+
parsed = json.loads(result.stdout)
|
|
148
|
+
assert parsed["status"] == "not-done"
|
|
149
|
+
assert parsed["gaps"][0]["kind"] == "failing_test"
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
def test_check_done_not_checked_signal_is_surfaced(monkeypatch, httpx_mock: HTTPXMock):
|
|
153
|
+
monkeypatch.setenv(ENV_API_TOKEN, "vr_live_tok")
|
|
154
|
+
repo_id = uuid4()
|
|
155
|
+
httpx_mock.add_response(
|
|
156
|
+
url=f"http://api.test/repositories/{repo_id}/check-done?branch=feature",
|
|
157
|
+
json=_verdict(
|
|
158
|
+
"done",
|
|
159
|
+
not_checked=[
|
|
160
|
+
{
|
|
161
|
+
"signal": "failing_tests",
|
|
162
|
+
"reason": "no junit.xml uploaded with this analysis",
|
|
163
|
+
}
|
|
164
|
+
],
|
|
165
|
+
),
|
|
166
|
+
)
|
|
167
|
+
|
|
168
|
+
result = _invoke(repo_id)
|
|
169
|
+
|
|
170
|
+
assert result.exit_code == 0
|
|
171
|
+
assert "not checked" in result.stdout
|
|
172
|
+
assert "failing_tests" in result.stdout
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
def test_check_done_exits_2_on_api_error(monkeypatch, httpx_mock: HTTPXMock):
|
|
176
|
+
monkeypatch.setenv(ENV_API_TOKEN, "vr_live_tok")
|
|
177
|
+
repo_id = uuid4()
|
|
178
|
+
httpx_mock.add_response(
|
|
179
|
+
url=f"http://api.test/repositories/{repo_id}/check-done?branch=feature",
|
|
180
|
+
status_code=404,
|
|
181
|
+
json={"detail": "no analysis found for branch 'feature'"},
|
|
182
|
+
)
|
|
183
|
+
|
|
184
|
+
result = _invoke(repo_id)
|
|
185
|
+
|
|
186
|
+
assert result.exit_code == 2
|
|
187
|
+
assert "no analysis found" in result.stderr
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
def test_check_done_exits_1_when_no_token(monkeypatch):
|
|
191
|
+
monkeypatch.delenv(ENV_API_TOKEN, raising=False)
|
|
192
|
+
|
|
193
|
+
result = _invoke(uuid4())
|
|
194
|
+
|
|
195
|
+
assert result.exit_code == 1
|
|
196
|
+
assert "no API token" in result.stderr
|
|
@@ -156,6 +156,31 @@ def test_upload_401_raises_api_error_with_detail(httpx_mock: HTTPXMock):
|
|
|
156
156
|
assert "invalid token" in ei.value.message
|
|
157
157
|
|
|
158
158
|
|
|
159
|
+
def test_upload_403_surfaces_plan_limit_message(httpx_mock: HTTPXMock):
|
|
160
|
+
repo_id = uuid4()
|
|
161
|
+
httpx_mock.add_response(
|
|
162
|
+
status_code=403,
|
|
163
|
+
json={
|
|
164
|
+
"detail": {
|
|
165
|
+
"code": "plan_branch_limit",
|
|
166
|
+
"message": "Your plan covers 10 active branches per repository.",
|
|
167
|
+
"limit": 10,
|
|
168
|
+
}
|
|
169
|
+
},
|
|
170
|
+
)
|
|
171
|
+
|
|
172
|
+
with pytest.raises(ApiError) as ei:
|
|
173
|
+
upload(
|
|
174
|
+
api_url="http://api.test",
|
|
175
|
+
token="t",
|
|
176
|
+
repo_id=repo_id,
|
|
177
|
+
payload=_PAYLOAD,
|
|
178
|
+
)
|
|
179
|
+
|
|
180
|
+
assert ei.value.status_code == 403
|
|
181
|
+
assert ei.value.message == "Your plan covers 10 active branches per repository."
|
|
182
|
+
|
|
183
|
+
|
|
159
184
|
def test_upload_413_carries_status_code(httpx_mock: HTTPXMock):
|
|
160
185
|
repo_id = uuid4()
|
|
161
186
|
httpx_mock.add_response(status_code=413, json={"detail": "too large"})
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|