verifaied 0.28.0.dev74__tar.gz → 0.29.0.dev76__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (75) hide show
  1. {verifaied-0.28.0.dev74 → verifaied-0.29.0.dev76}/PKG-INFO +48 -2
  2. {verifaied-0.28.0.dev74 → verifaied-0.29.0.dev76}/README.md +47 -1
  3. {verifaied-0.28.0.dev74 → verifaied-0.29.0.dev76}/pyproject.toml +1 -1
  4. {verifaied-0.28.0.dev74 → verifaied-0.29.0.dev76}/src/verifaied/agent_mcp.py +46 -8
  5. {verifaied-0.28.0.dev74 → verifaied-0.29.0.dev76}/src/verifaied/cli.py +73 -34
  6. verifaied-0.29.0.dev76/src/verifaied/collect_browser.js +242 -0
  7. verifaied-0.29.0.dev76/src/verifaied/collect_browser.py +296 -0
  8. {verifaied-0.28.0.dev74 → verifaied-0.29.0.dev76}/src/verifaied/instrument.py +20 -38
  9. verifaied-0.29.0.dev76/src/verifaied/instrument_snippets.py +118 -0
  10. {verifaied-0.28.0.dev74 → verifaied-0.29.0.dev76}/src/verifaied/proxy.py +14 -65
  11. {verifaied-0.28.0.dev74 → verifaied-0.29.0.dev76}/src/verifaied/record.py +179 -56
  12. {verifaied-0.28.0.dev74 → verifaied-0.29.0.dev76}/src/verifaied/recorder_driver.js +11 -14
  13. {verifaied-0.28.0.dev74 → verifaied-0.29.0.dev76}/src/verifaied/recorder_refs.js +0 -25
  14. verifaied-0.29.0.dev76/src/verifaied/replay_harvest.ts +77 -0
  15. verifaied-0.29.0.dev76/src/verifaied/runner_dir.py +67 -0
  16. {verifaied-0.28.0.dev74 → verifaied-0.29.0.dev76}/src/verifaied/stepcheck.py +189 -3
  17. {verifaied-0.28.0.dev74 → verifaied-0.29.0.dev76}/src/verifaied/testcmd.py +240 -91
  18. {verifaied-0.28.0.dev74 → verifaied-0.29.0.dev76}/tests/test_agent_mcp.py +65 -1
  19. verifaied-0.29.0.dev76/tests/test_collect_browser.py +621 -0
  20. verifaied-0.29.0.dev76/tests/test_console_label_parity.py +82 -0
  21. {verifaied-0.28.0.dev74 → verifaied-0.29.0.dev76}/tests/test_instrument_detect.py +13 -13
  22. verifaied-0.29.0.dev76/tests/test_instrument_snippets.py +85 -0
  23. {verifaied-0.28.0.dev74 → verifaied-0.29.0.dev76}/tests/test_proxy.py +58 -31
  24. {verifaied-0.28.0.dev74 → verifaied-0.29.0.dev76}/tests/test_record.py +368 -18
  25. {verifaied-0.28.0.dev74 → verifaied-0.29.0.dev76}/tests/test_stepcheck_parity.py +128 -0
  26. {verifaied-0.28.0.dev74 → verifaied-0.29.0.dev76}/tests/test_testcmd.py +424 -89
  27. {verifaied-0.28.0.dev74 → verifaied-0.29.0.dev76}/uv.lock +1 -1
  28. verifaied-0.28.0.dev74/src/verifaied/instrument_snippets.py +0 -245
  29. verifaied-0.28.0.dev74/tests/test_instrument_snippets.py +0 -146
  30. verifaied-0.28.0.dev74/tests/test_reporter_parity.py +0 -98
  31. {verifaied-0.28.0.dev74 → verifaied-0.29.0.dev76}/.gitignore +0 -0
  32. {verifaied-0.28.0.dev74 → verifaied-0.29.0.dev76}/LICENSE +0 -0
  33. {verifaied-0.28.0.dev74 → verifaied-0.29.0.dev76}/src/verifaied/__init__.py +0 -0
  34. {verifaied-0.28.0.dev74 → verifaied-0.29.0.dev76}/src/verifaied/__main__.py +0 -0
  35. {verifaied-0.28.0.dev74 → verifaied-0.29.0.dev76}/src/verifaied/agent_steps.py +0 -0
  36. {verifaied-0.28.0.dev74 → verifaied-0.29.0.dev76}/src/verifaied/analyzer.py +0 -0
  37. {verifaied-0.28.0.dev74 → verifaied-0.29.0.dev76}/src/verifaied/audit.py +0 -0
  38. {verifaied-0.28.0.dev74 → verifaied-0.29.0.dev76}/src/verifaied/client.py +0 -0
  39. {verifaied-0.28.0.dev74 → verifaied-0.29.0.dev76}/src/verifaied/config.py +0 -0
  40. {verifaied-0.28.0.dev74 → verifaied-0.29.0.dev76}/src/verifaied/failure_text.py +0 -0
  41. {verifaied-0.28.0.dev74 → verifaied-0.29.0.dev76}/src/verifaied/instrumenter.js +0 -0
  42. {verifaied-0.28.0.dev74 → verifaied-0.29.0.dev76}/src/verifaied/instrumenter.py +0 -0
  43. {verifaied-0.28.0.dev74 → verifaied-0.29.0.dev76}/src/verifaied/prompts.py +0 -0
  44. {verifaied-0.28.0.dev74 → verifaied-0.29.0.dev76}/src/verifaied/recorder_observer.js +0 -0
  45. {verifaied-0.28.0.dev74 → verifaied-0.29.0.dev76}/src/verifaied/recorder_picker.js +0 -0
  46. {verifaied-0.28.0.dev74 → verifaied-0.29.0.dev76}/src/verifaied/recorder_spec.js +0 -0
  47. {verifaied-0.28.0.dev74 → verifaied-0.29.0.dev76}/src/verifaied/repo_config.py +0 -0
  48. {verifaied-0.28.0.dev74 → verifaied-0.29.0.dev76}/src/verifaied/run_reporter.js +0 -0
  49. {verifaied-0.28.0.dev74 → verifaied-0.29.0.dev76}/src/verifaied/skill/SKILL.md +0 -0
  50. {verifaied-0.28.0.dev74 → verifaied-0.29.0.dev76}/src/verifaied/skill/reference/commands.md +0 -0
  51. {verifaied-0.28.0.dev74 → verifaied-0.29.0.dev76}/src/verifaied/skillcmd.py +0 -0
  52. {verifaied-0.28.0.dev74 → verifaied-0.29.0.dev76}/src/verifaied/source_index.py +0 -0
  53. {verifaied-0.28.0.dev74 → verifaied-0.29.0.dev76}/src/verifaied/subprocess_util.py +0 -0
  54. {verifaied-0.28.0.dev74 → verifaied-0.29.0.dev76}/src/verifaied/suite_report.py +0 -0
  55. {verifaied-0.28.0.dev74 → verifaied-0.29.0.dev76}/src/verifaied/uploader.py +0 -0
  56. {verifaied-0.28.0.dev74 → verifaied-0.29.0.dev76}/src/verifaied/varcmd.py +0 -0
  57. {verifaied-0.28.0.dev74 → verifaied-0.29.0.dev76}/tests/__init__.py +0 -0
  58. {verifaied-0.28.0.dev74 → verifaied-0.29.0.dev76}/tests/conftest.py +0 -0
  59. {verifaied-0.28.0.dev74 → verifaied-0.29.0.dev76}/tests/test_audit.py +0 -0
  60. {verifaied-0.28.0.dev74 → verifaied-0.29.0.dev76}/tests/test_check.py +0 -0
  61. {verifaied-0.28.0.dev74 → verifaied-0.29.0.dev76}/tests/test_check_done.py +0 -0
  62. {verifaied-0.28.0.dev74 → verifaied-0.29.0.dev76}/tests/test_clear_manual.py +0 -0
  63. {verifaied-0.28.0.dev74 → verifaied-0.29.0.dev76}/tests/test_cli.py +0 -0
  64. {verifaied-0.28.0.dev74 → verifaied-0.29.0.dev76}/tests/test_client.py +0 -0
  65. {verifaied-0.28.0.dev74 → verifaied-0.29.0.dev76}/tests/test_config.py +0 -0
  66. {verifaied-0.28.0.dev74 → verifaied-0.29.0.dev76}/tests/test_instrument.py +0 -0
  67. {verifaied-0.28.0.dev74 → verifaied-0.29.0.dev76}/tests/test_instrumenter.py +0 -0
  68. {verifaied-0.28.0.dev74 → verifaied-0.29.0.dev76}/tests/test_prompt_parity.py +0 -0
  69. {verifaied-0.28.0.dev74 → verifaied-0.29.0.dev76}/tests/test_repo_config.py +0 -0
  70. {verifaied-0.28.0.dev74 → verifaied-0.29.0.dev76}/tests/test_skill_parity.py +0 -0
  71. {verifaied-0.28.0.dev74 → verifaied-0.29.0.dev76}/tests/test_skillcmd.py +0 -0
  72. {verifaied-0.28.0.dev74 → verifaied-0.29.0.dev76}/tests/test_source_index.py +0 -0
  73. {verifaied-0.28.0.dev74 → verifaied-0.29.0.dev76}/tests/test_suite_report.py +0 -0
  74. {verifaied-0.28.0.dev74 → verifaied-0.29.0.dev76}/tests/test_uploader.py +0 -0
  75. {verifaied-0.28.0.dev74 → verifaied-0.29.0.dev76}/tests/test_varcmd.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: verifaied
3
- Version: 0.28.0.dev74
3
+ Version: 0.29.0.dev76
4
4
  Summary: Find what's untested in your code — locally, no account required
5
5
  Project-URL: Homepage, https://pypi.org/project/verifaied/
6
6
  Author: Kyle Richards
@@ -257,6 +257,50 @@ supported shapes.
257
257
  uploaded, then exit without contacting the backend. Skips the token
258
258
  requirement so you can audit without configuring auth.
259
259
 
260
+ ## `verifaied collect` — browser coverage from any session
261
+
262
+ Measures how much of your app a browser session reached, and uploads it
263
+ as it runs. Nothing in a page ever posts anything: the coverage is read
264
+ out of the browser from Node, so an instrumented tab never makes a
265
+ request of its own and never has one refused in its console.
266
+
267
+ ```bash
268
+ # Wrap your e2e suite: the Playwright fixture posts each page's coverage
269
+ # here as the suite runs. Exits with the suite's own code.
270
+ verifaied collect --root . -- npx playwright test
271
+
272
+ # Or collect by hand: a browser window opens at your app, and every page
273
+ # you click through in it is read from the outside. Close the window or
274
+ # press Ctrl-C to finish.
275
+ verifaied collect --root . --url http://localhost:5173
276
+ ```
277
+
278
+ A wrapped run uploads as the branch's **suite** contribution; a bare
279
+ session uploads as the **manual** dimension, which accumulates until
280
+ `verifaied clear-manual`. Pages in any browser other than the window it
281
+ opened are not collected, by design.
282
+
283
+ ### `collect` flags
284
+
285
+ - `--url <url>` — where the window a bare session opens should start
286
+ (default: the proxy when `--proxy` is set, else `base_url` from
287
+ `.verifaied/config.toml`)
288
+ - `--browser / --no-browser` — open the window (default), or only listen
289
+ for a harvester of your own posting from Node
290
+ - `--runner-dir <dir>` — where `@playwright/test` is installed, for the
291
+ window (default: found under `--root` and the repo root)
292
+ - `--port <n>` — collector port, 127.0.0.1 only (default: 4571 for a bare
293
+ session, any free port when wrapping a command)
294
+ - `--interval <seconds>` — seconds between uploads (default: 10)
295
+ - `--proxy` / `--proxy-target <origin>` / `--proxy-port <n>` — instrument
296
+ an app with no build step through a local proxy; the window opens at
297
+ the proxy, because only pages loaded through it are instrumented
298
+ - `--driver <human|agent|suite>` — who drove, for the session ledger
299
+ - `--test-results <dir>` — where a wrapped run leaves Playwright traces
300
+ and screenshots to attach (default: `./test-results`)
301
+ - `--repo / -r`, `--branch / -b`, `--root <dir>`, `--commit-sha`,
302
+ `--env <name>`, `--api-url`, `--token` — as for `upload`
303
+
260
304
  ## `verifaied check-done` — the "am I done?" gate
261
305
 
262
306
  `check-done` is the **stop condition** for an agent's test-writing loop.
@@ -956,7 +1000,9 @@ Shared by `record`, `run` and `studio`:
956
1000
  - `--root <dir>` — repo root the spec paths resolve against (default: cwd)
957
1001
  - `--runner-dir <dir>` — where to run `npx playwright` from (default: the
958
1002
  first directory under `--root` with `@playwright/test` installed)
959
- - `--port <n>` — coverage collector port, 127.0.0.1 only (default: 4571)
1003
+ - `--port <n>` — pin the coverage collector's port, 127.0.0.1 only
1004
+ (default: any free one; a replay's harvest reads the bound port from its
1005
+ own environment)
960
1006
  - `--interval <seconds>` — seconds between coverage uploads during a
961
1007
  replay (default: 10)
962
1008
  - `--env <name>` — which environment from `.verifaied/config.toml` to run
@@ -211,6 +211,50 @@ supported shapes.
211
211
  uploaded, then exit without contacting the backend. Skips the token
212
212
  requirement so you can audit without configuring auth.
213
213
 
214
+ ## `verifaied collect` — browser coverage from any session
215
+
216
+ Measures how much of your app a browser session reached, and uploads it
217
+ as it runs. Nothing in a page ever posts anything: the coverage is read
218
+ out of the browser from Node, so an instrumented tab never makes a
219
+ request of its own and never has one refused in its console.
220
+
221
+ ```bash
222
+ # Wrap your e2e suite: the Playwright fixture posts each page's coverage
223
+ # here as the suite runs. Exits with the suite's own code.
224
+ verifaied collect --root . -- npx playwright test
225
+
226
+ # Or collect by hand: a browser window opens at your app, and every page
227
+ # you click through in it is read from the outside. Close the window or
228
+ # press Ctrl-C to finish.
229
+ verifaied collect --root . --url http://localhost:5173
230
+ ```
231
+
232
+ A wrapped run uploads as the branch's **suite** contribution; a bare
233
+ session uploads as the **manual** dimension, which accumulates until
234
+ `verifaied clear-manual`. Pages in any browser other than the window it
235
+ opened are not collected, by design.
236
+
237
+ ### `collect` flags
238
+
239
+ - `--url <url>` — where the window a bare session opens should start
240
+ (default: the proxy when `--proxy` is set, else `base_url` from
241
+ `.verifaied/config.toml`)
242
+ - `--browser / --no-browser` — open the window (default), or only listen
243
+ for a harvester of your own posting from Node
244
+ - `--runner-dir <dir>` — where `@playwright/test` is installed, for the
245
+ window (default: found under `--root` and the repo root)
246
+ - `--port <n>` — collector port, 127.0.0.1 only (default: 4571 for a bare
247
+ session, any free port when wrapping a command)
248
+ - `--interval <seconds>` — seconds between uploads (default: 10)
249
+ - `--proxy` / `--proxy-target <origin>` / `--proxy-port <n>` — instrument
250
+ an app with no build step through a local proxy; the window opens at
251
+ the proxy, because only pages loaded through it are instrumented
252
+ - `--driver <human|agent|suite>` — who drove, for the session ledger
253
+ - `--test-results <dir>` — where a wrapped run leaves Playwright traces
254
+ and screenshots to attach (default: `./test-results`)
255
+ - `--repo / -r`, `--branch / -b`, `--root <dir>`, `--commit-sha`,
256
+ `--env <name>`, `--api-url`, `--token` — as for `upload`
257
+
214
258
  ## `verifaied check-done` — the "am I done?" gate
215
259
 
216
260
  `check-done` is the **stop condition** for an agent's test-writing loop.
@@ -910,7 +954,9 @@ Shared by `record`, `run` and `studio`:
910
954
  - `--root <dir>` — repo root the spec paths resolve against (default: cwd)
911
955
  - `--runner-dir <dir>` — where to run `npx playwright` from (default: the
912
956
  first directory under `--root` with `@playwright/test` installed)
913
- - `--port <n>` — coverage collector port, 127.0.0.1 only (default: 4571)
957
+ - `--port <n>` — pin the coverage collector's port, 127.0.0.1 only
958
+ (default: any free one; a replay's harvest reads the bound port from its
959
+ own environment)
914
960
  - `--interval <seconds>` — seconds between coverage uploads during a
915
961
  replay (default: 10)
916
962
  - `--env <name>` — which environment from `.verifaied/config.toml` to run
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "verifaied"
3
- version = "0.28.0.dev74"
3
+ version = "0.29.0.dev76"
4
4
  description = "Find what's untested in your code — locally, no account required"
5
5
  readme = "README.md"
6
6
  requires-python = ">=3.10"
@@ -66,8 +66,14 @@ from verifaied.agent_steps import (
66
66
  report,
67
67
  )
68
68
  from verifaied.config import web_url_for
69
- from verifaied.stepcheck import MAX_FLOW_STEPS, validate_step_line
69
+ from verifaied.stepcheck import (
70
+ MAX_FLOW_STEPS,
71
+ MAX_SCRIPT_CHARS,
72
+ compose_script,
73
+ validate_step_line,
74
+ )
70
75
  from verifaied.testcmd import (
76
+ CONSOLE_LABEL,
71
77
  ProcessRunner,
72
78
  RecordedTestError,
73
79
  RecordOutcome,
@@ -186,7 +192,7 @@ Guessing what a page will look like after a click is how flows break.
186
192
 
187
193
  {_LOCATOR_DOCTRINE}
188
194
 
189
- Each step is ONE of two shapes and nothing else:
195
+ Each step is ONE of three shapes and nothing else:
190
196
 
191
197
  - an ACTION (kind "action"): `await page.<locator>.<verb>(...);` with
192
198
  <verb> one of click, dblclick, fill, press, check, uncheck,
@@ -194,11 +200,32 @@ Each step is ONE of two shapes and nothing else:
194
200
  the one action with no locator.
195
201
  - an ASSERTION (kind "assert"): `await expect(page...)....;` with any
196
202
  @playwright/test matcher.
203
+ - a SCRIPT (kind "script"): JavaScript that runs inside the page
204
+ itself, exactly as if it were typed into the browser's dev console.
205
+ It is for state no click can reach — an event fired from outside the
206
+ page (`window.dispatchEvent(new Event('offline'))`), a seeded storage
207
+ key, a flag in the page's own state. Give the script's BODY only: it
208
+ is wrapped in `await page.evaluate(async () => {{ … }});` for you, so
209
+ `await` works inside it. It lands in the flow as an action labelled
210
+ "Run a script in the page".
197
211
 
198
212
  Rules for the code:
199
213
 
200
- - One self-contained statement per step, ending in a semicolon. No
201
- loops, no variables, no `if`, no comments, no `waitForTimeout`.
214
+ - An action or an assertion is one self-contained statement, ending in
215
+ a semicolon. No loops, no variables, no `if`, no comments, no
216
+ `waitForTimeout`.
217
+ - A script may run several statements over several lines, and it must
218
+ be complete on its own: every bracket, string and comment it opens, it
219
+ closes. It runs in the page, so it cannot name a declared variable —
220
+ `process.env` does not exist there — nor `page`, a ref or `expect`;
221
+ find elements with `document.querySelector`. No `/` outside a comment
222
+ (no regular-expression literal, no division — use `new RegExp('…')`),
223
+ no `${{...}}`, `require`, `import` or `eval`.
224
+ At most {MAX_SCRIPT_CHARS} characters.
225
+ - Reach for a script only when no action can do it. Clicking, typing
226
+ and navigating are always actions. A script checks nothing and has
227
+ the same 5 seconds as an action, so do not wait inside it: follow it
228
+ with an assertion that proves the page reacted.
202
229
  - A declared variable may be named as `process.env.VERIFAIED_VAR_<NAME>`,
203
230
  unquoted. Nothing else reaches outside the statement: no `require`,
204
231
  no `import`, no `${{...}}`.
@@ -276,8 +303,9 @@ today and fails next week. Replace it: `delete_step` the line and queue a
276
303
  corrected one naming the stable part yourself (`getByText(/Matches/)`,
277
304
  `toContainText('Matches')`).
278
305
 
279
- Arguments: `steps` is a list of `{{"kind": "action"|"assert", "code":
280
- "<one statement>", "label": "<a few words saying what it does>"}}`."""
306
+ Arguments: `steps` is a list of `{{"kind": "action"|"assert"|"script", "code":
307
+ "<one statement, or a script's body>", "label": "<a few words saying what it
308
+ does>"}}`."""
281
309
 
282
310
 
283
311
  # What each ending actually does, so a refusal says why waiting for the
@@ -744,12 +772,22 @@ class AgentRecorderServer:
744
772
  for entry in steps:
745
773
  fields = entry if isinstance(entry, dict) else {}
746
774
  kind = str(fields.get("kind") or "")
747
- code = str(fields.get("code") or "").strip()
775
+ # The raw text as well as the stripped statement: a script's
776
+ # first line may be indented on purpose, and the composer
777
+ # keeps it that way.
778
+ raw = str(fields.get("code") or "")
779
+ code = raw.strip()
748
780
  label = str(fields.get("label") or "").strip() or code
749
- reason = validate_step_line(kind, code)
781
+ reason = validate_step_line(kind, raw if kind == "script" else code)
750
782
  if reason:
751
783
  entries.append(f"REFUSED — {reason}")
752
784
  continue
785
+ if kind == "script":
786
+ # The body is the agent's; the wrapper and the label are
787
+ # not. Composed exactly as the studio's Run Code tab
788
+ # composes, so the block reopens there and keeps its name
789
+ # once the spec is saved.
790
+ kind, code, label = "action", compose_script(raw), CONSOLE_LABEL
753
791
  accepted.append({"kind": kind, "code": code, "label": label})
754
792
  entries.append(
755
793
  AgentStep(
@@ -713,10 +713,37 @@ def record_command(
713
713
  "--port",
714
714
  help=(
715
715
  "Port the collector listens on (127.0.0.1 only). Default: "
716
- f"{DEFAULT_PORT} for a bare session (what the in-page reporter "
717
- "assumes), an ephemeral port when wrapping a command — the "
718
- "child learns the real port from $VERIFAIED_RECORD_PORT, so a "
719
- "wrapped run can coexist with a bare session."
716
+ f"{DEFAULT_PORT} for a bare session, an ephemeral port when "
717
+ "wrapping a command — the child learns the real port from "
718
+ "$VERIFAIED_RECORD_PORT, so a wrapped run can coexist with a "
719
+ "bare session."
720
+ ),
721
+ ),
722
+ url: str | None = typer.Option(
723
+ None,
724
+ "--url",
725
+ help=(
726
+ "Where the browser window a bare session opens should start, "
727
+ "e.g. http://localhost:5173. Defaults to the proxy when --proxy "
728
+ f"is set, else base_url from {CONFIG_PATH}."
729
+ ),
730
+ ),
731
+ browser: bool = typer.Option(
732
+ True,
733
+ "--browser/--no-browser",
734
+ help=(
735
+ "A bare session opens a browser window and collects from it "
736
+ "(default). --no-browser only listens, for posting coverage from "
737
+ "your own Playwright fixture."
738
+ ),
739
+ ),
740
+ runner_dir: Path | None = typer.Option(
741
+ None,
742
+ "--runner-dir",
743
+ help=(
744
+ "Directory @playwright/test is installed in, for the window a "
745
+ "bare session opens. Found under --root and the repo root when "
746
+ "omitted."
720
747
  ),
721
748
  ),
722
749
  interval: float = typer.Option(
@@ -749,9 +776,9 @@ def record_command(
749
776
  "Put an instrumenting reverse proxy in front of your running "
750
777
  "app, for an app with no build step to hang a coverage plugin "
751
778
  "off. Every JavaScript response is rewritten with "
752
- "istanbul-lib-instrument and every HTML response gets the "
753
- "reporter injected — browse the proxy URL this prints, not the "
754
- "app's own port."
779
+ "istanbul-lib-instrument on the way past; the window a bare "
780
+ "session opens points at the proxy, and only pages loaded "
781
+ "through it are instrumented."
755
782
  ),
756
783
  ),
757
784
  proxy_target: str | None = typer.Option(
@@ -796,41 +823,45 @@ def record_command(
796
823
  commit. For a named flow you can replay forever, that is
797
824
  ``verifaied test record``.
798
825
 
799
- Runs a localhost-only collector that the in-page reporter (and the
800
- Playwright fixture, see ``verifaied instrument``) streams
801
- ``window.__coverage__`` to. It merges the chunks and uploads them as
802
- browser coverage every ``--interval`` seconds — so the Browser Tests
803
- panel fills in as the session runs, with no report files to manage.
826
+ Runs a localhost-only collector that ``window.__coverage__`` is posted
827
+ to — from the Playwright fixture (see ``verifaied instrument``) when
828
+ wrapping a suite, and from the browser window this opens when run
829
+ bare. Nothing in a page ever posts anything. The collector merges the
830
+ chunks and uploads them as browser coverage every ``--interval``
831
+ seconds — so the Browser Tests panel fills in as the session runs,
832
+ with no report files to manage.
804
833
 
805
834
  **Wrap your e2e suite** — one command, nothing to start by hand::
806
835
 
807
836
  verifaied collect --root . -- npx playwright test
808
837
 
809
- The wrapped command inherits ``VITE_COVERAGE=true`` and
810
- ``VERIFAIED_RECORD_PORT``, which Playwright's ``webServer`` forwards to
811
- the dev server, turning on instrumentation and the reporter. This exits
838
+ The wrapped command inherits ``VITE_COVERAGE=true`` (which Playwright's
839
+ ``webServer`` forwards to the dev server, turning on instrumentation)
840
+ and ``VERIFAIED_RECORD_PORT`` (which the fixture posts to). This exits
812
841
  with the command's own code, so it's transparent in CI.
813
842
 
814
843
  **Or run it bare, for a manual pass** — two terminals: start your dev
815
844
  server instrumented (``VITE_COVERAGE=true``, e.g. ``make frontend-cov``),
816
- then run this. Click around, Ctrl-C when done; the final state is
817
- flushed on exit.
845
+ then run this. A browser window opens at your app (``--url``, or
846
+ ``base_url`` from ``.verifaied/config.toml``); click through it there.
847
+ Pages in any other browser aren't collected. Close the window or press
848
+ Ctrl-C when done; the final state is flushed on exit.
818
849
 
819
850
  **No build step?** Add ``--proxy``. Both forms above assume something
820
851
  compiles your JavaScript and can be asked to instrument it on the way
821
852
  past. An app that serves hand-written ES modules straight off disk has
822
853
  nowhere to put a plugin, so the instrumentation moves to the last place
823
854
  the bytes go before the browser: a local proxy in front of your running
824
- app, rewriting the JavaScript and injecting the reporter itself::
855
+ app, rewriting the JavaScript on the way past::
825
856
 
826
857
  verifaied collect --root . --proxy
827
858
 
828
859
  It forwards to ``base_url`` from ``.verifaied/config.toml`` (or
829
860
  ``--proxy-target``), needs ``istanbul-lib-instrument`` installed in your
830
- repo, and changes nothing about the app. Browse the URL it prints — a
831
- page loaded from the app's own port bypasses all of this and produces
832
- no coverage, which is why the summary at the end says so if none
833
- arrived.
861
+ repo, and changes nothing about the app. The window opens at the proxy
862
+ — a page loaded from the app's own port bypasses all of this and
863
+ produces no coverage, which is why the summary at the end says so if
864
+ none arrived.
834
865
 
835
866
  The two modes feed two different dimensions, because they answer
836
867
  different questions. Wrapping a command uploads as this branch's
@@ -896,6 +927,9 @@ def record_command(
896
927
  proxy_target=proxy_target,
897
928
  proxy_port=proxy_port,
898
929
  env_name=env_name,
930
+ url=url,
931
+ browser=browser,
932
+ runner_dir=runner_dir,
899
933
  console=console,
900
934
  err_console=err_console,
901
935
  )
@@ -1400,11 +1434,11 @@ def test_studio_command(
1400
1434
  ),
1401
1435
  ),
1402
1436
  port: int = typer.Option(
1403
- DEFAULT_PORT,
1437
+ 0,
1404
1438
  "--port",
1405
1439
  help=(
1406
- "Port the coverage collector listens on during each replay "
1407
- "(127.0.0.1 only)."
1440
+ "Pin the port the coverage collector listens on during each "
1441
+ "replay (127.0.0.1 only). Default: any free one."
1408
1442
  ),
1409
1443
  ),
1410
1444
  interval: float = typer.Option(
@@ -1556,11 +1590,11 @@ def test_mcp_command(
1556
1590
  ),
1557
1591
  ),
1558
1592
  port: int = typer.Option(
1559
- DEFAULT_PORT,
1593
+ 0,
1560
1594
  "--port",
1561
1595
  help=(
1562
- "Port the coverage collector listens on during each replay "
1563
- "(127.0.0.1 only)."
1596
+ "Pin the port the coverage collector listens on during each "
1597
+ "replay (127.0.0.1 only). Default: any free one."
1564
1598
  ),
1565
1599
  ),
1566
1600
  interval: float = typer.Option(
@@ -1722,10 +1756,11 @@ def test_record_command(
1722
1756
  ),
1723
1757
  ),
1724
1758
  port: int = typer.Option(
1725
- DEFAULT_PORT,
1759
+ 0,
1726
1760
  "--port",
1727
1761
  help=(
1728
- "Port the coverage collector listens on during the replay (127.0.0.1 only)."
1762
+ "Pin the port the coverage collector listens on during the "
1763
+ "replay (127.0.0.1 only). Default: any free one."
1729
1764
  ),
1730
1765
  ),
1731
1766
  interval: float = typer.Option(
@@ -1981,9 +2016,12 @@ def test_run_command(
1981
2016
  ),
1982
2017
  ),
1983
2018
  port: int = typer.Option(
1984
- DEFAULT_PORT,
2019
+ 0,
1985
2020
  "--port",
1986
- help="Port the coverage collector listens on (127.0.0.1 only).",
2021
+ help=(
2022
+ "Pin the port the coverage collector listens on (127.0.0.1 only). "
2023
+ "Default: any free one."
2024
+ ),
1987
2025
  ),
1988
2026
  interval: float = typer.Option(
1989
2027
  DEFAULT_INTERVAL,
@@ -2137,10 +2175,11 @@ def test_edit_command(
2137
2175
  ),
2138
2176
  ),
2139
2177
  port: int = typer.Option(
2140
- DEFAULT_PORT,
2178
+ 0,
2141
2179
  "--port",
2142
2180
  help=(
2143
- "Port the coverage collector listens on during the replay (127.0.0.1 only)."
2181
+ "Pin the port the coverage collector listens on during the "
2182
+ "replay (127.0.0.1 only). Default: any free one."
2144
2183
  ),
2145
2184
  ),
2146
2185
  interval: float = typer.Option(
@@ -0,0 +1,242 @@
1
+ // The browser window a bare `verifaied collect` opens.
2
+ //
3
+ // Coverage leaves a page only through Node. The window this opens is a
4
+ // Playwright context, so the counters every page accumulates are read out
5
+ // of it from here — a binding call on unload, and a timer read for pages
6
+ // still open — and posted to the collector the CLI already bound. No
7
+ // script in any page ever makes a network request of its own.
8
+ //
9
+ // Same sidecar shape as `recorder_driver.js` and `instrumenter.js`: the
10
+ // config arrives as a JSON file named by argv[2], events go out as JSON
11
+ // lines on stdout, and commands come in as JSON lines on stdin. It
12
+ // resolves `@playwright/test` from the directory the CLI spawned it in —
13
+ // the user's own runner directory — because this file ships inside a
14
+ // Python wheel with no node_modules above it.
15
+ //
16
+ // Protocol
17
+ // in {cmd:"close"} finish: final read, then exit
18
+ // out {type:"ready"} the window is open
19
+ // {type:"warning", message} something non-fatal to print
20
+ // {type:"error", message} could not open; exit 1 follows
21
+ // {type:"closed", reason} "asked" (stdin) or "window"
22
+ // (the user closed it); exit 0
23
+
24
+ const crypto = require("crypto");
25
+ const fs = require("fs");
26
+ const readline = require("readline");
27
+
28
+ // Line-delimited JSON on fd 1, written synchronously — the same helper
29
+ // every driver the CLI spawns carries, for the same reason: an async
30
+ // stdout would lose whatever was buffered if the process went away.
31
+ function emit(event) {
32
+ const line = Buffer.from(JSON.stringify(event) + "\n");
33
+ let written = 0;
34
+ while (written < line.length) {
35
+ try {
36
+ written += fs.writeSync(1, line, written);
37
+ } catch (err) {
38
+ if (err.code !== "EAGAIN") return;
39
+ }
40
+ }
41
+ }
42
+
43
+ function loadPlaywright() {
44
+ const entry = require.resolve("@playwright/test", { paths: [process.cwd()] });
45
+ return require(entry);
46
+ }
47
+
48
+ const BINDING = "__verifaiedHarvest";
49
+
50
+ // Serialised into every document by `addInitScript`, so it references
51
+ // nothing from this file's scope.
52
+ function unloadHook() {
53
+ window.addEventListener("beforeunload", () => {
54
+ const w = window;
55
+ if (w.__coverage__ && w.__verifaiedHarvest) {
56
+ w.__verifaiedHarvest(JSON.stringify(w.__coverage__));
57
+ }
58
+ });
59
+ }
60
+
61
+ function readCoverage() {
62
+ return window.__coverage__ ? JSON.stringify(window.__coverage__) : null;
63
+ }
64
+
65
+ function digest(text) {
66
+ return crypto.createHash("sha1").update(text).digest("hex");
67
+ }
68
+
69
+ /**
70
+ * Open the window and keep reading it until asked to stop or it closes.
71
+ *
72
+ * `chromium`, `input`, `emit` and `fetchImpl` are parameters so a test can
73
+ * drive this with a fake browser and a fake stdin; `main` below is the
74
+ * only place the real ones are wired in. Resolves to the exit code.
75
+ */
76
+ async function run({ chromium, config, input, emit, fetchImpl = fetch }) {
77
+ const { url, port, userDataDir } = config;
78
+ const intervalMs = config.intervalMs || 5000;
79
+ const sink = `http://127.0.0.1:${port}/coverage`;
80
+
81
+ // Every post is tracked so the final step can wait for the ones a
82
+ // closing window fired through the binding a moment ago.
83
+ const pending = new Set();
84
+ const post = (body) => {
85
+ const request = fetchImpl(sink, {
86
+ method: "POST",
87
+ headers: { "content-type": "application/json" },
88
+ body,
89
+ }).catch(() => {});
90
+ pending.add(request);
91
+ request.finally(() => pending.delete(request));
92
+ return request;
93
+ };
94
+
95
+ let context;
96
+ try {
97
+ context = await chromium.launchPersistentContext(userDataDir, {
98
+ headless: false,
99
+ viewport: null,
100
+ });
101
+ } catch (err) {
102
+ emit({
103
+ type: "error",
104
+ message: `could not open a browser window: ${(err && err.message) || err}`,
105
+ });
106
+ return 1;
107
+ }
108
+ await context.exposeBinding(BINDING, (_source, body) => post(body));
109
+ await context.addInitScript(unloadHook);
110
+
111
+ // One read per page per change: the counters are cumulative, so an
112
+ // unchanged snapshot carries nothing the collector hasn't merged.
113
+ const lastSent = new WeakMap();
114
+ const readPage = async (page) => {
115
+ let body;
116
+ try {
117
+ body = await page.evaluate(readCoverage);
118
+ } catch {
119
+ return; // navigating, or already gone
120
+ }
121
+ if (!body) return;
122
+ const hash = digest(body);
123
+ if (lastSent.get(page) === hash) return;
124
+ lastSent.set(page, hash);
125
+ await post(body);
126
+ };
127
+ // One read at a time, and the read in flight is kept rather than
128
+ // dropped: the final read on close has to wait for a tick that is
129
+ // still walking the pages and then walk them all again itself, or the
130
+ // pages that tick had not reached would lose their last counters.
131
+ let inflight = null;
132
+ const readAll = () => {
133
+ if (inflight) return inflight;
134
+ inflight = (async () => {
135
+ try {
136
+ for (const page of context.pages()) await readPage(page);
137
+ } finally {
138
+ inflight = null;
139
+ }
140
+ })();
141
+ return inflight;
142
+ };
143
+
144
+ let finished;
145
+ const done = new Promise((resolve) => {
146
+ finished = resolve;
147
+ });
148
+ let closing = false;
149
+ let timer = null;
150
+
151
+ const finish = async (reason) => {
152
+ if (closing) return;
153
+ closing = true;
154
+ if (timer !== null) clearInterval(timer);
155
+ if (reason === "asked") {
156
+ // Whatever a tick was in the middle of, then everything once more:
157
+ // nothing is lost between the last tick and now.
158
+ if (inflight) await inflight.catch(() => {});
159
+ await readAll().catch(() => {});
160
+ try {
161
+ await context.close();
162
+ } catch {
163
+ // Already closed underneath us.
164
+ }
165
+ }
166
+ await Promise.allSettled([...pending]);
167
+ emit({ type: "closed", reason });
168
+ finished(0);
169
+ };
170
+
171
+ // Registered before the first navigation, which can take a while: a
172
+ // window the user shuts during it has to be reported, not missed.
173
+ context.on("close", () => {
174
+ finish("window").catch(() => finished(0));
175
+ });
176
+ const lines = readline.createInterface({ input });
177
+ lines.on("line", (line) => {
178
+ let command;
179
+ try {
180
+ command = JSON.parse(line);
181
+ } catch {
182
+ return;
183
+ }
184
+ if (command && command.cmd === "close") finish("asked").catch(() => finished(0));
185
+ });
186
+ // EOF on stdin is the parent going away: finish as if asked.
187
+ lines.on("close", () => {
188
+ finish("asked").catch(() => finished(0));
189
+ });
190
+
191
+ const first = context.pages()[0] || (await context.newPage());
192
+ if (url && !closing) {
193
+ try {
194
+ await first.goto(url);
195
+ } catch (err) {
196
+ if (!closing) {
197
+ emit({
198
+ type: "warning",
199
+ message: `could not open ${url}: ${(err && err.message) || err}`,
200
+ });
201
+ }
202
+ }
203
+ }
204
+ if (closing) return done;
205
+ emit({ type: "ready" });
206
+ timer = setInterval(() => {
207
+ if (!closing) readAll().catch(() => {});
208
+ }, intervalMs);
209
+
210
+ return done;
211
+ }
212
+
213
+ function main() {
214
+ let config;
215
+ try {
216
+ config = JSON.parse(fs.readFileSync(process.argv[2], "utf8"));
217
+ } catch (err) {
218
+ emit({ type: "error", message: `could not read the config: ${err.message}` });
219
+ process.exit(1);
220
+ }
221
+ let playwright;
222
+ try {
223
+ playwright = loadPlaywright();
224
+ } catch (err) {
225
+ emit({
226
+ type: "error",
227
+ message: `could not load @playwright/test from ${process.cwd()}: ${err.message}`,
228
+ });
229
+ process.exit(1);
230
+ }
231
+ run({ chromium: playwright.chromium, config, input: process.stdin, emit }).then(
232
+ (code) => process.exit(code),
233
+ (err) => {
234
+ emit({ type: "error", message: String((err && err.message) || err) });
235
+ process.exit(1);
236
+ },
237
+ );
238
+ }
239
+
240
+ module.exports = { run, BINDING };
241
+
242
+ if (require.main === module) main();