verifaied 0.31.1.dev79__tar.gz → 0.32.0.dev80__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {verifaied-0.31.1.dev79 → verifaied-0.32.0.dev80}/PKG-INFO +9 -2
- {verifaied-0.31.1.dev79 → verifaied-0.32.0.dev80}/README.md +8 -1
- {verifaied-0.31.1.dev79 → verifaied-0.32.0.dev80}/pyproject.toml +1 -1
- {verifaied-0.31.1.dev79 → verifaied-0.32.0.dev80}/src/verifaied/client.py +34 -0
- {verifaied-0.31.1.dev79 → verifaied-0.32.0.dev80}/src/verifaied/testcmd.py +282 -26
- {verifaied-0.31.1.dev79 → verifaied-0.32.0.dev80}/tests/test_client.py +54 -0
- {verifaied-0.31.1.dev79 → verifaied-0.32.0.dev80}/tests/test_testcmd.py +654 -1
- {verifaied-0.31.1.dev79 → verifaied-0.32.0.dev80}/uv.lock +1 -1
- {verifaied-0.31.1.dev79 → verifaied-0.32.0.dev80}/.gitignore +0 -0
- {verifaied-0.31.1.dev79 → verifaied-0.32.0.dev80}/LICENSE +0 -0
- {verifaied-0.31.1.dev79 → verifaied-0.32.0.dev80}/src/verifaied/__init__.py +0 -0
- {verifaied-0.31.1.dev79 → verifaied-0.32.0.dev80}/src/verifaied/__main__.py +0 -0
- {verifaied-0.31.1.dev79 → verifaied-0.32.0.dev80}/src/verifaied/agent_mcp.py +0 -0
- {verifaied-0.31.1.dev79 → verifaied-0.32.0.dev80}/src/verifaied/agent_steps.py +0 -0
- {verifaied-0.31.1.dev79 → verifaied-0.32.0.dev80}/src/verifaied/analyzer.py +0 -0
- {verifaied-0.31.1.dev79 → verifaied-0.32.0.dev80}/src/verifaied/audit.py +0 -0
- {verifaied-0.31.1.dev79 → verifaied-0.32.0.dev80}/src/verifaied/cli.py +0 -0
- {verifaied-0.31.1.dev79 → verifaied-0.32.0.dev80}/src/verifaied/collect_browser.js +0 -0
- {verifaied-0.31.1.dev79 → verifaied-0.32.0.dev80}/src/verifaied/collect_browser.py +0 -0
- {verifaied-0.31.1.dev79 → verifaied-0.32.0.dev80}/src/verifaied/config.py +0 -0
- {verifaied-0.31.1.dev79 → verifaied-0.32.0.dev80}/src/verifaied/failure_text.py +0 -0
- {verifaied-0.31.1.dev79 → verifaied-0.32.0.dev80}/src/verifaied/instrument.py +0 -0
- {verifaied-0.31.1.dev79 → verifaied-0.32.0.dev80}/src/verifaied/instrument_snippets.py +0 -0
- {verifaied-0.31.1.dev79 → verifaied-0.32.0.dev80}/src/verifaied/instrumenter.js +0 -0
- {verifaied-0.31.1.dev79 → verifaied-0.32.0.dev80}/src/verifaied/instrumenter.py +0 -0
- {verifaied-0.31.1.dev79 → verifaied-0.32.0.dev80}/src/verifaied/prompts.py +0 -0
- {verifaied-0.31.1.dev79 → verifaied-0.32.0.dev80}/src/verifaied/proxy.py +0 -0
- {verifaied-0.31.1.dev79 → verifaied-0.32.0.dev80}/src/verifaied/record.py +0 -0
- {verifaied-0.31.1.dev79 → verifaied-0.32.0.dev80}/src/verifaied/recorder_driver.js +0 -0
- {verifaied-0.31.1.dev79 → verifaied-0.32.0.dev80}/src/verifaied/recorder_observer.js +0 -0
- {verifaied-0.31.1.dev79 → verifaied-0.32.0.dev80}/src/verifaied/recorder_picker.js +0 -0
- {verifaied-0.31.1.dev79 → verifaied-0.32.0.dev80}/src/verifaied/recorder_refs.js +0 -0
- {verifaied-0.31.1.dev79 → verifaied-0.32.0.dev80}/src/verifaied/recorder_spec.js +0 -0
- {verifaied-0.31.1.dev79 → verifaied-0.32.0.dev80}/src/verifaied/replay_harvest.ts +0 -0
- {verifaied-0.31.1.dev79 → verifaied-0.32.0.dev80}/src/verifaied/repo_config.py +0 -0
- {verifaied-0.31.1.dev79 → verifaied-0.32.0.dev80}/src/verifaied/run_reporter.js +0 -0
- {verifaied-0.31.1.dev79 → verifaied-0.32.0.dev80}/src/verifaied/runner_dir.py +0 -0
- {verifaied-0.31.1.dev79 → verifaied-0.32.0.dev80}/src/verifaied/skill/SKILL.md +0 -0
- {verifaied-0.31.1.dev79 → verifaied-0.32.0.dev80}/src/verifaied/skill/reference/commands.md +0 -0
- {verifaied-0.31.1.dev79 → verifaied-0.32.0.dev80}/src/verifaied/skillcmd.py +0 -0
- {verifaied-0.31.1.dev79 → verifaied-0.32.0.dev80}/src/verifaied/source_index.py +0 -0
- {verifaied-0.31.1.dev79 → verifaied-0.32.0.dev80}/src/verifaied/stepcheck.py +0 -0
- {verifaied-0.31.1.dev79 → verifaied-0.32.0.dev80}/src/verifaied/subprocess_util.py +0 -0
- {verifaied-0.31.1.dev79 → verifaied-0.32.0.dev80}/src/verifaied/suite_report.py +0 -0
- {verifaied-0.31.1.dev79 → verifaied-0.32.0.dev80}/src/verifaied/uploader.py +0 -0
- {verifaied-0.31.1.dev79 → verifaied-0.32.0.dev80}/src/verifaied/varcmd.py +0 -0
- {verifaied-0.31.1.dev79 → verifaied-0.32.0.dev80}/tests/__init__.py +0 -0
- {verifaied-0.31.1.dev79 → verifaied-0.32.0.dev80}/tests/conftest.py +0 -0
- {verifaied-0.31.1.dev79 → verifaied-0.32.0.dev80}/tests/test_agent_mcp.py +0 -0
- {verifaied-0.31.1.dev79 → verifaied-0.32.0.dev80}/tests/test_audit.py +0 -0
- {verifaied-0.31.1.dev79 → verifaied-0.32.0.dev80}/tests/test_browser_settings_parity.py +0 -0
- {verifaied-0.31.1.dev79 → verifaied-0.32.0.dev80}/tests/test_caption_parity.py +0 -0
- {verifaied-0.31.1.dev79 → verifaied-0.32.0.dev80}/tests/test_check.py +0 -0
- {verifaied-0.31.1.dev79 → verifaied-0.32.0.dev80}/tests/test_check_done.py +0 -0
- {verifaied-0.31.1.dev79 → verifaied-0.32.0.dev80}/tests/test_clear_manual.py +0 -0
- {verifaied-0.31.1.dev79 → verifaied-0.32.0.dev80}/tests/test_cli.py +0 -0
- {verifaied-0.31.1.dev79 → verifaied-0.32.0.dev80}/tests/test_collect_browser.py +0 -0
- {verifaied-0.31.1.dev79 → verifaied-0.32.0.dev80}/tests/test_config.py +0 -0
- {verifaied-0.31.1.dev79 → verifaied-0.32.0.dev80}/tests/test_console_label_parity.py +0 -0
- {verifaied-0.31.1.dev79 → verifaied-0.32.0.dev80}/tests/test_instrument.py +0 -0
- {verifaied-0.31.1.dev79 → verifaied-0.32.0.dev80}/tests/test_instrument_detect.py +0 -0
- {verifaied-0.31.1.dev79 → verifaied-0.32.0.dev80}/tests/test_instrument_snippets.py +0 -0
- {verifaied-0.31.1.dev79 → verifaied-0.32.0.dev80}/tests/test_instrumenter.py +0 -0
- {verifaied-0.31.1.dev79 → verifaied-0.32.0.dev80}/tests/test_pause_parity.py +0 -0
- {verifaied-0.31.1.dev79 → verifaied-0.32.0.dev80}/tests/test_prompt_parity.py +0 -0
- {verifaied-0.31.1.dev79 → verifaied-0.32.0.dev80}/tests/test_proxy.py +0 -0
- {verifaied-0.31.1.dev79 → verifaied-0.32.0.dev80}/tests/test_record.py +0 -0
- {verifaied-0.31.1.dev79 → verifaied-0.32.0.dev80}/tests/test_replay_harvest_runner.py +0 -0
- {verifaied-0.31.1.dev79 → verifaied-0.32.0.dev80}/tests/test_repo_config.py +0 -0
- {verifaied-0.31.1.dev79 → verifaied-0.32.0.dev80}/tests/test_skill_parity.py +0 -0
- {verifaied-0.31.1.dev79 → verifaied-0.32.0.dev80}/tests/test_skillcmd.py +0 -0
- {verifaied-0.31.1.dev79 → verifaied-0.32.0.dev80}/tests/test_source_index.py +0 -0
- {verifaied-0.31.1.dev79 → verifaied-0.32.0.dev80}/tests/test_stepcheck_parity.py +0 -0
- {verifaied-0.31.1.dev79 → verifaied-0.32.0.dev80}/tests/test_suite_report.py +0 -0
- {verifaied-0.31.1.dev79 → verifaied-0.32.0.dev80}/tests/test_uploader.py +0 -0
- {verifaied-0.31.1.dev79 → verifaied-0.32.0.dev80}/tests/test_varcmd.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: verifaied
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.32.0.dev80
|
|
4
4
|
Summary: Find what's untested in your code — locally, no account required
|
|
5
5
|
Project-URL: Homepage, https://pypi.org/project/verifaied/
|
|
6
6
|
Author: Kyle Richards
|
|
@@ -1054,11 +1054,18 @@ cloud can stop it. A request that fails is reported with the reason and
|
|
|
1054
1054
|
the studio carries on — one bad pre-step chain is not a reason for it to
|
|
1055
1055
|
die.
|
|
1056
1056
|
|
|
1057
|
+
**Stopping a run from the web.** While a run is replaying, the spinning
|
|
1058
|
+
Run button on the page turns into **Stop**. The studio hears it on its next
|
|
1059
|
+
beat, ends the replay's whole process group (SIGTERM, then SIGKILL), and
|
|
1060
|
+
records the run as failed with "Stopped from the web"; its video and trace
|
|
1061
|
+
are not kept. This uses POSIX process groups (`os.killpg`), so it needs
|
|
1062
|
+
macOS or Linux — the studio does not support Windows.
|
|
1063
|
+
|
|
1057
1064
|
**Branches.** The branch is re-detected on every beat, so switching
|
|
1058
1065
|
branches under a running studio moves what the web shows and what the next
|
|
1059
1066
|
recording lands against. `--branch` pins it instead.
|
|
1060
1067
|
|
|
1061
|
-
**Ctrl-C** does the safe thing at either moment. Mid-recording, the
|
|
1068
|
+
**Ctrl-C** (and SIGTERM, or the terminal closing) does the safe thing at either moment. Mid-recording, the
|
|
1062
1069
|
interrupt is forwarded to the recorder exactly as it is for `test record`:
|
|
1063
1070
|
the browser closes, the spec is saved, the replay runs, the request
|
|
1064
1071
|
completes — and only then does the studio exit. At the idle prompt it
|
|
@@ -1008,11 +1008,18 @@ cloud can stop it. A request that fails is reported with the reason and
|
|
|
1008
1008
|
the studio carries on — one bad pre-step chain is not a reason for it to
|
|
1009
1009
|
die.
|
|
1010
1010
|
|
|
1011
|
+
**Stopping a run from the web.** While a run is replaying, the spinning
|
|
1012
|
+
Run button on the page turns into **Stop**. The studio hears it on its next
|
|
1013
|
+
beat, ends the replay's whole process group (SIGTERM, then SIGKILL), and
|
|
1014
|
+
records the run as failed with "Stopped from the web"; its video and trace
|
|
1015
|
+
are not kept. This uses POSIX process groups (`os.killpg`), so it needs
|
|
1016
|
+
macOS or Linux — the studio does not support Windows.
|
|
1017
|
+
|
|
1011
1018
|
**Branches.** The branch is re-detected on every beat, so switching
|
|
1012
1019
|
branches under a running studio moves what the web shows and what the next
|
|
1013
1020
|
recording lands against. `--branch` pins it instead.
|
|
1014
1021
|
|
|
1015
|
-
**Ctrl-C** does the safe thing at either moment. Mid-recording, the
|
|
1022
|
+
**Ctrl-C** (and SIGTERM, or the terminal closing) does the safe thing at either moment. Mid-recording, the
|
|
1016
1023
|
interrupt is forwarded to the recorder exactly as it is for `test record`:
|
|
1017
1024
|
the browser closes, the spec is saved, the replay runs, the request
|
|
1018
1025
|
completes — and only then does the studio exit. At the idle prompt it
|
|
@@ -722,6 +722,7 @@ def post_recorder_heartbeat(
|
|
|
722
722
|
kinds: list[str] | None = None,
|
|
723
723
|
environment: str | None = None,
|
|
724
724
|
busy: bool = False,
|
|
725
|
+
working: UUID | None = None,
|
|
725
726
|
timeout: float = 30.0,
|
|
726
727
|
) -> dict[str, Any]:
|
|
727
728
|
"""Say a studio is listening, and collect what to record next.
|
|
@@ -769,6 +770,10 @@ def post_recorder_heartbeat(
|
|
|
769
770
|
body["environment"] = environment
|
|
770
771
|
if busy:
|
|
771
772
|
body["busy"] = True
|
|
773
|
+
if working is not None:
|
|
774
|
+
# Which run this busy studio is on, so the service does not stop it
|
|
775
|
+
# as a run whose studio went away. Sent only when named.
|
|
776
|
+
body["working"] = str(working)
|
|
772
777
|
try:
|
|
773
778
|
response = _http().post(
|
|
774
779
|
request_url,
|
|
@@ -820,6 +825,35 @@ def post_recording_request_status(
|
|
|
820
825
|
raise ApiError(response.status_code, _detail(response))
|
|
821
826
|
|
|
822
827
|
|
|
828
|
+
def cancel_queued_runs(
|
|
829
|
+
*,
|
|
830
|
+
api_url: str,
|
|
831
|
+
token: str,
|
|
832
|
+
repo_id: UUID,
|
|
833
|
+
timeout: float = 10.0,
|
|
834
|
+
) -> int:
|
|
835
|
+
"""Take back this user's queued runs in a repository; how many.
|
|
836
|
+
|
|
837
|
+
The web's "Stop queued runs", called by a studio on its way out: a run
|
|
838
|
+
queued for it is one nothing else will pick up, because presence is
|
|
839
|
+
one studio per user per repository. Runs only, never one a studio has
|
|
840
|
+
already taken — the endpoint's own rule. A short timeout, because the
|
|
841
|
+
caller is a process that has been asked to exit.
|
|
842
|
+
"""
|
|
843
|
+
url = f"{api_url.rstrip('/')}/repositories/{repo_id}/recording-requests"
|
|
844
|
+
try:
|
|
845
|
+
response = httpx.delete(
|
|
846
|
+
url,
|
|
847
|
+
headers={"Authorization": f"Bearer {token}"},
|
|
848
|
+
timeout=timeout,
|
|
849
|
+
)
|
|
850
|
+
except httpx.HTTPError as e:
|
|
851
|
+
raise ApiError(0, f"could not reach {url}: {e}") from e
|
|
852
|
+
if response.status_code >= 400:
|
|
853
|
+
raise ApiError(response.status_code, _detail(response))
|
|
854
|
+
return int(response.json().get("cancelled", 0))
|
|
855
|
+
|
|
856
|
+
|
|
823
857
|
def delete_recorded_test(
|
|
824
858
|
*,
|
|
825
859
|
api_url: str,
|
|
@@ -125,6 +125,7 @@ from rich.table import Table
|
|
|
125
125
|
|
|
126
126
|
from verifaied.client import (
|
|
127
127
|
ApiError,
|
|
128
|
+
cancel_queued_runs,
|
|
128
129
|
create_recorded_test,
|
|
129
130
|
delete_recorded_test,
|
|
130
131
|
get_browser_test_settings,
|
|
@@ -296,6 +297,39 @@ STEP_POST_MIN_INTERVAL = 0.5
|
|
|
296
297
|
# the web reaches this machine, so an idle studio still has to call.
|
|
297
298
|
HEARTBEAT_INTERVAL = 2.0
|
|
298
299
|
|
|
300
|
+
# How a run stopped from the web closes. 130 is what a shell reports for a
|
|
301
|
+
# process ended by an interrupt, which is the nearest honest number for "a
|
|
302
|
+
# person ended this", and it is not 1 — so nothing reading the session can
|
|
303
|
+
# mistake a stop for a spec that found a bug, and a retry loop that retries
|
|
304
|
+
# exit 1 never replays a run somebody asked to end.
|
|
305
|
+
STOPPED_EXIT_CODE = 130
|
|
306
|
+
|
|
307
|
+
# The signals `test studio` treats as Ctrl-C: the terminal's own, its
|
|
308
|
+
# hang-up when the window closes, and a plain `kill` or a make teardown.
|
|
309
|
+
# Looked up by name, because this module is imported by every `verifaied`
|
|
310
|
+
# command and Windows has no SIGHUP: naming it outright would make
|
|
311
|
+
# `verifaied upload` die on import in a Windows CI job.
|
|
312
|
+
STUDIO_SIGNALS: tuple[signal.Signals, ...] = tuple(
|
|
313
|
+
signum
|
|
314
|
+
for signum in (
|
|
315
|
+
getattr(signal, name, None) for name in ("SIGINT", "SIGTERM", "SIGHUP")
|
|
316
|
+
)
|
|
317
|
+
if signum is not None
|
|
318
|
+
)
|
|
319
|
+
STOPPED_MESSAGE = "Stopped from the web"
|
|
320
|
+
# How a replay the studio was running closes when the studio itself is
|
|
321
|
+
# interrupted (Ctrl-C, SIGTERM, the terminal closing). The web reads it by
|
|
322
|
+
# these exact words.
|
|
323
|
+
INTERRUPTED_STOP_MESSAGE = "Stopped: the studio was interrupted"
|
|
324
|
+
|
|
325
|
+
# How long a stopped replay gets to exit on SIGTERM before it is killed.
|
|
326
|
+
# Long enough for Playwright to close its browsers, short enough that the
|
|
327
|
+
# page watching "Stopping…" is not left wondering.
|
|
328
|
+
STOP_GRACE_SECONDS = 5.0
|
|
329
|
+
|
|
330
|
+
# How often a stoppable child is checked on while it runs.
|
|
331
|
+
STOP_POLL_SECONDS = 0.2
|
|
332
|
+
|
|
299
333
|
# The faster beat while the studio's assistant is at work. Its steps
|
|
300
334
|
# arrive one tool call at a time, each written after the last one's
|
|
301
335
|
# verdict came back, and the pickup latency of an idle beat would be paid
|
|
@@ -530,15 +564,78 @@ def find_runner_dir(root: Path, explicit: Path | None = None) -> Path:
|
|
|
530
564
|
raise RecordedTestError(str(e)) from e
|
|
531
565
|
|
|
532
566
|
|
|
567
|
+
class RunStop(threading.Event):
|
|
568
|
+
"""A request to stop a child, and whether it was actually carried out.
|
|
569
|
+
|
|
570
|
+
Setting it asks; ``honoured`` says the child was really stopped — its
|
|
571
|
+
group signalled, or it was never started. The two differ exactly when
|
|
572
|
+
a stop lands after the child had already exited on its own, and that
|
|
573
|
+
run's own exit code and evidence are the truth about it: a stop that
|
|
574
|
+
arrived too late must not turn a real pass or fail into "stopped".
|
|
575
|
+
"""
|
|
576
|
+
|
|
577
|
+
def __init__(self) -> None:
|
|
578
|
+
super().__init__()
|
|
579
|
+
self.honoured = False
|
|
580
|
+
|
|
581
|
+
|
|
533
582
|
def run_process(
|
|
534
|
-
command: list[str],
|
|
583
|
+
command: list[str],
|
|
584
|
+
*,
|
|
585
|
+
cwd: Path,
|
|
586
|
+
env: dict[str, str] | None = None,
|
|
587
|
+
stop: RunStop | None = None,
|
|
588
|
+
grace: float = STOP_GRACE_SECONDS,
|
|
589
|
+
poll: float = STOP_POLL_SECONDS,
|
|
535
590
|
) -> int:
|
|
536
591
|
"""Run a child, inheriting stdio, and return its exit code.
|
|
537
592
|
|
|
538
593
|
The single seam every subprocess in this module goes through, so tests
|
|
539
594
|
can record the argv, cwd and env without ever launching ``npx``.
|
|
595
|
+
|
|
596
|
+
``stop`` makes the child stoppable from another thread: it is started
|
|
597
|
+
in a **process group of its own** — ``npx`` is a wrapper around the
|
|
598
|
+
runner, which is a parent of the browsers, and signalling only the
|
|
599
|
+
first of them would leave the rest replaying — and when the event is
|
|
600
|
+
set the whole group gets SIGTERM, then SIGKILL after ``grace``
|
|
601
|
+
seconds. A stop that arrives before the child exists means it is
|
|
602
|
+
never started. Either way ``stop.honoured`` is set; a child that exits
|
|
603
|
+
on its own leaves it unset, however soon after the stop arrives.
|
|
604
|
+
Without ``stop`` it is the plain call it always was.
|
|
540
605
|
"""
|
|
541
|
-
|
|
606
|
+
if stop is None:
|
|
607
|
+
return subprocess.call(command, cwd=str(cwd), env=env)
|
|
608
|
+
if stop.is_set():
|
|
609
|
+
stop.honoured = True
|
|
610
|
+
return STOPPED_EXIT_CODE
|
|
611
|
+
child = subprocess.Popen(command, cwd=str(cwd), env=env, start_new_session=True)
|
|
612
|
+
while True:
|
|
613
|
+
try:
|
|
614
|
+
return child.wait(timeout=poll)
|
|
615
|
+
except subprocess.TimeoutExpired:
|
|
616
|
+
if stop.is_set():
|
|
617
|
+
break
|
|
618
|
+
stop.honoured = _signal_group(child, signal.SIGTERM)
|
|
619
|
+
try:
|
|
620
|
+
return child.wait(timeout=grace)
|
|
621
|
+
except subprocess.TimeoutExpired:
|
|
622
|
+
_signal_group(child, signal.SIGKILL)
|
|
623
|
+
return child.wait()
|
|
624
|
+
|
|
625
|
+
|
|
626
|
+
def _signal_group(child: subprocess.Popen[Any], signum: int) -> bool:
|
|
627
|
+
"""Signal a child's whole process group; whether it was there to signal.
|
|
628
|
+
|
|
629
|
+
``OSError`` rather than only ``ProcessLookupError``: macOS answers
|
|
630
|
+
``killpg`` on a group whose leader has just become a zombie with
|
|
631
|
+
``PermissionError``, and a group that is gone either way has nothing
|
|
632
|
+
left to stop — the child exited on its own, and says so by its code.
|
|
633
|
+
"""
|
|
634
|
+
try:
|
|
635
|
+
os.killpg(child.pid, signum)
|
|
636
|
+
except OSError:
|
|
637
|
+
return False
|
|
638
|
+
return True
|
|
542
639
|
|
|
543
640
|
|
|
544
641
|
ProcessRunner = Callable[..., int]
|
|
@@ -3816,6 +3913,7 @@ def replay_spec(
|
|
|
3816
3913
|
progress_poster: Callable[..., None] = post_run_progress,
|
|
3817
3914
|
replay_override: ReplayConfig | None = None,
|
|
3818
3915
|
retry_follows: bool = False,
|
|
3916
|
+
stop: RunStop | None = None,
|
|
3819
3917
|
) -> int:
|
|
3820
3918
|
"""Run one recorded spec and record everything it produced.
|
|
3821
3919
|
|
|
@@ -3882,6 +3980,15 @@ def replay_spec(
|
|
|
3882
3980
|
that follows is the one the test's results row ends up pointing at,
|
|
3883
3981
|
and uploading evidence nobody is shown would only slow the retry.
|
|
3884
3982
|
|
|
3983
|
+
``stop`` is how a studio ends this replay because somebody asked from
|
|
3984
|
+
the web. Once it is set the runner is killed, and the run closes as
|
|
3985
|
+
:data:`STOPPED_EXIT_CODE` with :data:`STOPPED_MESSAGE` — a failure, and
|
|
3986
|
+
one that says why — with no step pinned and **no video or trace
|
|
3987
|
+
uploaded**: the runner never got to finish writing them, and a
|
|
3988
|
+
half-written video of a run somebody ended is not evidence of
|
|
3989
|
+
anything. Coverage the browser already sent stays, because those lines
|
|
3990
|
+
really did run.
|
|
3991
|
+
|
|
3885
3992
|
Returns the run's exit code — the spec's own, except that a clean run
|
|
3886
3993
|
whose final coverage upload failed returns 2, because coverage nobody
|
|
3887
3994
|
received looks exactly like coverage nobody produced, and so does a
|
|
@@ -4176,10 +4283,31 @@ def replay_spec(
|
|
|
4176
4283
|
console=console,
|
|
4177
4284
|
err_console=err_console,
|
|
4178
4285
|
flush_kwargs=flush_kwargs,
|
|
4286
|
+
stop=stop,
|
|
4179
4287
|
)
|
|
4288
|
+
except KeyboardInterrupt:
|
|
4289
|
+
# The studio's Ctrl-C, SIGTERM or hang-up, with this
|
|
4290
|
+
# replay's group already killed: the session closes
|
|
4291
|
+
# saying so, rather than as a failure with no words.
|
|
4292
|
+
# Only a studio passes a stop; a terminal `test run`
|
|
4293
|
+
# interrupted by its own user says nothing extra.
|
|
4294
|
+
if stop is not None:
|
|
4295
|
+
failure_message = INTERRUPTED_STOP_MESSAGE
|
|
4296
|
+
raise
|
|
4180
4297
|
finally:
|
|
4181
4298
|
config.unlink(missing_ok=True)
|
|
4182
|
-
|
|
4299
|
+
# Whether the child was really stopped, not whether a stop
|
|
4300
|
+
# was asked for: one that lands after the runner exited on
|
|
4301
|
+
# its own leaves that run's verdict and evidence standing.
|
|
4302
|
+
stopped = stop is not None and stop.honoured
|
|
4303
|
+
if stopped:
|
|
4304
|
+
# Whatever the runner exited with, it exited because it
|
|
4305
|
+
# was killed — its report, if it wrote one, describes a
|
|
4306
|
+
# run cut short rather than a step that failed.
|
|
4307
|
+
code = STOPPED_EXIT_CODE
|
|
4308
|
+
failure_message = STOPPED_MESSAGE
|
|
4309
|
+
console.print(f"[yellow]{STOPPED_MESSAGE}[/yellow] — {label}")
|
|
4310
|
+
elif code != 0:
|
|
4183
4311
|
# Only on a failure, and inside the temp directory's
|
|
4184
4312
|
# lifetime — the report is deleted with it, and a passing
|
|
4185
4313
|
# run has neither a step to name nor anything to explain
|
|
@@ -4187,7 +4315,11 @@ def replay_spec(
|
|
|
4187
4315
|
failed_step, failure_message = failure_from_report(
|
|
4188
4316
|
report_file, run_spec.resolve(), line_map, secrets
|
|
4189
4317
|
)
|
|
4190
|
-
if
|
|
4318
|
+
if (
|
|
4319
|
+
session_id is not None
|
|
4320
|
+
and not stopped
|
|
4321
|
+
and not (code == 1 and retry_follows)
|
|
4322
|
+
):
|
|
4191
4323
|
upload_artifacts(
|
|
4192
4324
|
new_artifacts_under(output_dir),
|
|
4193
4325
|
api_url=api_url,
|
|
@@ -4249,6 +4381,7 @@ def _run_playwright(
|
|
|
4249
4381
|
err_console: Console,
|
|
4250
4382
|
flush_kwargs: dict[str, Any],
|
|
4251
4383
|
var_env: dict[str, str] | None = None,
|
|
4384
|
+
stop: RunStop | None = None,
|
|
4252
4385
|
) -> int:
|
|
4253
4386
|
"""Run the spec in a worker thread, flushing coverage while it goes.
|
|
4254
4387
|
|
|
@@ -4262,6 +4395,14 @@ def _run_playwright(
|
|
|
4262
4395
|
answered. Layered over this process's own environment rather than
|
|
4263
4396
|
replacing it: the spec still needs PATH, and the machine's own
|
|
4264
4397
|
variables are what a ``from_env`` was read out of in the first place.
|
|
4398
|
+
|
|
4399
|
+
``stop`` is the web asking for this run to end. The runner is handed an
|
|
4400
|
+
event of **this** function's own rather than ``stop`` itself, because
|
|
4401
|
+
there are two reasons to kill a child that runs in its own process
|
|
4402
|
+
group and only one of them is the web: Ctrl-C in the studio's terminal
|
|
4403
|
+
no longer reaches the child directly, so an interrupt here kills it
|
|
4404
|
+
too before being re-raised. Whether the kill was carried out is copied
|
|
4405
|
+
back onto ``stop.honoured``, which is what the caller asks.
|
|
4265
4406
|
"""
|
|
4266
4407
|
env = {
|
|
4267
4408
|
**os.environ,
|
|
@@ -4278,24 +4419,43 @@ def _run_playwright(
|
|
|
4278
4419
|
str(spec_path),
|
|
4279
4420
|
]
|
|
4280
4421
|
result: dict[str, Any] = {}
|
|
4422
|
+
kill = RunStop()
|
|
4281
4423
|
|
|
4282
4424
|
def target() -> None:
|
|
4283
4425
|
try:
|
|
4284
|
-
|
|
4426
|
+
if stop is None:
|
|
4427
|
+
result["code"] = runner(command, cwd=runner_dir, env=env)
|
|
4428
|
+
else:
|
|
4429
|
+
result["code"] = runner(command, cwd=runner_dir, env=env, stop=kill)
|
|
4285
4430
|
except OSError as e:
|
|
4286
4431
|
result["error"] = e
|
|
4287
4432
|
|
|
4288
4433
|
worker = threading.Thread(target=target)
|
|
4289
4434
|
worker.start()
|
|
4290
|
-
|
|
4291
|
-
worker.
|
|
4292
|
-
|
|
4293
|
-
|
|
4435
|
+
if stop is None:
|
|
4436
|
+
while worker.is_alive():
|
|
4437
|
+
worker.join(timeout=interval)
|
|
4438
|
+
if worker.is_alive():
|
|
4439
|
+
_interval_flush(
|
|
4440
|
+
collector,
|
|
4441
|
+
console=console,
|
|
4442
|
+
err_console=err_console,
|
|
4443
|
+
flush_kwargs=flush_kwargs,
|
|
4444
|
+
)
|
|
4445
|
+
else:
|
|
4446
|
+
_watch_stoppable(
|
|
4447
|
+
worker,
|
|
4448
|
+
stop=stop,
|
|
4449
|
+
kill=kill,
|
|
4450
|
+
interval=interval,
|
|
4451
|
+
flush=lambda: _interval_flush(
|
|
4294
4452
|
collector,
|
|
4295
4453
|
console=console,
|
|
4296
4454
|
err_console=err_console,
|
|
4297
4455
|
flush_kwargs=flush_kwargs,
|
|
4298
|
-
)
|
|
4456
|
+
),
|
|
4457
|
+
)
|
|
4458
|
+
stop.honoured = kill.honoured
|
|
4299
4459
|
if "error" in result:
|
|
4300
4460
|
raise RecordedTestError(
|
|
4301
4461
|
f"could not run `npx` in {runner_dir}: {result['error']}. Node "
|
|
@@ -4304,6 +4464,47 @@ def _run_playwright(
|
|
|
4304
4464
|
return int(result.get("code", 1))
|
|
4305
4465
|
|
|
4306
4466
|
|
|
4467
|
+
def _watch_stoppable(
|
|
4468
|
+
worker: threading.Thread,
|
|
4469
|
+
*,
|
|
4470
|
+
stop: threading.Event,
|
|
4471
|
+
kill: threading.Event,
|
|
4472
|
+
interval: float,
|
|
4473
|
+
flush: Callable[[], None],
|
|
4474
|
+
clock: Callable[[], float] = time.monotonic,
|
|
4475
|
+
) -> None:
|
|
4476
|
+
"""Wait out a stoppable replay, passing a stop on and flushing as it goes.
|
|
4477
|
+
|
|
4478
|
+
Checked every :data:`STOP_POLL_SECONDS` rather than every flush
|
|
4479
|
+
interval, so a stop reaches the child within a beat of the studio
|
|
4480
|
+
hearing it — while coverage still goes up at the interval it always
|
|
4481
|
+
did. Nothing is flushed once the child is being killed: the final
|
|
4482
|
+
flush after it exits is the one that counts.
|
|
4483
|
+
|
|
4484
|
+
Ctrl-C lands here on the studio's main thread, and the child — in a
|
|
4485
|
+
process group of its own — never hears it. So the interrupt kills the
|
|
4486
|
+
child, waits for it to go, and is re-raised: the studio's own handling
|
|
4487
|
+
of an interrupted request is unchanged, and no browser is orphaned.
|
|
4488
|
+
"""
|
|
4489
|
+
last_flush = clock()
|
|
4490
|
+
try:
|
|
4491
|
+
while worker.is_alive():
|
|
4492
|
+
if stop.is_set():
|
|
4493
|
+
kill.set()
|
|
4494
|
+
worker.join(timeout=min(interval, STOP_POLL_SECONDS))
|
|
4495
|
+
if (
|
|
4496
|
+
worker.is_alive()
|
|
4497
|
+
and not kill.is_set()
|
|
4498
|
+
and clock() - last_flush >= interval
|
|
4499
|
+
):
|
|
4500
|
+
flush()
|
|
4501
|
+
last_flush = clock()
|
|
4502
|
+
except KeyboardInterrupt:
|
|
4503
|
+
kill.set()
|
|
4504
|
+
worker.join()
|
|
4505
|
+
raise
|
|
4506
|
+
|
|
4507
|
+
|
|
4307
4508
|
# --- driving the recorder ---------------------------------------------
|
|
4308
4509
|
|
|
4309
4510
|
|
|
@@ -8264,7 +8465,14 @@ def run_test_studio(
|
|
|
8264
8465
|
"[dim]Leave this running. Ctrl-C stops it (a recording in progress "
|
|
8265
8466
|
"is saved first).[/dim]"
|
|
8266
8467
|
)
|
|
8267
|
-
|
|
8468
|
+
# SIGTERM and SIGHUP are Ctrl-C too. A replay runs in a process group of
|
|
8469
|
+
# its own so a web stop can end all of it — which also means the
|
|
8470
|
+
# terminal's hang-up and a `kill -TERM -<studio pgid>` no longer reach
|
|
8471
|
+
# it. Handling them exactly as Ctrl-C is what kills that group on the
|
|
8472
|
+
# way out instead of leaving Playwright and its browsers running.
|
|
8473
|
+
previous = {
|
|
8474
|
+
signum: signal.signal(signum, state.on_sigint) for signum in STUDIO_SIGNALS
|
|
8475
|
+
}
|
|
8268
8476
|
beat_branch = branch
|
|
8269
8477
|
try:
|
|
8270
8478
|
while not state.shutdown:
|
|
@@ -8293,23 +8501,32 @@ def run_test_studio(
|
|
|
8293
8501
|
sleep(HEARTBEAT_INTERVAL)
|
|
8294
8502
|
continue
|
|
8295
8503
|
request_id = UUID(str(request["id"]))
|
|
8296
|
-
|
|
8297
|
-
|
|
8504
|
+
# Set from the keeper's thread when a beat names this request
|
|
8505
|
+
# among the runs somebody asked, from the web, to stop. Only a
|
|
8506
|
+
# run is ever named — the service marks nothing else.
|
|
8507
|
+
stop_asked = RunStop()
|
|
8508
|
+
|
|
8509
|
+
def busy_beat(
|
|
8510
|
+
branch_now: str | None = beat_branch,
|
|
8511
|
+
taken: UUID = request_id,
|
|
8512
|
+
asked: RunStop = stop_asked,
|
|
8513
|
+
) -> None:
|
|
8298
8514
|
# Presence and nothing else, from the keeper's thread. The
|
|
8299
8515
|
# loop's own beat is what takes work; this one says "still
|
|
8300
8516
|
# here" while the loop is busy doing it. A blip is swallowed
|
|
8301
8517
|
# for the reason the loop swallows one — the next beat is
|
|
8302
|
-
# seconds away. The branch
|
|
8303
|
-
#
|
|
8518
|
+
# seconds away. The branch, the request and its stop flag
|
|
8519
|
+
# are bound at definition: they are this request's.
|
|
8304
8520
|
#
|
|
8305
|
-
#
|
|
8306
|
-
#
|
|
8307
|
-
#
|
|
8308
|
-
# a line arriving mid-recording, interleaved with the
|
|
8521
|
+
# Nothing is printed from this thread: the branch it names
|
|
8522
|
+
# is registered server-side like any other beat's, but a
|
|
8523
|
+
# line arriving mid-recording, interleaved with the
|
|
8309
8524
|
# recorder's own output, is worse than one arriving on the
|
|
8310
|
-
# loop's next beat.
|
|
8525
|
+
# loop's next beat. The one thing read off its answer is a
|
|
8526
|
+
# stop — this is the only beat a replaying studio sends, so
|
|
8527
|
+
# it is the only one that can hear it.
|
|
8311
8528
|
try:
|
|
8312
|
-
post_recorder_heartbeat(
|
|
8529
|
+
ack = post_recorder_heartbeat(
|
|
8313
8530
|
api_url=api_url,
|
|
8314
8531
|
token=token,
|
|
8315
8532
|
repo_id=repo_id,
|
|
@@ -8317,9 +8534,14 @@ def run_test_studio(
|
|
|
8317
8534
|
kinds=list(STUDIO_KINDS),
|
|
8318
8535
|
environment=environment,
|
|
8319
8536
|
busy=True,
|
|
8537
|
+
# The run this beat vouches for: a run its studio
|
|
8538
|
+
# stops mentioning is stopped as disconnected.
|
|
8539
|
+
working=taken,
|
|
8320
8540
|
)
|
|
8321
8541
|
except ApiError:
|
|
8322
|
-
|
|
8542
|
+
return
|
|
8543
|
+
if str(taken) in {str(i) for i in ack.get("stop") or []}:
|
|
8544
|
+
asked.set()
|
|
8323
8545
|
|
|
8324
8546
|
try:
|
|
8325
8547
|
# A server that predates runs sends no kind, and every
|
|
@@ -8335,7 +8557,7 @@ def run_test_studio(
|
|
|
8335
8557
|
with keeper(busy_beat):
|
|
8336
8558
|
kind = request.get("kind")
|
|
8337
8559
|
if kind in ("run", "run_headless"):
|
|
8338
|
-
_run_requested(
|
|
8560
|
+
cancelled = _run_requested(
|
|
8339
8561
|
request,
|
|
8340
8562
|
# A single Run is somebody sitting in front of
|
|
8341
8563
|
# this machine watching one flow; a whole folder
|
|
@@ -8358,6 +8580,7 @@ def run_test_studio(
|
|
|
8358
8580
|
env_name=env_name,
|
|
8359
8581
|
runner=runner,
|
|
8360
8582
|
shell_runner=shell_runner,
|
|
8583
|
+
stop=stop_asked,
|
|
8361
8584
|
)
|
|
8362
8585
|
elif kind == "amend":
|
|
8363
8586
|
_amend_requested(
|
|
@@ -8434,7 +8657,8 @@ def run_test_studio(
|
|
|
8434
8657
|
# A discarded recording is neither: nothing was saved, and
|
|
8435
8658
|
# nothing went wrong either, so it gets the ending that
|
|
8436
8659
|
# says exactly that rather than being flattened into one
|
|
8437
|
-
# of the other two
|
|
8660
|
+
# of the other two — and so is a run stopped from the web,
|
|
8661
|
+
# whose verdict ("Stopped from the web") is on its session.
|
|
8438
8662
|
post_recording_request_status(
|
|
8439
8663
|
api_url=api_url,
|
|
8440
8664
|
token=token,
|
|
@@ -8477,7 +8701,23 @@ def run_test_studio(
|
|
|
8477
8701
|
except KeyboardInterrupt:
|
|
8478
8702
|
pass
|
|
8479
8703
|
finally:
|
|
8480
|
-
|
|
8704
|
+
for signum, handler in previous.items():
|
|
8705
|
+
signal.signal(signum, handler)
|
|
8706
|
+
# Before the goodbye: a run queued for this studio has nothing else
|
|
8707
|
+
# coming for it once it leaves, so it is taken back rather than left
|
|
8708
|
+
# reading Queued. Best-effort — nothing here may hold up an exit;
|
|
8709
|
+
# the service also takes them back when the goodbye lands.
|
|
8710
|
+
try:
|
|
8711
|
+
cancelled = cancel_queued_runs(
|
|
8712
|
+
api_url=api_url, token=token, repo_id=repo_id
|
|
8713
|
+
)
|
|
8714
|
+
except ApiError:
|
|
8715
|
+
cancelled = 0
|
|
8716
|
+
if cancelled:
|
|
8717
|
+
console.print(
|
|
8718
|
+
f"[dim]Took back {cancelled} queued run"
|
|
8719
|
+
f"{'' if cancelled == 1 else 's'} nothing is left to run.[/dim]"
|
|
8720
|
+
)
|
|
8481
8721
|
try:
|
|
8482
8722
|
post_recorder_heartbeat(
|
|
8483
8723
|
api_url=api_url,
|
|
@@ -8758,7 +8998,8 @@ def _run_requested(
|
|
|
8758
8998
|
runner: ProcessRunner,
|
|
8759
8999
|
shell_runner: ShellRunner,
|
|
8760
9000
|
env_name: str | None = None,
|
|
8761
|
-
|
|
9001
|
+
stop: RunStop | None = None,
|
|
9002
|
+
) -> bool:
|
|
8762
9003
|
"""Replay the one test a heartbeat handed over.
|
|
8763
9004
|
|
|
8764
9005
|
The same shape as :func:`_record_requested`, and for the same reasons:
|
|
@@ -8781,6 +9022,12 @@ def _run_requested(
|
|
|
8781
9022
|
leaves the rest of this checkout's configuration standing; a request
|
|
8782
9023
|
that names none at all — every run started from a terminal, and
|
|
8783
9024
|
every row queued before this existed — reads as "the repo decides".
|
|
9025
|
+
|
|
9026
|
+
``stop`` is set by the studio's presence thread when a beat says
|
|
9027
|
+
somebody asked, from the web, for this run to end. Returns whether it
|
|
9028
|
+
did, so the loop settles the request ``cancelled`` rather than
|
|
9029
|
+
``done``; a stop heard before the replay began means it never begins,
|
|
9030
|
+
and no session is opened for a run nobody wants any more.
|
|
8784
9031
|
"""
|
|
8785
9032
|
request_id = UUID(str(request["id"]))
|
|
8786
9033
|
post_recording_request_status(
|
|
@@ -8810,6 +9057,10 @@ def _run_requested(
|
|
|
8810
9057
|
"from the queue." + ("" if headed else " [dim](headless)[/dim]")
|
|
8811
9058
|
)
|
|
8812
9059
|
asked = request.get("video")
|
|
9060
|
+
if stop is not None and stop.is_set():
|
|
9061
|
+
console.print(f"[yellow]{STOPPED_MESSAGE}[/yellow] before it started.")
|
|
9062
|
+
stop.honoured = True
|
|
9063
|
+
return True
|
|
8813
9064
|
replay_spec(
|
|
8814
9065
|
api_url=api_url,
|
|
8815
9066
|
token=token,
|
|
@@ -8847,7 +9098,12 @@ def _run_requested(
|
|
|
8847
9098
|
if isinstance(asked, dict)
|
|
8848
9099
|
else None
|
|
8849
9100
|
),
|
|
9101
|
+
stop=stop,
|
|
8850
9102
|
)
|
|
9103
|
+
# Honoured, not merely asked: a stop that arrived after the replay
|
|
9104
|
+
# ended on its own leaves a real verdict on the session, and the
|
|
9105
|
+
# request says ``done`` for it like any finished run.
|
|
9106
|
+
return stop is not None and stop.honoured
|
|
8851
9107
|
|
|
8852
9108
|
|
|
8853
9109
|
def _fail_request(
|
|
@@ -19,6 +19,7 @@ from verifaied.client import (
|
|
|
19
19
|
UploadResult,
|
|
20
20
|
_http,
|
|
21
21
|
_media_type,
|
|
22
|
+
cancel_queued_runs,
|
|
22
23
|
clear_manual_coverage,
|
|
23
24
|
create_recorded_test,
|
|
24
25
|
create_session,
|
|
@@ -1420,6 +1421,29 @@ def test_post_recorder_heartbeat_marks_a_presence_only_beat(httpx_mock: HTTPXMoc
|
|
|
1420
1421
|
}
|
|
1421
1422
|
|
|
1422
1423
|
|
|
1424
|
+
def test_post_recorder_heartbeat_names_the_run_a_busy_beat_is_on(
|
|
1425
|
+
httpx_mock: HTTPXMock,
|
|
1426
|
+
):
|
|
1427
|
+
httpx_mock.add_response(
|
|
1428
|
+
method="POST", status_code=200, json={"request": None, "queue_depth": 0}
|
|
1429
|
+
)
|
|
1430
|
+
working = uuid4()
|
|
1431
|
+
post_recorder_heartbeat(
|
|
1432
|
+
api_url="http://api.test",
|
|
1433
|
+
token="t",
|
|
1434
|
+
repo_id=uuid4(),
|
|
1435
|
+
branch="feature-a",
|
|
1436
|
+
busy=True,
|
|
1437
|
+
working=working,
|
|
1438
|
+
)
|
|
1439
|
+
assert json.loads(httpx_mock.get_request().content) == {
|
|
1440
|
+
"branch": "feature-a",
|
|
1441
|
+
"closing": False,
|
|
1442
|
+
"busy": True,
|
|
1443
|
+
"working": str(working),
|
|
1444
|
+
}
|
|
1445
|
+
|
|
1446
|
+
|
|
1423
1447
|
def test_post_recorder_heartbeat_raises_on_error(httpx_mock: HTTPXMock):
|
|
1424
1448
|
httpx_mock.add_response(method="POST", status_code=404, json={"detail": "gone"})
|
|
1425
1449
|
with pytest.raises(ApiError) as exc:
|
|
@@ -1503,6 +1527,36 @@ def test_post_recording_request_status_wraps_a_transport_failure(
|
|
|
1503
1527
|
assert "could not reach" in exc.value.message
|
|
1504
1528
|
|
|
1505
1529
|
|
|
1530
|
+
def test_cancel_queued_runs_asks_the_stop_queued_runs_endpoint(
|
|
1531
|
+
httpx_mock: HTTPXMock,
|
|
1532
|
+
):
|
|
1533
|
+
repo_id = uuid4()
|
|
1534
|
+
httpx_mock.add_response(
|
|
1535
|
+
method="DELETE",
|
|
1536
|
+
url=f"http://api.test/repositories/{repo_id}/recording-requests",
|
|
1537
|
+
status_code=200,
|
|
1538
|
+
json={"cancelled": 3},
|
|
1539
|
+
)
|
|
1540
|
+
assert (
|
|
1541
|
+
cancel_queued_runs(api_url="http://api.test/", token="t", repo_id=repo_id) == 3
|
|
1542
|
+
)
|
|
1543
|
+
assert httpx_mock.get_request().headers["Authorization"] == "Bearer t"
|
|
1544
|
+
|
|
1545
|
+
|
|
1546
|
+
def test_cancel_queued_runs_raises_on_a_refusal(httpx_mock: HTTPXMock):
|
|
1547
|
+
httpx_mock.add_response(method="DELETE", status_code=404, json={"detail": "gone"})
|
|
1548
|
+
with pytest.raises(ApiError) as exc:
|
|
1549
|
+
cancel_queued_runs(api_url="http://api.test", token="t", repo_id=uuid4())
|
|
1550
|
+
assert (exc.value.status_code, exc.value.message) == (404, "gone")
|
|
1551
|
+
|
|
1552
|
+
|
|
1553
|
+
def test_cancel_queued_runs_raises_when_unreachable(httpx_mock: HTTPXMock):
|
|
1554
|
+
httpx_mock.add_exception(httpx.ConnectError("refused"))
|
|
1555
|
+
with pytest.raises(ApiError) as exc:
|
|
1556
|
+
cancel_queued_runs(api_url="http://api.test", token="t", repo_id=uuid4())
|
|
1557
|
+
assert exc.value.status_code == 0
|
|
1558
|
+
|
|
1559
|
+
|
|
1506
1560
|
def test_delete_recorded_test_removes_the_row(httpx_mock: HTTPXMock):
|
|
1507
1561
|
"""The discard's other half: the machine that did the recording is
|
|
1508
1562
|
what removes the row a first recording left behind."""
|