quality-loop 0.1.0__py3-none-any.whl

quality_loop/runner.py

@@ -0,0 +1,160 @@
+"""A run: for each scenario, a room, its operator, and a report.
+
+    build the room (TASK.md is the scenario's Prompt, verbatim)
+    stand fixtures in for the operator's files
+    serve the fixture pages, write the screen manifest
+    operator phase 1 - puts the screen up; the room must come out unchanged
+    the room's session
+    collect evidence, put the operator's files back
+    audit the transcript
+    operator phase 2 - writes REPORT.md
+    close any screen still open
+    ...
+    FINDINGS.md and findings.json, concatenated from the reports
+
+There is no verdict. A room that could not be measured - a cancelled phase 1,
+an isolation breach, a lost product or model - still gets a report, written
+here as a harness problem, so the human reading the findings sees it.
+"""
+
+from __future__ import annotations
+
+import contextlib
+import datetime
+import json
+import pathlib
+import shutil
+import signal
+import sys
+import tempfile
+
+from quality_loop import (audit, operator, pages, report, room, scenario, screens,
+                          transcript)
+
+
+class RunnerError(Exception):
+    """A room the runner will not start."""
+
+
+def refuse_a_reachable_marker(marker: pathlib.Path, built: pathlib.Path,
+                              add_dirs: list[pathlib.Path]) -> None:
+    """The operator's set-aside files must be out of the room's reach.
+
+    The first run of the toy put them inside the one directory the room could
+    write to. The room saw a second style.txt beside the fixture and ignored
+    it. A room that read it would have measured the harness, and one that
+    wrote to it could have touched the operator's own file.
+    """
+    marker = pathlib.Path(marker).resolve()
+    for reach in [built, *add_dirs]:
+        reach = pathlib.Path(reach).resolve()
+        if marker == reach or reach in marker.parents:
+            raise RunnerError(
+                f"the stand-in keeps the operator's files in {marker}, inside {reach}, "
+                f"which the room can reach - put the marker outside it")
+
+
+def _cancelled(case, what: str, problem_id: str, evidence: str) -> str:
+    return (f"# Report: {case.title}\n\nThis room could not be measured.\n\n"
+            f"### [critical] {what}\n- **Id:** `{problem_id}`\n- **Kind:** harness\n"
+            f"- **Evidence:** {evidence}\n- **Where it would be fixed:** the runner\n"
+            f"- **How a fix is confirmed:** the room runs and is reported on\n")
+
+
+def run(adapter, names: list[str] | None = None, model: str | None = None,
+        operator_model: str | None = None, say=print) -> pathlib.Path:
+    """Run every scenario ``names`` picks, and return FINDINGS.md's path."""
+    sys.stdout.reconfigure(line_buffering=True)
+    # SIGTERM unwinds like Ctrl-C, so the stand-in puts the operator's files
+    # back instead of dying with them set aside.
+    signal.signal(signal.SIGTERM, lambda *_: (_ for _ in ()).throw(KeyboardInterrupt()))
+    cases = scenario.load(adapter.scenarios, names)
+    stamp = datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
+    parent = adapter.rooms / f"{adapter.version}-{stamp}"
+    parent.mkdir(parents=True)
+    say(f"rooms:   {parent}")
+    reports: list[tuple[str, str, str]] = []
+    bundle = adapter.bundle()
+    try:
+        with adapter.product():
+            for case, variant in adapter.runs(cases, named=bool(names)):
+                label = case.name + (f".{variant}" if variant else "")
+                say(f"\n== {label}: {case.title}")
+                reports.append((label, case.title, _one(
+                    adapter, case, variant, parent / label, bundle, model,
+                    operator_model, say)))
+    except KeyboardInterrupt:
+        say("\n  interrupted - the room in progress was collected and the "
+            "operator's files put back")
+    findings = report.write_findings(parent, reports)
+    say(f"\n  findings: {findings}")
+    return findings
+
+
+def _one(adapter, case, variant, here: pathlib.Path, bundle, model, operator_model,
+         say) -> str:
+    built = room.build(here, prompt=case.prompt, rules=adapter.rules(), bundle=bundle,
+                       skill_name=adapter.skill_name, rooms=adapter.rooms,
+                       checkout=adapter.checkout)
+    evidence = built / "evidence"
+    evidence.mkdir()
+    private = pathlib.Path(tempfile.mkdtemp(prefix="quality-loop-private-"))
+    directory = here.with_name(here.name + ".operator")
+    session = adapter.session(built)
+    if model:
+        session.model = model
+    stand_in = adapter.stand_in(case, variant, built)
+    if stand_in is not None:
+        refuse_a_reachable_marker(stand_in.marker, built, session.add_dirs)
+    stand_in = stand_in or contextlib.nullcontext()
+    served = pages.Pages(adapter.fixtures) if adapter.fixtures else contextlib.nullcontext()
+    op = None
+    try:
+        with stand_in, served as site:
+            manifest = screens.write_manifest(
+                private / "manifest.json", adapter.screens,
+                fixtures=adapter.fixtures or adapter.checkout,
+                origin=site.origin if site else "", python=adapter.python,
+                state=directory, evidence=adapter.evidence)
+            op = operator.Operator(
+                room=built, directory=directory, scenario=case.path,
+                checkout=adapter.checkout,
+                evidence_help=adapter.evidence_help, manifest=manifest,
+                python=adapter.python, model=operator_model)
+            say("  operator: phase 1 - the screen")
+            op.phase_one(sys.platform, say=say)
+            say("  room:    running")
+            ran = session.run(built, alive=adapter.alive, say=say)
+            if site:
+                site.write(evidence / "screen-events.jsonl")
+        (evidence / "run.json").write_text(json.dumps(
+            {"scenario": case.name, "variant": variant, "platform": sys.platform,
+             "exit_code": ran.exit_code, "product_lost": ran.product_lost,
+             "model_lost": transcript.lost_the_model(ran.transcript)}, indent=2),
+            encoding="utf-8")
+        breaches, refused = audit.audit(built, adapter.policy(built))
+        (evidence / "audit.json").write_text(json.dumps(
+            {"breaches": breaches, "refused": refused}, indent=2), encoding="utf-8")
+        for line in breaches:
+            say(f"  BREACH: {line}")
+        say("  operator: phase 2 - the report")
+        written = op.phase_two(say=say)
+        text = written.read_text(encoding="utf-8") if written.exists() else ""
+        if breaches:
+            text = _cancelled(case, "The room broke isolation; nothing it did is evidence",
+                              "isolation-breach", "; ".join(breaches)) + "\n" + text
+        if ran.product_lost or transcript.lost_the_model(ran.transcript):
+            what = "the product" if ran.product_lost else "the model's API"
+            text = _cancelled(case, f"{what} stopped answering during the run",
+                              "lost-" + ("product" if ran.product_lost else "model"),
+                              "run.json") + "\n" + text
+        return text
+    except operator.OperatorError as error:
+        say(f"  CANCELLED: {error}")
+        return _cancelled(case, "Operator phase 1 changed the room", "phase-one-touched-room",
+                          str(error))
+    finally:
+        closed = screens.close_all(directory) if directory.exists() else []
+        if closed:
+            say(f"  closed:  {', '.join(closed)} (left open by the operator)")
+        shutil.rmtree(private, ignore_errors=True)

quality_loop/scenario.py

@@ -0,0 +1,160 @@
+"""Scenarios: one Markdown file per task, written in natural language.
+
+```markdown
++++
+title = "Stop F1 doing anything in VS Code"
+edits_config = true
++++
+
+## Prompt
+
+What a user would actually type - the only part that reaches the room.
+
+## Screen
+
+What has to be in front before the room starts, for the operator to put up.
+
+## Must
+
+- The observable outcomes, then what to read for.
+
+## Must not
+
+- What a run must not do.
+```
+
+The front matter is TOML between ``+++`` lines. ``title`` is required; every
+other key is the product's to define (which configs to run on, which input
+files to stage, a screen only the operator can supply) and is handed to it
+as-is in ``Scenario.meta``.
+
+The sections are the ``## `` headings. **Prompt** and **Must** are required.
+The prompt is taken verbatim - the text between its heading and the next one -
+because it is the one thing that reaches the room, and the runner, not the
+operator agent, is what copies it there. A Must line that also appears in the
+prompt refuses the scenario: that is the scoring key leaking, and it has
+happened through a copy-paste one line too long.
+"""
+
+from __future__ import annotations
+
+import dataclasses
+import pathlib
+import re
+import tomllib
+
+#: Sections every scenario has.
+REQUIRED = ("Prompt", "Must")
+
+_HEADING = re.compile(r"^## +(.+?)\s*$")
+_FENCE = re.compile(r"^(```|~~~)")
+
+
+class ScenarioError(Exception):
+    """A scenario file that cannot be run as written."""
+
+
+@dataclasses.dataclass
+class Scenario:
+    """One task, as its file states it.
+
+    Attributes:
+        path: The file.
+        name: Its stem: ``11_f1_off_in_vscode``.
+        number: The leading number of the stem, or None.
+        title: From the front matter.
+        meta: The whole front matter, for the product to read.
+        sections: ``{heading: text}``, in file order.
+    """
+    path: pathlib.Path
+    name: str
+    number: int | None
+    title: str
+    meta: dict
+    sections: dict[str, str]
+
+    @property
+    def prompt(self) -> str:
+        """What the room is handed, exactly as written."""
+        return self.sections["Prompt"]
+
+    def section(self, heading: str) -> str:
+        """A section's text, or "" when the scenario has none."""
+        return self.sections.get(heading, "")
+
+
+def parse(path: pathlib.Path) -> Scenario:
+    """Read one scenario file, or say what is wrong with it."""
+    path = pathlib.Path(path)
+    text = path.read_text(encoding="utf-8").replace("\r\n", "\n")
+    meta, body = _front_matter(text, path)
+    if not isinstance(meta.get("title"), str) or not meta["title"].strip():
+        raise ScenarioError(f"{path.name}: the front matter has no title")
+    sections = _sections(body)
+    if missing := [h for h in REQUIRED if not sections.get(h, "").strip()]:
+        raise ScenarioError(f"{path.name}: no {', '.join(missing)} section")
+    _refuse_a_leak(sections, path)
+    number = re.match(r"(\d+)", path.stem)
+    return Scenario(path=path, name=path.stem,
+                    number=int(number.group(1)) if number else None,
+                    title=meta["title"].strip(), meta=meta, sections=sections)
+
+
+def load(directory: pathlib.Path, names: list[str] | None = None) -> list[Scenario]:
+    """Every scenario in ``directory`` in file order, or those ``names`` picks -
+    by number, by stem, or by the stem without its number."""
+    found = [parse(path) for path in sorted(pathlib.Path(directory).glob("*.md"))
+             if not path.name.startswith(("_", "README"))]
+    if not names:
+        return found
+    wanted = []
+    for name in map(str, names):
+        match = [s for s in found
+                 if name in (s.name, s.name.split("_", 1)[-1])
+                 or (name.isdigit() and s.number == int(name))]
+        if not match:
+            raise ScenarioError(f"no scenario {name!r}; have: "
+                                + ", ".join(s.name for s in found))
+        wanted += match
+    return wanted
+
+
+def _front_matter(text: str, path: pathlib.Path) -> tuple[dict, str]:
+    if not text.startswith("+++\n"):
+        raise ScenarioError(f"{path.name}: does not start with +++ front matter")
+    end = text.find("\n+++\n", 4)
+    if end < 0:
+        raise ScenarioError(f"{path.name}: the front matter is never closed with +++")
+    try:
+        meta = tomllib.loads(text[4:end])
+    except tomllib.TOMLDecodeError as error:
+        raise ScenarioError(f"{path.name}: front matter: {error}") from None
+    return meta, text[end + len("\n+++\n"):]
+
+
+def _sections(body: str) -> dict[str, str]:
+    """``## `` headings and the text under each, ignoring fenced code."""
+    sections: dict[str, list[str]] = {}
+    current = None
+    fenced = False
+    for line in body.split("\n"):
+        if _FENCE.match(line):
+            fenced = not fenced
+        heading = None if fenced else _HEADING.match(line)
+        if heading:
+            current = heading.group(1)
+            sections.setdefault(current, [])
+        elif current is not None:
+            sections[current].append(line)
+    return {h: "\n".join(lines).strip("\n") + "\n" for h, lines in sections.items()}
+
+
+def _refuse_a_leak(sections: dict[str, str], path: pathlib.Path) -> None:
+    prompt = " ".join(sections["Prompt"].split()).lower()
+    for heading in ("Must", "Must not"):
+        for line in sections.get(heading, "").splitlines():
+            said = " ".join(line.strip().lstrip("-*").split()).lower()
+            if len(said) >= 20 and said in prompt:
+                raise ScenarioError(
+                    f"{path.name}: the prompt carries a line of {heading}: {line.strip()!r}"
+                    f" - that is the scoring key reaching the room")

quality_loop/screens/__init__.py

@@ -0,0 +1,192 @@
+"""Putting a scenario's screen up, by name, on either OS.
+
+The operator agent reads a scenario's Screen section and decides what has to be
+in front. It can only ask for a screen **by name**, from the list the
+product's adapter declares, and it never passes a path, a URL or a script. An
+agent that can write files and also run a script it names can run anything,
+so the mechanics belong in code:
+
+```python
+SCREENS = {
+    "approval-queue": Page("dialog.html", title="Approval queue"),
+    "deploy":         Terminal("deploy.py", title="deploy - acme"),
+}
+```
+
+The runner writes a manifest of those declarations, with every path made
+absolute, into a directory the operator cannot write to, and points
+``QUALITY_LOOP_MANIFEST`` at it. ``quality-loop screen open <name>`` opens
+one, waits until its window is up, and records it. ``close`` closes only what
+``open`` recorded. Both OSes are implemented here, one module each. The
+window's title is the contract between the declaration and the check that
+the screen is up.
+
+The screen is checked with the OS's own window list, never with the product
+under test: a broken product would otherwise hide a failed setup and take
+the blame for it.
+"""
+
+from __future__ import annotations
+
+import dataclasses
+import json
+import os
+import pathlib
+import shutil
+import sys
+import tempfile
+import time
+
+MANIFEST = "QUALITY_LOOP_MANIFEST"
+
+
+class ScreenError(Exception):
+    """A screen that could not be put up or taken down."""
+
+
+@dataclasses.dataclass(frozen=True)
+class Page:
+    """An HTML page from the product's fixtures, served by the runner.
+
+    Attributes:
+        file: Its path under the served fixture directory.
+        title: Its ``<title>``, which is how its window is recognised.
+    """
+    file: str
+    title: str
+    kind: str = "page"
+
+
+@dataclasses.dataclass(frozen=True)
+class Terminal:
+    """A Python script run in a terminal window of its own.
+
+    Python so that one script serves both OSes; run with the product's
+    interpreter, from a copy outside the checkout, because a terminal shows
+    its command line and a path under the repository tells the room where the
+    source it may not read is.
+
+    Attributes:
+        script: Its path, relative to the adapter's fixture directory.
+        title: The window's title, set by the backend rather than the script.
+    """
+    script: str
+    title: str
+    kind: str = "terminal"
+
+
+def write_manifest(path: pathlib.Path, screens: dict, *, fixtures: pathlib.Path,
+                   origin: str, python: str, state: pathlib.Path,
+                   evidence: dict | None = None) -> pathlib.Path:
+    """The declarations, made absolute, for the operator's tools to read.
+
+    ``evidence`` is ``{name: {"argv": [...], "help": "..."}}``: the product's
+    evidence commands, run by the operator's MCP server and nothing else.
+    """
+    entries = {}
+    for name, screen in screens.items():
+        entry = dataclasses.asdict(screen)
+        if isinstance(screen, Page):
+            entry["url"] = f"{origin}/{screen.file}"
+        else:
+            entry["script"] = str((pathlib.Path(fixtures) / screen.script).resolve())
+        entries[name] = entry
+    path.write_text(json.dumps({"screens": entries, "python": python,
+                                "state": str(state), "evidence": evidence or {}},
+                               indent=2), encoding="utf-8")
+    return path
+
+
+def _manifest() -> dict:
+    where = os.environ.get(MANIFEST)
+    if not where:
+        raise ScreenError(f"{MANIFEST} is not set - the runner sets it for the operator")
+    return json.loads(pathlib.Path(where).read_text(encoding="utf-8"))
+
+
+def _state_file(manifest: dict) -> pathlib.Path:
+    return pathlib.Path(manifest["state"]) / "screens.json"
+
+
+def _opened(manifest: dict) -> list[dict]:
+    path = _state_file(manifest)
+    return json.loads(path.read_text(encoding="utf-8")) if path.exists() else []
+
+
+def _record(manifest: dict, opened: list[dict]) -> None:
+    path = _state_file(manifest)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(json.dumps(opened, indent=2), encoding="utf-8")
+
+
+def backend():
+    """This OS's implementation."""
+    if sys.platform == "darwin":
+        from quality_loop.screens import _mac as module
+    elif os.name == "nt":
+        from quality_loop.screens import _windows as module
+    else:
+        raise ScreenError(f"no screen backend for {sys.platform}")
+    return module
+
+
+def names() -> dict:
+    """``{name: entry}`` - what may be opened."""
+    return _manifest()["screens"]
+
+
+def open_screen(name: str, patience: float = 15.0, os_backend=None) -> dict:
+    """Open the screen ``name`` and wait until its window is up."""
+    manifest = _manifest()
+    screens = manifest["screens"]
+    if name not in screens:
+        raise ScreenError(f"no screen {name!r}; the adapter declares: "
+                          + ", ".join(sorted(screens)))
+    entry = screens[name]
+    os_backend = os_backend or backend()
+    if entry["kind"] == "page":
+        handle = os_backend.open_page(entry["url"], entry["title"], patience)
+    elif entry["kind"] == "terminal":
+        copy = pathlib.Path(tempfile.mkdtemp(prefix="quality-loop-screen-"))
+        script = copy / pathlib.Path(entry["script"]).name
+        shutil.copy2(entry["script"], script)
+        handle = os_backend.open_terminal([manifest["python"], str(script)],
+                                          entry["title"], patience)
+    else:
+        raise ScreenError(f"{name}: unknown kind {entry['kind']!r}")
+    opened = _opened(manifest)
+    record = {"id": f"{name}-{len(opened) + 1}", "name": name, "kind": entry["kind"],
+              "title": entry["title"], "handle": handle, "at": time.time()}
+    _record(manifest, opened + [record])
+    return record
+
+
+def close_screen(screen_id: str, os_backend=None) -> dict:
+    """Close one screen ``open_screen`` recorded, and nothing else."""
+    manifest = _manifest()
+    opened = _opened(manifest)
+    record = next((r for r in opened if r["id"] == screen_id), None)
+    if record is None:
+        raise ScreenError(f"{screen_id!r} was not opened here; opened: "
+                          + (", ".join(r["id"] for r in opened) or "nothing"))
+    (os_backend or backend()).close(record["kind"], record["handle"])
+    _record(manifest, [r for r in opened if r["id"] != screen_id])
+    return record
+
+
+def close_all(state: pathlib.Path, os_backend=None) -> list[str]:
+    """Close everything still recorded in ``state`` - the runner's own sweep,
+    after the operator has had its turn, so a screen it forgot to close does
+    not become the next room's second window."""
+    path = pathlib.Path(state) / "screens.json"
+    if not path.exists():
+        return []
+    closed = []
+    for record in json.loads(path.read_text(encoding="utf-8")):
+        try:
+            (os_backend or backend()).close(record["kind"], record["handle"])
+            closed.append(record["id"])
+        except ScreenError:
+            pass
+    path.unlink()
+    return closed

quality_loop/screens/_mac.py

@@ -0,0 +1,152 @@
+"""Screens on macOS: Safari for pages, Terminal for scripts, through AppleScript.
+
+Every step of the page load was paid for by a run of the harness this
+replaces. `make new document with properties {URL:...}` returns a document
+and does not load it, and a plain two-step sets the URL before the new
+document is ready. So the window is made, given a moment, given its URL, and
+then waited for by the title the page will take. A screen that never arrives
+refuses the run instead of buying it: a room that started without its screen
+once hunted for its task across every application the operator had open.
+"""
+
+from __future__ import annotations
+
+import pathlib
+import subprocess
+import tempfile
+import time
+
+from quality_loop.screens import ScreenError
+
+
+def _quote(text: str) -> str:
+    return '"' + text.replace("\\", "\\\\").replace('"', '\\"') + '"'
+
+
+def _osascript(script: str) -> str:
+    done = subprocess.run(["osascript", "-e", script], capture_output=True,
+                          text=True, check=False)
+    if done.returncode != 0:
+        raise ScreenError(done.stderr.strip() or "osascript failed")
+    return done.stdout.strip()
+
+
+def page_script(url: str) -> str:
+    """The AppleScript that opens ``url`` in a Safari window of its own and
+    answers that window's id."""
+    return f'''tell application "Safari"
+    activate
+    make new document
+    delay 0.5
+    set URL of front document to {_quote(url)}
+    return id of front window
+end tell'''
+
+
+#: The shell a terminal screen runs in: bash, without the operator's startup
+#: files and with a fixed prompt, so the screen reads the same on every machine.
+SHELL = "/bin/bash --noprofile --norc"
+
+
+def wrapper(command: list[str]) -> str:
+    """A bash script that clears the screen, runs ``command``, and then stays in
+    the fixed shell, so the output is still on screen for the room to read."""
+    run = " ".join(_shell_quote(part) for part in command)
+    return ("#!/bin/bash\n"
+            "printf '\\033c'\n"
+            f"{run}\n"
+            "export PS1='$ '\n"
+            # macOS's bash announces, every time, that zsh is the default now.
+            "export BASH_SILENCE_DEPRECATION_WARNING=1\n"
+            f"exec {SHELL} -i\n")
+
+
+def terminal_script(launch: str, title: str) -> str:
+    """The AppleScript that runs ``launch`` in a new Terminal window titled
+    ``title`` and answers that window's id.
+
+    Terminal types its command into the operator's own default shell, which
+    may not be a POSIX shell at all - on one machine it was a shell that knows
+    no ``exec``, and the screen never ran. So what is typed is a single
+    absolute path with no arguments, which any shell runs, and everything
+    else happens in the script it names.
+    """
+    return f'''tell application "Terminal"
+    activate
+    set t to do script {_quote(launch)}
+    set custom title of t to {_quote(title)}
+    set title displays custom title of t to true
+    return id of front window
+end tell'''
+
+
+def _shell_quote(part: str) -> str:
+    return "'" + part.replace("'", "'\\''") + "'"
+
+
+def _wait(application: str, window_id: int, title: str, patience: float) -> None:
+    deadline = time.monotonic() + patience
+    while time.monotonic() < deadline:
+        try:
+            name = _osascript(f'tell application "{application}" to get name of '
+                              f'(first window whose id is {window_id})')
+        except ScreenError:
+            name = ""
+        if title in name:
+            return
+        time.sleep(0.5)
+    raise ScreenError(f"{application} window {window_id} never showed {title!r}. A "
+                      f"sheet left over a window blocks this and shows nowhere else, "
+                      f"and {application} may need Automation permission.")
+
+
+def open_page(url: str, title: str, patience: float) -> int:
+    window = int(_osascript(page_script(url)))
+    _wait("Safari", window, title, patience)
+    return window
+
+
+def open_terminal(command: list[str], title: str, patience: float) -> int:
+    directory = pathlib.Path(tempfile.mkdtemp(prefix="quality-loop-terminal-"))
+    launch = directory / "screen"
+    launch.write_text(wrapper(command), encoding="utf-8")
+    launch.chmod(0o755)
+    window = int(_osascript(terminal_script(str(launch), title)))
+    _wait("Terminal", window, title, patience)
+    return window
+
+
+def _exists(application: str, handle: int) -> bool:
+    return _osascript(f'''if application "{application}" is running then
+    tell application "{application}" to return (exists (first window whose id is {int(handle)}))
+end if
+return false''') == "true"
+
+
+def close(kind: str, handle: int, patience: float = 5.0) -> None:
+    """Close the window ``handle``, and make sure it went.
+
+    A Terminal window whose tab still lists a process answers a close with a
+    "terminate running processes?" sheet and stays, saying nothing - which is
+    how a closed screen used to stay on the desktop. So the processes on that
+    window's own terminal are ended first. They are the script this module
+    started and the shell around it, found by the tab's tty, so nothing of
+    the operator's is touched.
+    """
+    application = "Safari" if kind == "page" else "Terminal"
+    if not _exists(application, handle):
+        return
+    if kind == "terminal":
+        tty = _osascript(f'tell application "Terminal" to get tty of selected tab of '
+                         f'(first window whose id is {int(handle)})')
+        if tty.startswith("/dev/tty"):
+            subprocess.run(["pkill", "-t", tty.removeprefix("/dev/")], check=False)
+            time.sleep(0.5)
+    _osascript(f'tell application "{application}" to close '
+               f'(first window whose id is {int(handle)})')
+    deadline = time.monotonic() + patience
+    while time.monotonic() < deadline:
+        if not _exists(application, handle):
+            return
+        time.sleep(0.25)
+    raise ScreenError(f"{application} window {handle} is still open after closing it")