quality-loop 0.2.1__tar.gz → 0.3.0__tar.gz

{quality_loop-0.2.1/quality_loop.egg-info → quality_loop-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: quality-loop
-Version: 0.2.1
+Version: 0.3.0
 Summary: Measure what a product teaches - its skill, tools or manual - with isolated agent runs of real tasks, reported by an operator agent
 Author: craftware
 License-Expression: MIT

{quality_loop-0.2.1 → quality_loop-0.3.0}/quality_loop/__init__.py

@@ -12,4 +12,4 @@ room is handed, putting the user's machine back - is code here, not an
 agent's discretion.
 """
 
-__version__ = "0.2.1"
+__version__ = "0.3.0"

{quality_loop-0.2.1 → quality_loop-0.3.0}/quality_loop/adapter.py

@@ -103,6 +103,22 @@ class Adapter:
         """
         return [(case, "") for case in cases]
 
+    def screen_backend(self) -> dict | None:
+        """Where the declared screens are put up, when not on this desktop.
+
+        ``{"path": file, "settings": {...}}``: a Python file with
+        ``open_page``, ``open_terminal`` and ``close`` as in
+        `quality_loop.screens`, and what its ``configure(settings)`` is given.
+        Asked once per room, after the product is up, so settings may name
+        where it runs this time. None, the default, is this OS's own backend.
+        """
+        return None
+
+    def platform(self) -> str:
+        """The OS the room's app runs on, as ``sys.platform`` names it - what
+        the operator is told. Not this machine's when the product runs in a VM."""
+        return sys.platform
+
     def alive(self) -> bool:
         """Whether the product is still answering, asked on the heartbeat."""
         return True

{quality_loop-0.2.1 → quality_loop-0.3.0}/quality_loop/report.py

@@ -125,7 +125,12 @@ def parse(text: str, room: str = "") -> tuple[list[Problem], list[str]]:
             value = field["value"].strip()
             key = by_label[field["label"].strip().lower()]
             if key == "id":
-                value = value.strip("`")
+                # The id is the first code span, or else the first word. The
+                # first Keyhac run with a known list had operators write
+                # "`dialog-gone` (known)", and the note became part of the id,
+                # so every known problem was counted as new.
+                code = re.match(r"`([^`]+)`", value)
+                value = code[1].strip() if code else (value.split() or [""])[0]
             if key == "kind":
                 value = value.lower()
             setattr(current, key, value)

{quality_loop-0.2.1 → quality_loop-0.3.0}/quality_loop/runner.py

@@ -385,22 +385,24 @@ def _one(adapter, case, variant, here: pathlib.Path, bundle, model, operator_mod
     served = pages.Pages(adapter.fixtures) if adapter.fixtures else contextlib.nullcontext()
     own_product = adapter.product_for(built) or contextlib.nullcontext()
     op = None
+    backend = None
     try:
         with own_product, stand_in, served as site:
+            backend = adapter.screen_backend()
             manifest = screens.write_manifest(
                 private / "manifest.json", adapter.screens,
                 fixtures=adapter.fixtures or adapter.checkout,
                 origin=site.origin if site else "", python=adapter.python,
                 state=directory, evidence=adapter.evidence,
                 apps=drive.manifest_entries(adapter.apps) if live else None,
-                record=evidence / "live")
+                record=evidence / "live", backend=backend)
             op = operator.Operator(
                 room=built, directory=directory, scenario=case.path,
                 checkout=adapter.checkout,
                 evidence_help=adapter.evidence_help, manifest=manifest,
                 python=adapter.python, model=operator_model, known=known)
             say("  operator: phase 1 - the screen")
-            op.phase_one(sys.platform, say=say)
+            op.phase_one(adapter.platform(), say=say)
             _checkpoint()
             status.phase(label, "room")
             say("  room:    running")
@@ -465,7 +467,9 @@ def _one(adapter, case, variant, here: pathlib.Path, bundle, model, operator_mod
     finally:
         if op is not None:
             status.cost(label, op.cost)
-        closed = screens.close_all(directory) if directory.exists() else []
+        closed = (screens.close_all(directory, screens.load_backend(backend)
+                                    if backend else None)
+                  if directory.exists() else [])
         if closed:
             say(f"  closed:  {', '.join(closed)} (left open by the operator)")
         shutil.rmtree(private, ignore_errors=True)

{quality_loop-0.2.1 → quality_loop-0.3.0}/quality_loop/screens/__init__.py

@@ -78,13 +78,16 @@ class Terminal:
 def write_manifest(path: pathlib.Path, screens: dict, *, fixtures: pathlib.Path,
                    origin: str, python: str, state: pathlib.Path,
                    evidence: dict | None = None, apps: dict | None = None,
-                   record: pathlib.Path | None = None) -> pathlib.Path:
+                   record: pathlib.Path | None = None,
+                   backend: dict | None = None) -> pathlib.Path:
     """The declarations, made absolute, for the operator's tools to read.
 
     ``evidence`` is ``{name: {"argv": [...], "help": "..."}}``: the product's
     evidence commands, run by the operator's MCP server and nothing else.
     ``apps`` (from `quality_loop.drive.manifest_entries`) are the apps its
     live check may drive, and ``record`` where those steps are written.
+    ``backend`` is the adapter's own screen backend (`Adapter.screen_backend`),
+    used in place of this OS's.
     """
     entries = {}
     for name, screen in screens.items():
@@ -98,6 +101,8 @@ def write_manifest(path: pathlib.Path, screens: dict, *, fixtures: pathlib.Path,
                 "evidence": evidence or {}}
     if apps:
         manifest.update(apps=apps, record=str(record))
+    if backend:
+        manifest["backend"] = backend
     path.write_text(json.dumps(manifest, indent=2), encoding="utf-8")
     return path
 
@@ -124,8 +129,33 @@ def _record(manifest: dict, opened: list[dict]) -> None:
     path.write_text(json.dumps(opened, indent=2), encoding="utf-8")
 
 
-def backend():
-    """This OS's implementation."""
+def load_backend(spec: dict):
+    """An adapter's own screen backend, from ``{"path": file, "settings": {...}}``.
+
+    The file is imported by path - it lives in the product's checkout, not in
+    an installed package - and, if it has ``configure(settings)``, that is
+    called before it is used. It answers what this OS's module answers:
+    ``open_page(url, title, patience)``, ``open_terminal(command, title,
+    patience)`` and ``close(kind, handle)``. It is how a product puts its
+    screens where its app runs - in a VM, say - rather than on this desktop.
+    """
+    import importlib.util
+    path = pathlib.Path(spec["path"])
+    module_spec = importlib.util.spec_from_file_location(
+        f"quality_loop_screen_backend_{path.stem}", path)
+    if module_spec is None or module_spec.loader is None:
+        raise ScreenError(f"cannot load the screen backend {path}")
+    module = importlib.util.module_from_spec(module_spec)
+    module_spec.loader.exec_module(module)
+    if hasattr(module, "configure"):
+        module.configure(spec.get("settings", {}))
+    return module
+
+
+def backend(manifest: dict | None = None):
+    """The manifest's own backend if the adapter named one, else this OS's."""
+    if manifest and manifest.get("backend"):
+        return load_backend(manifest["backend"])
     if sys.platform == "darwin":
         from quality_loop.screens import _mac as module
     elif os.name == "nt":
@@ -148,7 +178,7 @@ def open_screen(name: str, patience: float = 15.0, os_backend=None) -> dict:
         raise ScreenError(f"no screen {name!r}; the adapter declares: "
                           + ", ".join(sorted(screens)))
     entry = screens[name]
-    os_backend = os_backend or backend()
+    os_backend = os_backend or backend(manifest)
     if entry["kind"] == "page":
         handle = os_backend.open_page(entry["url"], entry["title"], patience)
     elif entry["kind"] == "terminal":
@@ -174,7 +204,7 @@ def close_screen(screen_id: str, os_backend=None) -> dict:
     if record is None:
         raise ScreenError(f"{screen_id!r} was not opened here; opened: "
                           + (", ".join(r["id"] for r in opened) or "nothing"))
-    (os_backend or backend()).close(record["kind"], record["handle"])
+    (os_backend or backend(manifest)).close(record["kind"], record["handle"])
     _record(manifest, [r for r in opened if r["id"] != screen_id])
     return record
 

{quality_loop-0.2.1 → quality_loop-0.3.0/quality_loop.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: quality-loop
-Version: 0.2.1
+Version: 0.3.0
 Summary: Measure what a product teaches - its skill, tools or manual - with isolated agent runs of real tasks, reported by an operator agent
 Author: craftware
 License-Expression: MIT

{quality_loop-0.2.1 → quality_loop-0.3.0}/tests/test_scenario_report.py

@@ -146,6 +146,14 @@ def test_a_field_continued_as_a_nested_list_is_kept():
     assert "indented paragraph" in problems[0].detail
 
 
+@pytest.mark.parametrize("written", ["`dialog-gone` (known)", "dialog-gone (known)",
+                                     "`dialog-gone`", "dialog-gone"])
+def test_a_note_after_the_id_is_not_part_of_it(written):
+    problems, malformed = report.parse(
+        f"### [medium] Dialog\n- **Id:** {written}\n- **Evidence:** calls\n")
+    assert not malformed and problems[0].id == "dialog-gone"
+
+
 def test_findings_concatenate_and_count_most_severe_first(tmp_path):
     path = report.write_findings(tmp_path, [
         ("11_f1", "F1 off", REPORT),

{quality_loop-0.2.1 → quality_loop-0.3.0}/tests/test_screens.py

@@ -165,3 +165,46 @@ def test_live_a_terminal_opens_with_its_title_and_closes(tmp_path, monkeypatch):
     monkeypatch.setenv(screens.MANIFEST, str(path))
     record = screens.open_screen("hello")
     screens.close_screen(record["id"])
+
+
+BACKEND = '''
+calls = []
+settings = {}
+
+def configure(given):
+    settings.update(given)
+
+def open_page(url, title, patience):
+    calls.append(("page", url, title, settings.get("where")))
+    return {"vm": settings.get("where")}
+
+def open_terminal(command, title, patience):
+    calls.append(("terminal", command, title))
+    return 7
+
+def close(kind, handle):
+    calls.append(("close", kind, handle))
+'''
+
+
+def test_an_adapter_s_own_backend_opens_and_closes_the_screens(tmp_path, monkeypatch):
+    path = tmp_path / "guest_screens.py"
+    path.write_text(BACKEND, encoding="utf-8")
+    spec = {"path": str(path), "settings": {"where": "a-vm"}}
+    manifest = screens.write_manifest(
+        tmp_path / "manifest.json", {"approval-queue": Page("dialog.html", title="Queue")},
+        fixtures=tmp_path, origin="http://127.0.0.1:5000", python="/venv/python",
+        state=tmp_path / "operator", backend=spec)
+    monkeypatch.setenv(screens.MANIFEST, str(manifest))
+    record = screens.open_screen("approval-queue")
+    assert record["handle"] == {"vm": "a-vm"}, "configured before it was used"
+    # The runner's sweep has no manifest: it is handed the same backend.
+    backend = screens.load_backend(spec)
+    assert screens.close_all(tmp_path / "operator", os_backend=backend) == ["approval-queue-1"]
+    assert backend.calls == [("close", "page", {"vm": "a-vm"})]
+
+
+def test_without_a_backend_of_its_own_it_is_this_os_s():
+    if sys.platform not in ("darwin", "win32"):
+        pytest.skip("no screen backend for this OS")
+    assert screens.backend({"screens": {}}) is screens.backend()