quality-loop 0.1.0__py3-none-any.whl

quality_loop/operator.py

@@ -0,0 +1,186 @@
+"""The operator agent: one session per room, in two phases.
+
+Phase 1 puts the scenario's screen up; phase 2, after the room has finished
+and the runner has collected and audited, reports the problems. The same
+session is resumed for phase 2, so the agent that judges a run is the one
+that set it up.
+
+What the operator may do is a short list, and this module is where it is
+enforced:
+
+- `--restricted` with an explicit `--tools`: no settings file of the
+  operator's applies, so their broad permissions cannot widen this one.
+- **No shell.** Claude Code lets read-only commands through whatever the
+  allow list says, and `cat` reads anywhere; the first real operator ran
+  `cd; ls; cat ...` that way. Its only way to act is quality-loop's own MCP
+  server (`quality_loop.mcp`): the declared screens, and the product's
+  declared evidence commands run without a shell.
+- No other MCP server, the product's least of all: the screen is checked
+  with the OS's own window list, never through the product under test.
+- Writes only into the operator's own directory and the room's REPORT.md.
+- Phase 1 must not change the room. Every file in it is hashed before and
+  after, and any difference cancels the room: TASK.md in particular has to
+  reach the room exactly as the runner wrote it.
+"""
+
+from __future__ import annotations
+
+import dataclasses
+import hashlib
+import json
+import pathlib
+import shutil
+import subprocess
+import sys
+import uuid
+
+from quality_loop import session as sessions
+
+INSTRUCTIONS = pathlib.Path(__file__).with_name("OPERATOR.md")
+
+
+class OperatorError(Exception):
+    """A phase that cannot be trusted, and the room it cancels."""
+
+
+def absolute(path: pathlib.Path) -> str:
+    """A path as a Claude Code permission rule spells an absolute one:
+    ``//Users/me/x`` on macOS, ``//c/Users/me/x`` on Windows."""
+    posix = pathlib.Path(path).resolve().as_posix()
+    if len(posix) > 1 and posix[1] == ":":
+        posix = "/" + posix[0].lower() + posix[2:]
+    return "/" + posix
+
+
+def fingerprint(room: pathlib.Path) -> dict[str, str]:
+    """``{relative path: sha256}`` of every file in the room."""
+    found = {}
+    for path in sorted(pathlib.Path(room).rglob("*")):
+        if path.is_file():
+            found[path.relative_to(room).as_posix()] = hashlib.sha256(
+                path.read_bytes()).hexdigest()
+    return found
+
+
+@dataclasses.dataclass
+class Operator:
+    """One room's operator.
+
+    Attributes:
+        room: The room directory.
+        directory: The operator's own, beside the room and outside it.
+        scenario: The scenario file.
+        checkout: The product's repository, readable in both phases.
+        evidence_help: One paragraph on what the evidence tools answer.
+        manifest: The manifest the runner wrote: screens and evidence tools.
+        python: The interpreter quality-loop's MCP server runs under.
+    """
+    room: pathlib.Path
+    directory: pathlib.Path
+    scenario: pathlib.Path
+    checkout: pathlib.Path
+    evidence_help: str
+    manifest: pathlib.Path
+    python: str = sys.executable
+    max_turns: int = 60
+    model: str | None = None
+    claude: str | None = None
+    session_id: str = dataclasses.field(default_factory=lambda: str(uuid.uuid4()))
+
+    def allow(self) -> list[str]:
+        """The permission list: read, quality-loop's tools, two places to write.
+
+        Writes are granted as Edit rules, which govern every built-in tool
+        that changes a file, Write included. Their paths are absolute, which
+        Claude Code spells with a leading ``//``. A single ``/`` is relative to
+        the settings file, and the first real run was refused every write.
+        """
+        return ["Read", "Glob", "Grep", "mcp__quality-loop",
+                f"Edit({absolute(self.directory)}/**)",
+                f"Edit({absolute(self.room / 'REPORT.md')})"]
+
+    def servers(self) -> dict:
+        from quality_loop.screens import MANIFEST
+        return {"quality-loop": {"command": self.python, "args": ["-m", "quality_loop.mcp"],
+                                 "env": {MANIFEST: str(self.manifest)}}}
+
+    def command(self, phase: int, message: str) -> list[str]:
+        command = [self.claude or sessions.find_claude(), "--restricted",
+                   "--tools", "Read,Glob,Grep,Write,Edit",
+                   "--add-dir", str(self.room), "--add-dir", str(self.checkout),
+                   "--add-dir", str(self.manifest.parent),
+                   "--mcp-config", json.dumps({"mcpServers": self.servers()}),
+                   "--strict-mcp-config",
+                   "--settings", json.dumps({"permissions": {"allow": self.allow()}}),
+                   "--permission-mode", "dontAsk",
+                   "--max-turns", str(self.max_turns),
+                   "--output-format", "stream-json", "--verbose"]
+        command += (["--session-id", self.session_id] if phase == 1
+                    else ["--resume", self.session_id])
+        if self.model:
+            command += ["--model", self.model]
+        return command + ["-p", message]
+
+    def environment(self) -> dict:
+        return sessions.Session(allow=[]).environment(self.directory)
+
+    @property
+    def instructions(self) -> pathlib.Path:
+        """OPERATOR.md, copied into the operator's own directory.
+
+        The package's copy is wherever quality-loop is installed, which is
+        outside everything the operator may read. A product's first real run
+        had an operator that never saw its job description, and wrote a
+        verdict.
+        """
+        return self.directory / "OPERATOR.md"
+
+    def phase_one_message(self, platform: str) -> str:
+        return (f"Phase 1. Read {self.instructions} first; it is your whole job description.\n\n"
+                f"Scenario: {self.scenario}\nRoom: {self.room}\n"
+                f"Your directory: {self.directory}\nThis OS: {platform}\n"
+                f"Your tools: screen_list, screen_open, screen_close, evidence.\n")
+
+    def phase_two_message(self) -> str:
+        return (f"Phase 2. The room has finished. Room: {self.room}\n"
+                f"Write {self.room / 'REPORT.md'} in the format {self.instructions} gives - "
+                f"problems with severities, no verdict - then close the screens you opened.\n\n"
+                f"The product's evidence tools are in the `evidence` tool - call each "
+                f"with args [\"--help\"] first.\n{self.evidence_help}\n")
+
+    def _run(self, phase: int, message: str, say) -> int:
+        self.directory.mkdir(parents=True, exist_ok=True)
+        log = self.directory / f"phase{phase}.jsonl"
+        with log.open("w", encoding="utf-8") as out, \
+                subprocess.Popen(self.command(phase, message), cwd=self.directory,
+                                 stdin=subprocess.DEVNULL, stdout=subprocess.PIPE,
+                                 stderr=subprocess.STDOUT, text=True, encoding="utf-8",
+                                 bufsize=1, env=self.environment()) as agent:
+            for line in agent.stdout:
+                out.write(line)
+                try:
+                    event = json.loads(line)
+                except ValueError:
+                    continue
+                if isinstance(event, dict) and (text := sessions.digest(
+                        event, self.directory, f"operator {phase}")):
+                    say(text)
+            return agent.wait()
+
+    def phase_one(self, platform: str, say=print) -> None:
+        """Put the screen up. Raises when the room was touched."""
+        self.directory.mkdir(parents=True, exist_ok=True)
+        shutil.copy2(INSTRUCTIONS, self.instructions)
+        before = fingerprint(self.room)
+        self._run(1, self.phase_one_message(platform), say)
+        after = fingerprint(self.room)
+        if changed := sorted(set(before.items()) ^ set(after.items())):
+            raise OperatorError(
+                "phase 1 changed the room, so it is cancelled: "
+                + ", ".join(sorted({path for path, _hash in changed})))
+
+    def phase_two(self, say=print) -> pathlib.Path:
+        """Write the report. Returns REPORT.md's path, which may not exist if
+        the operator wrote nothing - and that is itself reported."""
+        self._run(2, self.phase_two_message(), say)
+        return self.room / "REPORT.md"

quality_loop/pages.py

@@ -0,0 +1,84 @@
+"""Serving fixture pages, and recording what is done on them.
+
+A task that drives a web page is given one served over a local HTTP origin,
+from a copy outside the checkout. Not ``file://``: the address bar is part of
+the screen, and a path under the repository tells the room where the source it
+may not read is. Safari also refuses a ``file://`` page under the temp
+directory outright. An HTTP origin says nothing but a port number, which is
+what the applications these tasks really drive look like.
+
+A page reports what was done on it - an approval, a submission - by POSTing a
+JSON object to ``event``. That list is the screen's own record of the run, and
+it is what a report cites, rather than the room's account.
+"""
+
+from __future__ import annotations
+
+import functools
+import json
+import pathlib
+import shutil
+import tempfile
+import threading
+from http.server import SimpleHTTPRequestHandler, ThreadingHTTPServer
+
+
+class Pages:
+    """Fixture pages served for the length of a ``with`` block.
+
+    Attributes:
+        origin: ``http://127.0.0.1:<port>``.
+        events: What the pages reported, in order.
+    """
+
+    def __init__(self, fixtures: pathlib.Path):
+        self.fixtures = pathlib.Path(fixtures)
+        self.events: list[dict] = []
+        self.origin = ""
+        self._lock = threading.Lock()
+        self._server = None
+        self._served = None
+
+    def url(self, page: str) -> str:
+        return f"{self.origin}/{page}"
+
+    def __enter__(self) -> "Pages":
+        self._served = pathlib.Path(tempfile.mkdtemp(prefix="quality-loop-pages-"))
+        shutil.copytree(self.fixtures, self._served, dirs_exist_ok=True)
+        pages = self
+
+        class Handler(SimpleHTTPRequestHandler):
+            def log_message(self, *args, **kwargs):
+                pass
+
+            def do_POST(self):
+                length = int(self.headers.get("Content-Length") or 0)
+                try:
+                    event = json.loads(self.rfile.read(length).decode("utf-8"))
+                except ValueError:
+                    event = None
+                if isinstance(event, dict):
+                    with pages._lock:
+                        pages.events.append(event)
+                self.send_response(204)
+                self.end_headers()
+
+        self._server = ThreadingHTTPServer(
+            ("127.0.0.1", 0), functools.partial(Handler, directory=str(self._served)))
+        threading.Thread(target=self._server.serve_forever, daemon=True).start()
+        self.origin = f"http://127.0.0.1:{self._server.server_port}"
+        return self
+
+    def __exit__(self, *exc) -> bool:
+        if self._server is not None:
+            self._server.shutdown()
+            self._server.server_close()
+        if self._served is not None:
+            shutil.rmtree(self._served, ignore_errors=True)
+        return False
+
+    def write(self, path: pathlib.Path) -> None:
+        """The events as JSON lines - the screen's record, for the evidence."""
+        with self._lock:
+            path.write_text("".join(json.dumps(e) + "\n" for e in self.events),
+                            encoding="utf-8")

quality_loop/report.py

@@ -0,0 +1,170 @@
+"""The report: problems, each with a severity, and what a loop needs to act on it.
+
+The operator agent writes ``REPORT.md`` in the room, one ``### `` heading per
+problem:
+
+```markdown
+### [high] F1 binding misses Fn+F1 on macOS
+- **Id:** `fn-f1-not-bound`
+- **Kind:** skill gap
+- **Must:** "With VS Code focused, F1 does nothing ... includes Fn+F1"
+- **Evidence:** `replay --app Code --keys Fn-F1` -> consumed: false
+- **Where it would be fixed:** references/configuration.md, Key expressions
+- **How a fix is confirmed:** re-run the scenario; the same replay reports consumed: true
+
+Free text under the fields, as long as it needs to be.
+```
+
+There is no verdict. A human reads the report and decides what to do with
+it. **Id**, **Where it would be fixed** and **How a fix is confirmed** are
+there so the report can also feed the next step: a later run says of each id
+whether it is fixed, still there, or new, and a fix is checked against the
+same evidence the problem was found with.
+
+`parse` is lenient about everything but what a loop depends on: a problem
+with no severity, an unknown severity or kind, or no id is reported as
+malformed rather than silently dropped.
+"""
+
+from __future__ import annotations
+
+import dataclasses
+import json
+import pathlib
+import re
+
+#: Most severe first.
+SEVERITIES = {
+    "critical": "The run did harm: data destroyed, the operator's files rewritten "
+                "or no longer loading, a destructive control pressed.",
+    "high": "A Must not met, or a result that is silently wrong - one that looks "
+            "right and is not.",
+    "medium": "Met, but only by guessing something the product should have said, "
+              "or after rework a user's agent would repeat.",
+    "low": "Friction that cost little: an empty search, a retry, an unclear message.",
+}
+
+KINDS = {
+    "skill gap": "The product's agent-facing text should have said it.",
+    "product": "The product itself behaves wrongly.",
+    "harness": "This runner, or the room's tools.",
+    "out of scope": "True of the task or the application, not of the product.",
+}
+
+FIELDS = {
+    "id": "Id",
+    "kind": "Kind",
+    "must": "Must",
+    "evidence": "Evidence",
+    "where": "Where it would be fixed",
+    "confirm": "How a fix is confirmed",
+}
+
+_PROBLEM = re.compile(r"^### +\[(?P<severity>[^\]]+)\] +(?P<title>.+?)\s*$")
+_FIELD = re.compile(r"^- +\*\*(?P<label>[^*]+?):\*\* *(?P<value>.*)$")
+
+
+@dataclasses.dataclass
+class Problem:
+    severity: str
+    title: str
+    id: str = ""
+    kind: str = ""
+    must: str = ""
+    evidence: str = ""
+    where: str = ""
+    confirm: str = ""
+    detail: str = ""
+    room: str = ""
+
+    def problems(self) -> list[str]:
+        """What is wrong with this entry as a loop would read it."""
+        found = []
+        if self.severity not in SEVERITIES:
+            found.append(f"severity {self.severity!r} is not one of {list(SEVERITIES)}")
+        if self.kind and self.kind not in KINDS:
+            found.append(f"kind {self.kind!r} is not one of {list(KINDS)}")
+        if not self.id:
+            found.append("no id")
+        if not self.evidence:
+            found.append("no evidence")
+        return found
+
+
+def parse(text: str, room: str = "") -> tuple[list[Problem], list[str]]:
+    """``(problems, malformed)`` from a REPORT.md."""
+    by_label = {label.lower(): key for key, label in FIELDS.items()}
+    problems: list[Problem] = []
+    current: Problem | None = None
+    detail: list[str] = []
+
+    def close():
+        if current is not None:
+            current.detail = "\n".join(detail).strip()
+            problems.append(current)
+
+    for line in text.replace("\r\n", "\n").split("\n"):
+        if heading := _PROBLEM.match(line):
+            close()
+            current = Problem(severity=heading["severity"].strip().lower(),
+                              title=heading["title"], room=room)
+            detail = []
+        elif current is not None and (field := _FIELD.match(line)) \
+                and field["label"].strip().lower() in by_label:
+            value = field["value"].strip()
+            key = by_label[field["label"].strip().lower()]
+            if key == "id":
+                value = value.strip("`")
+            if key == "kind":
+                value = value.lower()
+            setattr(current, key, value)
+        elif current is not None:
+            detail.append(line)
+    close()
+    malformed = [f"{p.title}: {issue}" for p in problems for issue in p.problems()]
+    return problems, malformed
+
+
+def ordered(problems: list[Problem]) -> list[Problem]:
+    """Most severe first, then by room, then by title."""
+    rank = {name: n for n, name in enumerate(SEVERITIES)}
+    return sorted(problems, key=lambda p: (rank.get(p.severity, len(rank)), p.room, p.title))
+
+
+def findings(reports: list[tuple[str, str, str]]) -> tuple[str, list[dict]]:
+    """FINDINGS.md and its machine-readable twin, from every room's report.
+
+    ``reports`` is ``(room name, scenario title, REPORT.md text)`` per room. A
+    concatenation and a count, made mechanically - no second agent rewrites
+    what the operators wrote.
+    """
+    everything, sections = [], []
+    for room, title, text in reports:
+        problems, malformed = parse(text, room)
+        everything += problems
+        counts = ", ".join(f"{sum(p.severity == s for p in problems)} {s}"
+                           for s in SEVERITIES)
+        sections += [f"## {room}", "", f"{title}. {counts}.", ""]
+        sections += [f"- **malformed report** - {m}" for m in malformed]
+        sections += ([text.strip(), ""] if text.strip()
+                     else ["(no report - the operator wrote nothing)", ""])
+    total = ", ".join(f"{sum(p.severity == s for p in everything)} {s}"
+                      for s in SEVERITIES)
+    head = ["# Findings", "",
+            f"{len(reports)} room(s): {total}. No verdict - each problem is the "
+            f"operator's report of what it found, with the evidence it cites.", ""]
+    table = ["| Severity | Room | Problem | Kind |", "|---|---|---|---|"]
+    table += [f"| {p.severity} | {p.room} | {p.title} | {p.kind} |"
+              for p in ordered(everything)]
+    machine = [dataclasses.asdict(p) for p in ordered(everything)]
+    return "\n".join(head + (table + [""] if everything else []) + sections), machine
+
+
+def write_findings(directory: pathlib.Path, reports: list[tuple[str, str, str]]) -> pathlib.Path:
+    """FINDINGS.md and findings.json in ``directory``."""
+    text, machine = findings(reports)
+    (directory / "FINDINGS.md").write_text(text, encoding="utf-8")
+    (directory / "findings.json").write_text(json.dumps(machine, indent=2,
+                                                        ensure_ascii=False),
+                                             encoding="utf-8")
+    return directory / "FINDINGS.md"

quality_loop/room.py

@@ -0,0 +1,105 @@
+"""Building a room: the product's agent-facing surface, the rules, the task.
+
+A room is a directory holding the bundle under test and nothing else the room
+may learn from. It may sit inside the product's checkout - next to the rest
+of the work, where its notes are kept - only because the session is started
+with `--restricted` (see `session`), which stops Claude Code loading the
+checkout's CLAUDE.md and skills. A room anywhere else in the checkout is
+refused: it would look deliberate and be protected by nothing.
+"""
+
+from __future__ import annotations
+
+import pathlib
+import shutil
+import zipfile
+
+#: The room's own CLAUDE.md, which a correctly started session never sees.
+#:
+#: `--restricted` skips CLAUDE.md discovery, so under the runner this file is
+#: dead weight. Under a plain `cd <room> && claude` it is loaded first - along
+#: with the CLAUDE.md of the repository above, which is the contamination it
+#: is here to announce. `audit.audit` looks for the phrase it asks for in the
+#: room's reply.
+CANARY = """\
+# If this arrived on its own, stop
+
+**Did you open this file yourself?** Then nothing is wrong - it is a check
+addressed to a different reader, and you should carry on with TASK.md.
+
+This text is only a problem if it reached you *without* being asked for, as
+part of the context you started with. That would mean the session was started
+without `--restricted`, and Claude Code will have loaded the CLAUDE.md of the
+repository above this directory too. The measurement this room exists to take
+would already be spoilt.
+
+In that case, and only that case: **do not do the task.** Say that the room was
+opened without --restricted and stop, because nothing written from there is
+evidence about what you were given.
+"""
+
+#: What the room is told to write into, already there so that it appends.
+QUESTIONS = """\
+# What the skill did not tell me
+
+One section per thing you had to guess, in the form RULES.md gives. If nothing
+had to be guessed, leave it as it is.
+"""
+
+
+class RoomError(Exception):
+    """A room that cannot be built where it was asked for."""
+
+
+def check_location(room: pathlib.Path, rooms: pathlib.Path,
+                   checkout: pathlib.Path | None) -> None:
+    """Refuse a room inside the checkout but outside its rooms directory."""
+    room, rooms = room.resolve(), rooms.resolve()
+    if checkout is None:
+        return
+    checkout = checkout.resolve()
+    inside = room == checkout or checkout in room.parents
+    if inside and rooms not in room.parents:
+        raise RoomError(
+            f"{room} is inside the checkout but not under {rooms}. A room "
+            f"there is only clean while the session is started with "
+            f"--restricted, and that is worth keeping in one place.")
+
+
+def build(room: pathlib.Path, *, prompt: str, rules: str, bundle: pathlib.Path,
+          skill_name: str, rooms: pathlib.Path,
+          checkout: pathlib.Path | None = None) -> pathlib.Path:
+    """Build a room and return its path.
+
+    Args:
+        room: Where to build it. Must not exist, or be empty.
+        prompt: The task, exactly as the user would type it. Written into
+            TASK.md as given - never a paraphrase, which is how an answer
+            leaks into a room.
+        rules: RULES.md's text: what the room may and may not read.
+        bundle: The surface under test: a zip (as a user downloads it) or a
+            directory. Unpacked under ``skills/<skill_name>``.
+        skill_name: The skill's directory name inside the room.
+        rooms: The directory rooms are built under.
+        checkout: The product's repository, when the rooms are inside it.
+    """
+    room = pathlib.Path(room)
+    check_location(room, rooms, checkout)
+    if room.exists() and any(room.iterdir()):
+        raise RoomError(f"{room} is not empty - a room starts with nothing in it")
+    room.mkdir(parents=True, exist_ok=True)
+
+    target = room / "skills" / skill_name
+    bundle = pathlib.Path(bundle)
+    if bundle.is_dir():
+        shutil.copytree(bundle, target, ignore=shutil.ignore_patterns(
+            "__pycache__", "*.pyc"))
+    else:
+        with zipfile.ZipFile(bundle) as archive:
+            archive.extractall(target)
+
+    (room / "CLAUDE.md").write_text(CANARY, encoding="utf-8")
+    (room / "RULES.md").write_text(rules, encoding="utf-8")
+    (room / "TASK.md").write_text(f"# The task\n\n{prompt.strip()}\n", encoding="utf-8")
+    (room / "QUESTIONS.md").write_text(QUESTIONS, encoding="utf-8")
+    return room