quality-loop 0.1.0__py3-none-any.whl

quality_loop/transcript.py

@@ -0,0 +1,130 @@
+"""Reading a room's stream-json transcript.
+
+`claude -p --output-format stream-json --verbose` writes one JSON object per
+line. Everything here reads that record rather than the room's account of what
+it did, and everything here is tolerant of it: a line that does not parse, an
+event whose ``message`` is a plain string, a ``content`` that is a string
+rather than a list of parts. A refusal arrives in exactly that shape - XeFM's
+first parser assumed a dict and crashed on every real room.
+"""
+
+from __future__ import annotations
+
+import dataclasses
+import json
+import pathlib
+from typing import Iterator
+
+
+def events(transcript: pathlib.Path) -> Iterator[dict]:
+    """Every event in the transcript that parses as a JSON object."""
+    try:
+        text = pathlib.Path(transcript).read_text(encoding="utf-8")
+    except OSError:
+        return
+    for line in text.splitlines():
+        try:
+            event = json.loads(line)
+        except ValueError:
+            continue
+        if isinstance(event, dict):
+            yield event
+
+
+def parts(event: dict) -> list[dict]:
+    """The content parts of an event's message, or none.
+
+    None when the message is missing, is a string, or carries its content as a
+    string: those events say nothing about a tool call.
+    """
+    message = event.get("message")
+    if not isinstance(message, dict):
+        return []
+    content = message.get("content")
+    if not isinstance(content, list):
+        return []
+    return [part for part in content if isinstance(part, dict)]
+
+
+def result_text(content) -> str:
+    """A tool result's content as text, whichever shape it came in."""
+    if isinstance(content, str):
+        return content
+    if isinstance(content, list):
+        return "\n".join(str(part.get("text", "")) for part in content
+                         if isinstance(part, dict))
+    return str(content or "")
+
+
+@dataclasses.dataclass
+class ToolCall:
+    """One tool call, with what came back."""
+    id: str
+    name: str
+    input: dict
+    result: str = ""
+    is_error: bool = False
+    answered: bool = False
+
+
+def tool_calls(transcript: pathlib.Path) -> list[ToolCall]:
+    """Every tool call in the transcript, in order, paired with its result."""
+    calls: list[ToolCall] = []
+    by_id: dict[str, ToolCall] = {}
+    for event in events(transcript):
+        kind = event.get("type")
+        for part in parts(event):
+            if kind == "assistant" and part.get("type") == "tool_use":
+                given = part.get("input")
+                call = ToolCall(str(part.get("id", "")), str(part.get("name", "")),
+                                given if isinstance(given, dict) else {})
+                calls.append(call)
+                by_id[call.id] = call
+            elif kind == "user" and part.get("type") == "tool_result":
+                call = by_id.get(str(part.get("tool_use_id", "")))
+                if call is not None:
+                    call.result = result_text(part.get("content"))
+                    call.is_error = bool(part.get("is_error"))
+                    call.answered = True
+    return calls
+
+
+def init_event(transcript: pathlib.Path) -> dict | None:
+    """The session's init event, wherever it is.
+
+    Looked for rather than assumed to be first: a rate_limit_event can come
+    ahead of it, and reading line one as the init event once disqualified a
+    clean run.
+    """
+    return next((e for e in events(transcript)
+                 if e.get("type") == "system" and e.get("subtype") == "init"), None)
+
+
+def final_reply(transcript: pathlib.Path) -> str:
+    """The session's last word: the result event's text."""
+    replies = [e.get("result", "") for e in events(transcript)
+               if e.get("type") == "result"]
+    return str(replies[-1]) if replies else ""
+
+
+def result_event(transcript: pathlib.Path) -> dict | None:
+    """The last result event: turns, duration, cost, and whether it errored."""
+    found = [e for e in events(transcript) if e.get("type") == "result"]
+    return found[-1] if found else None
+
+
+def lost_the_model(transcript: pathlib.Path) -> bool:
+    """Whether the session ended because it could not reach the model.
+
+    A machine that sleeps with a room open wakes with no network, and the
+    session's last word is "API Error: Can't reach the API server". Its result
+    event still says `success`, with `is_error` set, and everything the run
+    left behind looks like an ordinary end. A run the model never finished
+    measured the network, not the product.
+    """
+    for event in events(transcript):
+        if event.get("type") == "assistant" and event.get("error"):
+            return True
+        if event.get("type") == "result" and event.get("is_error"):
+            return True
+    return False

quality_loop-0.1.0.dist-info/METADATA

@@ -0,0 +1,74 @@
+Metadata-Version: 2.4
+Name: quality-loop
+Version: 0.1.0
+Summary: Measure what an agent-facing product teaches: isolated agent runs of real tasks, reported by an operator agent
+Author: craftware
+License-Expression: MIT
+Project-URL: Homepage, https://github.com/crftwr/quality-loop
+Project-URL: Repository, https://github.com/crftwr/quality-loop
+Project-URL: Issues, https://github.com/crftwr/quality-loop/issues
+Classifier: Development Status :: 2 - Pre-Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Operating System :: MacOS
+Classifier: Operating System :: Microsoft :: Windows
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Topic :: Software Development :: Quality Assurance
+Classifier: Topic :: Software Development :: Testing
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Provides-Extra: dev
+Requires-Dist: pytest>=8; extra == "dev"
+Requires-Dist: pytest-timeout>=2; extra == "dev"
+Dynamic: license-file
+
+# quality-loop
+
+Measure what an agent-facing product teaches.
+
+A skill bundle, an MCP server or a CLI is only as good as what an agent with
+nothing else can do with it. quality-loop runs that agent on a real task, in a
+**room** that holds the product's agent-facing surface and nothing else. The
+room is started so it cannot read the repository where the answers are.
+Afterwards an **operator** agent reports the problems it finds in what the
+room did, each with a severity. A human decides what to do with the report.
+
+> Status: early development, run end to end against a toy product
+> (`examples/toy/run.py`). In place: rooms and their isolation, the audit,
+> the stand-in, fixture pages, screens by name on macOS and Windows,
+> Markdown scenarios, the operator agent and its MCP tools, and reports with
+> severities. The first real product, Keyhac, is next. The design is
+> [crftwr/keyhac#205](https://github.com/crftwr/keyhac/issues/205).
+
+## What stays code, and why
+
+Anything a run depends on to mean something is code, not something an agent
+decides:
+
+- **The prompt the room is handed** is copied verbatim. The operator knows
+  what a good answer looks like, so any paraphrase of the prompt can leak it.
+- **Isolation.** The room runs with `--restricted`, an explicit permission
+  list under `dontAsk`, one MCP configuration and nothing else, git stopped at
+  the room, and stdin closed. A transcript audit then checks what no flag can
+  enforce.
+- **The operator's machine.** Their files are moved aside and replaced with
+  fixtures. On the way out they are put back in a `finally`, whether the run
+  ended normally, by an exception, by Ctrl-C or by SIGTERM.
+- **Evidence.** A report cites what tools recorded, not what the room said.
+
+## Products using it
+
+- [Keyhac](https://github.com/crftwr/keyhac)
+- [XeFM](https://github.com/crftwr/xefm) (planned)
+
+## Development
+
+```
+make test
+```
+
+MIT licensed.

quality_loop-0.1.0.dist-info/RECORD

@@ -0,0 +1,23 @@
+quality_loop/OPERATOR.md,sha256=SDi3-CnWu-nfnBAXY0h9njxKL6DKe3HZhXvyVPSORd4,4806
+quality_loop/__init__.py,sha256=9aGtGjzNFc4KpykObxIijddHx_evJHkAaW7inNg-WqE,725
+quality_loop/__main__.py,sha256=o95_bhSSku11h5nP6Z9Ga2n_Pe-GQzgDGl-ZO-Vx_mY,1696
+quality_loop/adapter.py,sha256=VHVT4McTAaq1mzJy5D6HqbH9V6xgtkKkuDnTEqCAqBQ,3497
+quality_loop/audit.py,sha256=Z4jHcqTIb3dJJp5rdR1HJET3ZfMycHWRza-0spdlL5Q,6711
+quality_loop/mcp.py,sha256=0n16CIZwerOQJaDNRMZhievoc22vOfuupyogVpWVXMY,6789
+quality_loop/operator.py,sha256=XA3WM0AK7gD-lODkSoOYvrFrNkcE33_VRTU9sLMo60Y,8295
+quality_loop/pages.py,sha256=hQHi3ddVvKi4NkbtDldRnG5G3qWUvNFlxLOcSadO0hc,3119
+quality_loop/report.py,sha256=O3ciNgZZUQKiEBAi5ZlsouK8E2KsIEzymY2oxb5JXKY,6833
+quality_loop/room.py,sha256=xpPuwR0DRK3oYYKNtIG8aVzx1Nn8_MImekbJt5SDZ8M,4485
+quality_loop/runner.py,sha256=YlAZ-F9SftGycGZwlLBuljtUXPyZyxTB0O3VCWLChwc,7532
+quality_loop/scenario.py,sha256=XluMrTMEOys1K46-Wy6swkThh3oMdZaanQLUDksDys0,5653
+quality_loop/session.py,sha256=DucTKTWNQLZSW6R2HNK4oDXZ2HaZyGJKp6eD3qyBqrM,13115
+quality_loop/standin.py,sha256=TPdDlEH6_zj8Jb6IwUnjJTQFz7SmQNrNFB_lmYkn_Kk,8069
+quality_loop/transcript.py,sha256=T9uHTz3Zh9asf_0IPYTwYL4kI0JivAY16rcNs0KxYoU,4758
+quality_loop/screens/__init__.py,sha256=tLnk7-JlOWGy-yHS10KUq8s-CIuJvRwoXh4aFEe2DX0,7031
+quality_loop/screens/_mac.py,sha256=qTTnI9ITIHn7LYWWe6jPeBhHdRCTAEw86SIIKlhWReA,5979
+quality_loop/screens/_windows.py,sha256=FEBHtSOS6NIFJj4qRXsy7SWK87uHuXxI6gdvSaYxFbA,4154
+quality_loop-0.1.0.dist-info/licenses/LICENSE,sha256=zaQF900Kv_gvMjyFe8OCTFqqz_hO8dly1xzeVpIx4is,1066
+quality_loop-0.1.0.dist-info/METADATA,sha256=HFBGzmTQMV68PqzJRQWztrLQwFDJKkeOkzbY9PRhdis,3095
+quality_loop-0.1.0.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
+quality_loop-0.1.0.dist-info/top_level.txt,sha256=yT17UlBVxJo-kb3h1Mt18a4E-4iKnY2Fry31C6jY3Y8,13
+quality_loop-0.1.0.dist-info/RECORD,,

quality_loop-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (84.0.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

quality_loop-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 craftware
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

quality_loop-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+quality_loop