nookku 0.4.0__py3-none-any.whl

nookku/__init__.py

@@ -0,0 +1,3 @@
+"""Relay a test conversation between a person and a chat agent byte for byte, and audit it."""
+
+__version__ = "0.4.0"

nookku/__main__.py

@@ -0,0 +1,21 @@
+"""The `nookku` command. A hook event takes a short path that does not import the CLI (#214)."""
+
+from __future__ import annotations
+
+import sys
+from collections.abc import Sequence
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    args = sys.argv[1:] if argv is None else list(argv)
+    if args[:1] == ["hook"]:
+        from nookku.hook import main as hook
+
+        return hook(args[1:])
+    from nookku.cli import main as cli
+
+    return cli(args)
+
+
+if __name__ == "__main__":
+    sys.exit(main())

nookku/adapters.py

@@ -0,0 +1,214 @@
+"""Adapters of SPEC.md section 4.1: find the message in a request and the reply in a response."""
+
+from __future__ import annotations
+
+import json
+import re
+from typing import Any, Protocol
+
+
+class AdapterError(ValueError):
+    """The adapter cannot find the message or the reply."""
+
+
+class StreamError(ValueError):
+    """The agent's stream failed: it ended early, sent an error, or sent a malformed chunk."""
+
+
+# One earlier turn of the session: (tester message, agent reply).
+History = list[tuple[str, str]]
+
+
+class Adapter(Protocol):
+    def request(self, message: str, history: History) -> bytes: ...
+    def accepts(self, method: str, path: str) -> bool: ...
+    def message(self, body: bytes) -> str: ...
+    def reply(self, body: bytes) -> str: ...
+    def stream_reply(self, body: bytes) -> str: ...
+
+
+def is_stream(content_type: str | None) -> bool:
+    """True if a Content-Type is text/event-stream (SPEC.md section 4.1)."""
+    return (content_type or "").split(";")[0].strip().lower() == "text/event-stream"
+
+
+def sse_events(body: bytes) -> list[tuple[str, str]]:
+    """The (name, data) of each event of an SSE body, in the order of the body.
+
+    The rules are those of the WHATWG HTML standard: a blank line ends an event, a line that starts
+    with a colon is a comment, and an event without a data line is not an event. The last event
+    does not need a blank line after it. The `openai` adapter and the model API proxies
+    (`model_api.events`) use this one parser.
+    """
+    try:
+        text = body.decode("utf-8").removeprefix("")
+    except UnicodeDecodeError:
+        raise StreamError("the stream is not UTF-8") from None
+    events: list[tuple[str, str]] = []
+    name, data = "", list[str]()
+    for line in [*re.split(r"\r\n|\r|\n", text), ""]:
+        if line == "":
+            if data:
+                events.append((name or "message", "\n".join(data)))
+            name, data = "", []
+        elif not line.startswith(":"):
+            key, _, value = line.partition(":")
+            value = value.removeprefix(" ")
+            if key == "data":
+                data.append(value)
+            elif key == "event":
+                name = value
+    return events
+
+
+def _json(body: bytes, what: str) -> Any:
+    try:
+        return json.loads(body.decode("utf-8"))
+    except (UnicodeDecodeError, json.JSONDecodeError):
+        raise AdapterError(f"the {what} body is not UTF-8 JSON") from None
+
+
+def pick(data: Any, path: str) -> Any:
+    """Follow a dot path such as `choices.0.message.content`."""
+    for key in path.split("."):
+        if isinstance(data, list) and key.isdigit() and int(key) < len(data):
+            data = data[int(key)]
+        elif isinstance(data, dict) and key in data:
+            data = data[key]
+        else:
+            raise AdapterError(f"no value at {path!r}")
+    return data
+
+
+def _text(value: Any, path: str) -> str:
+    if not isinstance(value, str):
+        raise AdapterError(f"the value at {path!r} is not a string")
+    return value
+
+
+class JsonAdapter:
+    """Any POST. The message and the reply are at field paths of JSON bodies."""
+
+    def __init__(self, message_field: str = "text", reply_field: str = "reply") -> None:
+        self.message_field, self.reply_field = message_field, reply_field
+
+    def request(self, message: str, history: History) -> bytes:
+        body: Any = message
+        for key in reversed(self.message_field.split(".")):
+            body = {key: body}
+        return json.dumps(body, ensure_ascii=False).encode()
+
+    def accepts(self, method: str, path: str) -> bool:
+        return method == "POST"
+
+    def message(self, body: bytes) -> str:
+        return _text(pick(_json(body, "request"), self.message_field), self.message_field)
+
+    def reply(self, body: bytes) -> str:
+        return _text(pick(_json(body, "response"), self.reply_field), self.reply_field)
+
+    def stream_reply(self, body: bytes) -> str:
+        raise AdapterError("the json adapter does not read a streamed response")
+
+
+class OpenAIAdapter:
+    """A POST to an OpenAI-compatible `/chat/completions` endpoint. The reply can be a stream."""
+
+    def __init__(self, model: str = "", stream: bool = False) -> None:
+        self.model, self.stream = model, stream
+
+    def request(self, message: str, history: History) -> bytes:
+        messages = []
+        for said, reply in history:
+            messages += [{"role": "user", "content": said}, {"role": "assistant", "content": reply}]
+        messages.append({"role": "user", "content": message})
+        # With the option openai_stream, the request asks for a stream (SPEC.md section 5).
+        body: dict[str, Any] = {"messages": messages, "stream": self.stream}
+        if self.model:
+            body["model"] = self.model
+        return json.dumps(body, ensure_ascii=False).encode()
+
+    def accepts(self, method: str, path: str) -> bool:
+        return method == "POST" and path.split("?")[0].rstrip("/").endswith("/chat/completions")
+
+    def message(self, body: bytes) -> str:
+        messages = pick(_json(body, "request"), "messages")
+        if not isinstance(messages, list):
+            raise AdapterError("'messages' is not a list")
+        users = [m for m in messages if isinstance(m, dict) and m.get("role") == "user"]
+        if not users:
+            raise AdapterError("the request has no message with role 'user'")
+        content = users[-1].get("content")
+        if isinstance(content, list):
+            texts = [p for p in content if isinstance(p, dict) and p.get("type") == "text"]
+            if len(texts) != 1 or len(content) != 1:
+                raise AdapterError("the user content must be a string or exactly one text part")
+            content = texts[0].get("text")
+        return _text(content, "messages[-1].content")
+
+    def reply(self, body: bytes) -> str:
+        path = "choices.0.message.content"
+        return _text(pick(_json(body, "response"), path), path)
+
+    def stream_reply(self, body: bytes) -> str:
+        """Join the `choices[0].delta.content` texts of a stream (SPEC.md section 4.1).
+
+        Raise StreamError if the stream has no `data: [DONE]` event, has an event after it, has an
+        error, or has a malformed chunk. Raise AdapterError if no chunk has a content text.
+        """
+        parts: list[str] = []
+        done = False
+        for name, data in sse_events(body):
+            if done:
+                raise StreamError("the stream has an event after data: [DONE]")
+            if data == "[DONE]":
+                done = True
+                continue
+            if name == "error":
+                raise StreamError(f"the agent sent an error event: {data}")
+            try:
+                chunk = json.loads(data)
+            except json.JSONDecodeError:
+                raise StreamError(f"a chunk is not JSON: {data[:80]!r}") from None
+            if not isinstance(chunk, dict):
+                raise StreamError(f"a chunk is not a JSON object: {data[:80]!r}")
+            if chunk.get("error") is not None:
+                raise StreamError(f"the agent sent an error: {json.dumps(chunk['error'])}")
+            choices = chunk.get("choices", [])
+            if not isinstance(choices, list):
+                raise StreamError("the 'choices' of a chunk is not a list")
+            for choice in choices:
+                delta = choice.get("delta", {}) if isinstance(choice, dict) else None
+                if not isinstance(delta, dict):
+                    raise StreamError("a choice of a chunk has no 'delta' object")
+                if choice.get("index", 0) != 0:
+                    continue
+                content = delta.get("content")
+                if content is not None and not isinstance(content, str):
+                    raise StreamError("the 'delta.content' of a chunk is not a string")
+                if content is not None:
+                    parts.append(content)
+        if not done:
+            raise StreamError("the stream ended before data: [DONE]")
+        if not parts:
+            raise AdapterError("no chunk of the stream has a 'delta.content' text")
+        # A chunk can end in the middle of a surrogate pair. Join each pair into one character, as
+        # a JavaScript string does. A lone surrogate is not a Unicode scalar value (SPEC.md 2).
+        try:
+            return "".join(parts).encode("utf-16-le", "surrogatepass").decode("utf-16-le")
+        except UnicodeDecodeError:
+            raise StreamError("the stream reply has a lone surrogate") from None
+
+
+def make(
+    name: str,
+    message_field: str = "text",
+    reply_field: str = "reply",
+    model: str = "",
+    stream: bool = False,
+) -> Adapter:
+    if name == "json":
+        return JsonAdapter(message_field, reply_field)
+    if name == "openai":
+        return OpenAIAdapter(model, stream)
+    raise ValueError(f"unknown adapter {name!r}")

nookku/agent.py

@@ -0,0 +1,51 @@
+"""A helper for an entry in Python: speak the agent contract (SPEC.md section 6) for a function.
+
+from nookku.agent import serve
+
+def reply(message: str, history: list[tuple[str, str]]) -> str:
+    return shop.answer(message)
+
+serve(reply)
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import sys
+from collections.abc import Callable
+
+from nookku import contract
+from nookku.adapters import History
+
+
+def _line(rid: str, **fields: str) -> bytes:
+    return json.dumps({"v": contract.VERSION, "id": rid, **fields}, ensure_ascii=False).encode()
+
+
+def serve(reply: Callable[[str, History], str]) -> None:
+    """Answer each input line on stdin with one output line, until stdin closes.
+
+    Stdout carries only contract lines. All other output of the process, from print() or from a
+    child process, goes to stderr. This includes text that the process printed before serve() and
+    that is still in the stdout buffer.
+    """
+    out = os.fdopen(os.dup(1), "wb", buffering=0)
+    os.dup2(2, 1)
+    # Flush after dup2, so that the buffered text goes to stderr and not to the contract channel.
+    sys.stdout.flush()
+    for raw in sys.stdin.buffer:
+        line = raw.removesuffix(b"\n").removesuffix(b"\r")
+        try:
+            rid, message, history = contract.parse_request(line)
+        except contract.ContractError as e:
+            out.write(_line("", error=f"nookku agent: {e}") + b"\n")
+            continue
+        try:
+            text = reply(message, history)
+            if not isinstance(text, str):
+                raise TypeError(f"reply() returned {type(text).__name__}, not str")
+            result = _line(rid, reply=text)
+        except Exception as e:  # the error goes back to the tester as an error turn
+            result = _line(rid, error=f"{type(e).__name__}: {e}")
+        out.write(result + b"\n")

nookku/audit.py

@@ -0,0 +1,279 @@
+"""The audit of SPEC.md section 3: compare the relay record with the tap record."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from itertools import pairwise
+from pathlib import Path
+from typing import Any
+
+from nookku.record import (
+    BlockedCall,
+    Exchange,
+    ModelSession,
+    RecordError,
+    Turn,
+    Unparsed,
+    read_relay,
+    read_tap,
+)
+
+CHECKS = [
+    "altered_input",
+    "injected_input",
+    "duplicate_send",
+    "out_of_order",
+    "not_delivered",
+    "altered_reply",
+    "unshown_reply",
+    "script_mismatch",
+]
+EXCERPT = 24
+
+
+@dataclass
+class Finding:
+    kind: str
+    relay_line: int | None
+    tap_line: int | None
+    evidence: dict[str, Any] = field(default_factory=dict)
+
+    def as_dict(self) -> dict[str, Any]:
+        return {
+            "class": self.kind,
+            "relay_line": self.relay_line,
+            "tap_line": self.tap_line,
+            "evidence": self.evidence,
+        }
+
+
+@dataclass
+class Report:
+    exit: int
+    turns: int = 0
+    exchanges: int = 0
+    blocked_calls: int = 0
+    model_sessions: int = 0
+    breaks: list[Finding] = field(default_factory=list)
+    notes: list[Finding] = field(default_factory=list)
+    errors: list[dict[str, str]] = field(default_factory=list)
+    skipped: list[str] = field(default_factory=list)
+
+    def as_dict(self) -> dict[str, Any]:
+        return {
+            "v": "0.2",
+            "exit": self.exit,
+            "turns": self.turns,
+            "exchanges": self.exchanges,
+            "blocked_calls": self.blocked_calls,
+            "model_sessions": self.model_sessions,
+            "breaks": [b.as_dict() for b in self.breaks],
+            "notes": [n.as_dict() for n in self.notes],
+            "errors": self.errors,
+            "skipped": self.skipped,
+        }
+
+
+def difference(expected: str, actual: str) -> dict[str, Any]:
+    """Locate the first character where two texts differ, with an excerpt of each side."""
+    i = next(
+        (k for k, (a, b) in enumerate(zip(expected, actual, strict=False)) if a != b),
+        min(len(expected), len(actual)),
+    )
+    start = max(0, i - EXCERPT // 2)
+    return {
+        "first_difference": i,
+        "expected": expected[start : start + EXCERPT],
+        "actual": actual[start : start + EXCERPT],
+        "expected_length": len(expected),
+        "actual_length": len(actual),
+    }
+
+
+def _anchors(turns: list[Turn], exchanges: list[Exchange]) -> list[tuple[int, int]]:
+    """Return index pairs of a longest common subsequence, built in file order (SPEC 3.2 step 1)."""
+    n, m = len(turns), len(exchanges)
+    longest = [[0] * (m + 1) for _ in range(n + 1)]
+    for i in range(n - 1, -1, -1):
+        for j in range(m - 1, -1, -1):
+            if turns[i].said == exchanges[j].input:
+                longest[i][j] = 1 + longest[i + 1][j + 1]
+            else:
+                longest[i][j] = max(longest[i + 1][j], longest[i][j + 1])
+    pairs, i, j = [], 0, 0
+    while i < n and j < m:
+        if turns[i].said == exchanges[j].input and longest[i][j] == 1 + longest[i + 1][j + 1]:
+            pairs.append((i, j))
+            i, j = i + 1, j + 1
+        elif longest[i + 1][j] == longest[i][j]:
+            i += 1
+        else:
+            j += 1
+    return pairs
+
+
+def compare(turns: list[Turn], exchanges: list[Exchange]) -> tuple[list[Finding], list[Finding]]:
+    """Return (breaks, notes) for two valid records."""
+    breaks: list[Finding] = []
+    anchors = _anchors(turns, exchanges)
+    pairs = list(anchors)
+    free_t = [i for i in range(len(turns)) if i not in {a for a, _ in anchors}]
+    free_e = [j for j in range(len(exchanges)) if j not in {b for _, b in anchors}]
+
+    # Step 2: a message that arrived at another position.
+    for i in list(free_t):
+        j = next((j for j in free_e if exchanges[j].input == turns[i].said), None)
+        if j is not None:
+            free_t.remove(i)
+            free_e.remove(j)
+            pairs.append((i, j))
+            breaks.append(Finding("out_of_order", turns[i].line, exchanges[j].line))
+
+    # Step 3: an extra copy of a message that the tester typed.
+    said = {t.said for t in turns}
+    for j in [j for j in free_e if exchanges[j].input in said]:
+        free_e.remove(j)
+        breaks.append(Finding("duplicate_send", None, exchanges[j].line))
+
+    # Step 4: pair what is left in each gap between adjacent anchors as changed input.
+    bounds = [(-1, -1), *anchors, (len(turns), len(exchanges))]
+    for (ti, ej), (tk, ek) in pairwise(bounds):
+        gap_t = [i for i in free_t if ti < i < tk]
+        gap_e = [j for j in free_e if ej < j < ek]
+        for i, j in zip(gap_t, gap_e, strict=False):
+            free_t.remove(i)
+            free_e.remove(j)
+            pairs.append((i, j))
+            breaks.append(
+                Finding(
+                    "altered_input",
+                    turns[i].line,
+                    exchanges[j].line,
+                    difference(turns[i].said, exchanges[j].input),
+                )
+            )
+
+    # Step 5: what is still left.
+    breaks += [Finding("injected_input", None, exchanges[j].line) for j in free_e]
+    breaks += [Finding("not_delivered", turns[i].line, None) for i in free_t]
+
+    # Step 6: the reply of each pair.
+    notes: list[Finding] = []
+    for i, j in pairs:
+        t, e = turns[i], exchanges[j]
+        # A null reply: a status that is not 2xx, no response, or a failed stream.
+        if e.reply is None:
+            notes.append(Finding("agent_error", t.line, e.line, {"status": e.status}))
+        elif t.shown is None:
+            breaks.append(Finding("unshown_reply", t.line, e.line))
+        elif t.shown != e.reply:
+            breaks.append(Finding("altered_reply", t.line, e.line, difference(e.reply, t.shown)))
+
+    def order(f: Finding) -> tuple[int, int, int]:
+        big = 1 << 30
+        return (
+            f.relay_line or big,
+            f.tap_line or big,
+            CHECKS.index(f.kind) if f.kind in CHECKS else 0,
+        )
+
+    return sorted(breaks, key=order), sorted(notes, key=order)
+
+
+def compare_script(turns: list[Turn], messages: list[str]) -> list[Finding]:
+    """The script_mismatch breaks of a scripted test (SPEC.md section 3.2, step 7). Turn n must
+    be a script turn whose said is script message n, and the counts must be equal."""
+    found = []
+    for n in range(max(len(turns), len(messages))):
+        turn = turns[n] if n < len(turns) else None
+        message = messages[n] if n < len(messages) else None
+        evidence: dict[str, Any] = {"script_message": n + 1 if message is not None else None}
+        if turn is None:
+            found.append(Finding("script_mismatch", None, None, evidence))
+        elif message is None or turn.harness != "script":
+            evidence["harness"] = turn.harness
+            found.append(Finding("script_mismatch", turn.line, None, evidence))
+        elif turn.said != message:
+            evidence.update(difference(message, turn.said))
+            found.append(Finding("script_mismatch", turn.line, None, evidence))
+    return found
+
+
+def audit(tap_path: Path, relay_path: Path, script_path: Path | None = None) -> Report:
+    """Compare the records. With a script, also compare the relay record with the script."""
+    from nookku import script
+
+    errors, tap_rows, relay_rows = [], None, None
+    for name, path, reader in (("tap", tap_path, read_tap), ("relay", relay_path, read_relay)):
+        try:
+            rows = reader(path)
+        except RecordError as e:
+            errors.append({"class": e.kind, "record": name, "detail": str(e)})
+            continue
+        if name == "tap":
+            tap_rows = rows
+        else:
+            relay_rows = rows
+    for row in tap_rows or []:
+        if isinstance(row, Unparsed):
+            errors.append(
+                {
+                    "class": "tap_unparsed",
+                    "record": "tap",
+                    "detail": f"line {row.line}: {row.method} {row.path}: {row.error}",
+                }
+            )
+    messages: list[str] = []
+    if script_path is not None:
+        try:
+            messages = script.read(script_path)[0]
+        except script.ScriptError as e:
+            kind = "record_missing" if e.unreadable else "record_invalid"
+            errors.append({"class": kind, "record": "script", "detail": str(e)})
+    if errors:
+        skipped = CHECKS if script_path is not None else CHECKS[:-1]
+        return Report(exit=2, errors=errors, skipped=list(skipped))
+    assert tap_rows is not None and relay_rows is not None
+    turns = [r for r in relay_rows if isinstance(r, Turn)]
+    exchanges = [r for r in tap_rows if isinstance(r, Exchange)]
+    breaks, notes = compare(turns, exchanges)
+    if script_path is not None:
+        breaks += compare_script(turns, messages)
+    return Report(
+        exit=1 if breaks else 0,
+        turns=len(turns),
+        exchanges=len(exchanges),
+        blocked_calls=sum(isinstance(r, BlockedCall) for r in relay_rows),
+        model_sessions=sum(isinstance(r, ModelSession) for r in tap_rows),
+        breaks=breaks,
+        notes=notes,
+    )
+
+
+def render(report: Report) -> str:
+    """A short text report for a terminal."""
+    out = []
+    if report.errors:
+        out += [f"ERROR {e['class']}: {e['detail']}" for e in report.errors]
+        out.append("Skipped checks: " + ", ".join(report.skipped))
+        out.append("Result: the audit cannot run (exit 2)")
+        return "\n".join(out)
+    for b in report.breaks:
+        where = f"relay line {b.relay_line or '-'}, tap line {b.tap_line or '-'}"
+        line = f"BREAK {b.kind:<15} {where}"
+        if "first_difference" in b.evidence:
+            ev = b.evidence
+            line += (
+                f": first difference at character {ev['first_difference']}:"
+                f" expected {ev['expected']!r}, got {ev['actual']!r}"
+            )
+        out.append(line)
+    for n in report.notes:
+        out.append(f"NOTE  {n.kind:<15} relay line {n.relay_line}, tap line {n.tap_line}")
+    out.append(
+        f"{report.turns} turns, {report.exchanges} exchanges, "
+        f"{report.blocked_calls} blocked model calls, {report.model_sessions} model sessions, "
+        f"{len(report.breaks)} breaks"
+    )
+    out.append("Result: " + ("clean (exit 0)" if report.exit == 0 else "breaks found (exit 1)"))
+    return "\n".join(out)