nodloop 0.7.0__tar.gz → 0.9.0__tar.gz

{nodloop-0.7.0 → nodloop-0.9.0}/.gitignore

@@ -17,7 +17,7 @@ __pycache__/
 /.claude/nodloop/
 .claude/settings.local.json
 CLAUDE.local.md
-/tmp
+/tmp/
 
 # environment
 .env*

{nodloop-0.7.0 → nodloop-0.9.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: nodloop
-Version: 0.7.0
+Version: 0.9.0
 Summary: Record AI runs and the verdicts people give on them, and hand the knowledge they approve to the next run in the same place
 Project-URL: Homepage, https://github.com/jeon-jihyeon/nodloop
 Project-URL: Repository, https://github.com/jeon-jihyeon/nodloop
@@ -50,15 +50,41 @@ From then on every `knowledge("support-bot", acme)` carries the item, and a run
 
 | Method | Returns |
 |---|---|
-| `knowledge(producer, labels)` | `Knowledge` with `items` and the `text` to put in the prompt |
-| `record(producer, output, *, labels, applied, subject)` | the run id a verdict cites |
+| `knowledge(producer, labels, holdout_key)` | `Knowledge` with `items`, the `text` to put in the prompt and `withheld`. With a `holdout_key` such as `<service>:<hour>`, the `holdout` share of config.json gets no items and `withheld` instead |
+| `record(producer, output, *, labels, applied, subject, withheld, holdout_key)` | the run id a verdict cites. Pass `knowledge.withheld` as `withheld` and the `holdout_key` the run passed to `knowledge`. `report effect` counts only runs recorded with the key and counts a run without it as `undrawn`. `withheld` takes `Ref` or `Item` values |
 | `judge(run, verdict, reason, reason_code, edited, reviewer)` | nothing |
 | `propose(kind, content, from_run, producer, labels, new_labels, item_id, veto)` | `Candidate` with `id`, `version`, `status` and `overlaps`, the current items of the same kind one run may carry with it |
-| `approve(item_id, version, approver)` | `Approval` with `id`, `version`, `status`, `approver`, `veto`, `compaction_due` when one run would carry more than five items, and `export_error` and `folder_error` when a step after the approval failed |
+| `approve(item_id, version, approver, labels=)` | Approves as proposed, or for the runs carrying `labels` when given, and `{}` for every run of the producer. `Approval` with `id`, `version`, `status`, `approver`, `veto`, `compaction_due` when one run would carry more than five items, and `export_error` and `folder_error` when a step after the approval failed |
 | `check_call(tool, arguments, producer)` | `Decision` with `action` of allow, block or ask, `veto` and `reason` |
+| `outcome(run, result, confirmed_cause, note, reviewer)` | nothing. `result` is a `Result`, confirmed, refuted or inconclusive: what a real check found, apart from the verdict on the output |
+| `report(name)` | `Report` with `report`, the report as JSON, and `since`, the RFC3339 start of the runs it read or empty for every run. `name` is a `ReportName`: `loop`, `extract`, `critic`, `effect`, `health` or `replay` |
 
 A tool that refuses, such as a proposal from a run nobody corrected, raises `ToolError` with the reason.
 
+Every adapter takes an optional `holdout_key`, passes it to `knowledge` and `record` and records what came back as applied or withheld.
+
+## A stream producer with a holdout
+
+A producer that runs on a schedule, such as an anomaly check over each window of a stream, draws the holdout on a key it keeps for the run. Use `<service>:<time block of an hour or more>` so the windows of one block land in the same arm and a lesson seen in one window does not leak into a withheld window next to it.
+
+```python
+from datetime import datetime, timezone
+
+block = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H")  # one hour
+orders = {"service": ["orders"]}
+knowledge = await c.knowledge("lag-check", orders, holdout_key=f"orders:{block}")
+answer = await check(window, knowledge.text)
+run = await c.record(
+    "lag-check", answer, labels=orders, applied=knowledge.items, withheld=knowledge.withheld, holdout_key=f"orders:{block}"
+)
+
+# later, when the cause was checked
+await c.outcome(run, "confirmed", confirmed_cause="partition 7 was hot")
+effect = (await c.report("effect")).report
+```
+
+The share comes from `holdout` in config.json. With `url` the share is the one in the config.json of the nodloop server, read when the server starts.
+
 ## Adapters
 
 LangChain `create_agent` middleware fetches the items once per run and puts them in the system message, records the final answer and refuses a vetoed tool call:
@@ -89,12 +115,14 @@ Claude Agent SDK hooks, the same loop the Claude Code plugin runs:
 ```python
 from nodloop.claude_agent import NodloopHooks
 
-hooks = NodloopHooks(c, "support-bot", acme)
+hooks = NodloopHooks(c, "support-bot", acme, holdout_key=conversation_id)
 options = ClaudeAgentOptions(hooks=hooks.hooks())
 ```
 
 Use one adapter instance per run or conversation, since it keeps the items it handed out and the id of the run it recorded.
 
+The Claude Agent hooks draw once per conversation since the key stays the same for every turn, so a whole conversation is in one arm. `report effect` counts every turn of one key as one draw unit, so the turns of a conversation widen its intervals instead of passing for independent samples.
+
 ## A shared server
 
 `Client(url="https://nodloop.example/mcp", key="nl_...")` connects to `nodloop-server serve` instead of a local binary. The key decides the tenant and what the client may do, and an approval through it is recorded under the key's name.

{nodloop-0.7.0 → nodloop-0.9.0}/README.md

@@ -32,15 +32,41 @@ From then on every `knowledge("support-bot", acme)` carries the item, and a run
 
 | Method | Returns |
 |---|---|
-| `knowledge(producer, labels)` | `Knowledge` with `items` and the `text` to put in the prompt |
-| `record(producer, output, *, labels, applied, subject)` | the run id a verdict cites |
+| `knowledge(producer, labels, holdout_key)` | `Knowledge` with `items`, the `text` to put in the prompt and `withheld`. With a `holdout_key` such as `<service>:<hour>`, the `holdout` share of config.json gets no items and `withheld` instead |
+| `record(producer, output, *, labels, applied, subject, withheld, holdout_key)` | the run id a verdict cites. Pass `knowledge.withheld` as `withheld` and the `holdout_key` the run passed to `knowledge`. `report effect` counts only runs recorded with the key and counts a run without it as `undrawn`. `withheld` takes `Ref` or `Item` values |
 | `judge(run, verdict, reason, reason_code, edited, reviewer)` | nothing |
 | `propose(kind, content, from_run, producer, labels, new_labels, item_id, veto)` | `Candidate` with `id`, `version`, `status` and `overlaps`, the current items of the same kind one run may carry with it |
-| `approve(item_id, version, approver)` | `Approval` with `id`, `version`, `status`, `approver`, `veto`, `compaction_due` when one run would carry more than five items, and `export_error` and `folder_error` when a step after the approval failed |
+| `approve(item_id, version, approver, labels=)` | Approves as proposed, or for the runs carrying `labels` when given, and `{}` for every run of the producer. `Approval` with `id`, `version`, `status`, `approver`, `veto`, `compaction_due` when one run would carry more than five items, and `export_error` and `folder_error` when a step after the approval failed |
 | `check_call(tool, arguments, producer)` | `Decision` with `action` of allow, block or ask, `veto` and `reason` |
+| `outcome(run, result, confirmed_cause, note, reviewer)` | nothing. `result` is a `Result`, confirmed, refuted or inconclusive: what a real check found, apart from the verdict on the output |
+| `report(name)` | `Report` with `report`, the report as JSON, and `since`, the RFC3339 start of the runs it read or empty for every run. `name` is a `ReportName`: `loop`, `extract`, `critic`, `effect`, `health` or `replay` |
 
 A tool that refuses, such as a proposal from a run nobody corrected, raises `ToolError` with the reason.
 
+Every adapter takes an optional `holdout_key`, passes it to `knowledge` and `record` and records what came back as applied or withheld.
+
+## A stream producer with a holdout
+
+A producer that runs on a schedule, such as an anomaly check over each window of a stream, draws the holdout on a key it keeps for the run. Use `<service>:<time block of an hour or more>` so the windows of one block land in the same arm and a lesson seen in one window does not leak into a withheld window next to it.
+
+```python
+from datetime import datetime, timezone
+
+block = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H")  # one hour
+orders = {"service": ["orders"]}
+knowledge = await c.knowledge("lag-check", orders, holdout_key=f"orders:{block}")
+answer = await check(window, knowledge.text)
+run = await c.record(
+    "lag-check", answer, labels=orders, applied=knowledge.items, withheld=knowledge.withheld, holdout_key=f"orders:{block}"
+)
+
+# later, when the cause was checked
+await c.outcome(run, "confirmed", confirmed_cause="partition 7 was hot")
+effect = (await c.report("effect")).report
+```
+
+The share comes from `holdout` in config.json. With `url` the share is the one in the config.json of the nodloop server, read when the server starts.
+
 ## Adapters
 
 LangChain `create_agent` middleware fetches the items once per run and puts them in the system message, records the final answer and refuses a vetoed tool call:
@@ -71,12 +97,14 @@ Claude Agent SDK hooks, the same loop the Claude Code plugin runs:
 ```python
 from nodloop.claude_agent import NodloopHooks
 
-hooks = NodloopHooks(c, "support-bot", acme)
+hooks = NodloopHooks(c, "support-bot", acme, holdout_key=conversation_id)
 options = ClaudeAgentOptions(hooks=hooks.hooks())
 ```
 
 Use one adapter instance per run or conversation, since it keeps the items it handed out and the id of the run it recorded.
 
+The Claude Agent hooks draw once per conversation since the key stays the same for every turn, so a whole conversation is in one arm. `report effect` counts every turn of one key as one draw unit, so the turns of a conversation widen its intervals instead of passing for independent samples.
+
 ## A shared server
 
 `Client(url="https://nodloop.example/mcp", key="nl_...")` connects to `nodloop-server serve` instead of a local binary. The key decides the tenant and what the client may do, and an approval through it is recorded under the key's name.

{nodloop-0.7.0 → nodloop-0.9.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "nodloop"
-version = "0.7.0"
+version = "0.9.0"
 description = "Record AI runs and the verdicts people give on them, and hand the knowledge they approve to the next run in the same place"
 readme = "README.md"
 license = "MIT"

{nodloop-0.7.0 → nodloop-0.9.0}/src/nodloop/__init__.py

@@ -1,5 +1,5 @@
 """Record AI runs and the verdicts people give on them, and hand the knowledge they approve to the next run in the same place"""
 
-from .client import Approval, Candidate, Client, Decision, Item, Knowledge, ToolError
+from .client import Approval, Candidate, Client, Decision, Item, Knowledge, Ref, Report, ReportName, Result, ToolError
 
-__all__ = ["Approval", "Candidate", "Client", "Decision", "Item", "Knowledge", "ToolError"]
+__all__ = ["Approval", "Candidate", "Client", "Decision", "Item", "Knowledge", "Ref", "Report", "ReportName", "Result", "ToolError"]

{nodloop-0.7.0 → nodloop-0.9.0}/src/nodloop/claude_agent.py

@@ -11,7 +11,7 @@ from typing import Any
 
 from claude_agent_sdk import HookMatcher
 
-from .client import Client, Item
+from .client import Client, Item, Ref
 
 Labels = dict[str, list[str]]
 
@@ -22,11 +22,16 @@ class NodloopHooks:
     Use one instance per conversation because it keeps the items of the turn and the id of the last run
     """
 
-    def __init__(self, client: Client, producer: str, labels: Labels | None = None) -> None:
+    def __init__(self, client: Client, producer: str, labels: Labels | None = None, holdout_key: str = "") -> None:
         self.client = client
         self.producer = producer
         self.labels = labels or {}
+        # Passed to knowledge so a share of conversations gets no item and report effect compares them
+        # The key stays the same for every turn so a whole conversation falls in one arm
+        # report effect counts the turns of one key as one draw unit so they weigh as a cluster and not as independent runs
+        self.holdout_key = holdout_key
         self.applied: list[Item] = []
+        self.withheld: list[Ref] = []
         # The run the last answer was recorded as so a verdict can cite it
         self.run: str = ""
 
@@ -38,8 +43,8 @@ class NodloopHooks:
         }
 
     async def prompt(self, input_data: dict[str, Any], tool_use_id: str | None, context: Any) -> dict[str, Any]:
-        knowledge = await self.client.knowledge(self.producer, self.labels)
-        self.applied = knowledge.items
+        knowledge = await self.client.knowledge(self.producer, self.labels, self.holdout_key)
+        self.applied, self.withheld = knowledge.items, knowledge.withheld
         if not knowledge.text:
             return {}
         return {"hookSpecificOutput": {"hookEventName": "UserPromptSubmit", "additionalContext": knowledge.text}}
@@ -49,7 +54,9 @@ class NodloopHooks:
         # A turn without one records nothing
         answer = input_data.get("last_assistant_message") or ""
         if answer.strip():
-            self.run = await self.client.record(self.producer, answer, labels=self.labels, applied=self.applied)
+            self.run = await self.client.record(
+                self.producer, answer, labels=self.labels, applied=self.applied, withheld=self.withheld, holdout_key=self.holdout_key
+            )
         return {}
 
     async def pre_tool_use(self, input_data: dict[str, Any], tool_use_id: str | None, context: Any) -> dict[str, Any]:

{nodloop-0.7.0 → nodloop-0.9.0}/src/nodloop/client.py

@@ -6,7 +6,7 @@ import json
 import os
 from contextlib import AsyncExitStack
 from dataclasses import dataclass, field
-from typing import Any
+from typing import Any, Literal
 
 from mcp import ClientSession, StdioServerParameters
 from mcp.client.stdio import stdio_client
@@ -14,16 +14,31 @@ from mcp.client.streamable_http import create_mcp_http_client, streamable_http_c
 
 from . import binary
 
-__all__ = ["Approval", "Candidate", "Client", "Decision", "Item", "Knowledge", "ToolError"]
+__all__ = ["Approval", "Candidate", "Client", "Decision", "Item", "Knowledge", "Ref", "Report", "ReportName", "Result", "ToolError"]
 
 # The binary release the client fetches when no nodloop is installed
-VERSION = "0.7.0"
+VERSION = "0.9.0"
+
+
+# The reports report reads
+ReportName = Literal["loop", "extract", "critic", "effect", "health", "replay"]
+
+# What a real check found for a run
+Result = Literal["confirmed", "refuted", "inconclusive"]
 
 
 class ToolError(RuntimeError):
     """A tool answered an error such as a refused proposal"""
 
 
+@dataclass(frozen=True)
+class Ref:
+    """One knowledge version as a run names it"""
+
+    id: str
+    version: int
+
+
 @dataclass(frozen=True)
 class Item:
     """One approved knowledge version a run receives"""
@@ -89,6 +104,27 @@ class Knowledge:
     items: list[Item]
     # Empty without items so a prompt gains nothing
     text: str
+    # The id and version of each item the holdout kept from the run
+    # Empty unless the holdout drew the key
+    # Pass them to record as withheld so the run counts in the withheld arm of report effect
+    withheld: list[Ref] = field(default_factory=list)
+
+
+@dataclass(frozen=True)
+class Report:
+    """One report of the records with the start of the runs it read"""
+
+    # The report as JSON
+    report: Any
+    # The RFC3339 start of the window and empty when the report read every run
+    since: str = ""
+
+
+def _ref(v: Item | Ref | dict[str, Any]) -> dict[str, Any]:
+    """The id and version the run tool takes from an item or a ref"""
+    if isinstance(v, dict):
+        return {"id": v["id"], "version": v["version"]}
+    return {"id": v.id, "version": v.version}
 
 
 class Client:
@@ -156,11 +192,20 @@ class Client:
             return structured
         return json.loads(text) if text else None
 
-    async def knowledge(self, producer: str, labels: dict[str, list[str]] | None = None) -> Knowledge:
-        """The approved items a run of the producer with the labels receives and the text to put in its prompt"""
-        answer = await self.call("knowledge_for", {"producer": producer, "labels": labels or {}})
+    async def knowledge(self, producer: str, labels: dict[str, list[str]] | None = None, holdout_key: str = "") -> Knowledge:
+        """The approved items a run of the producer with the labels receives and the text to put in its prompt
+
+        holdout_key is a string kept for one run such as a service and an hour block and the holdout share of config.json draws on it
+        A drawn run gets no item and withheld instead
+        Without a key nothing is withheld
+        """
+        args: dict[str, Any] = {"producer": producer, "labels": labels or {}}
+        if holdout_key:
+            args["holdout_key"] = holdout_key
+        answer = await self.call("knowledge_for", args)
         items = [Item(i["id"], i["version"], i["kind"], i["content"], i.get("scope", {})) for i in answer.get("items") or []]
-        return Knowledge(items, answer.get("context", ""))
+        withheld = [Ref(r["id"], r["version"]) for r in answer.get("withheld") or []]
+        return Knowledge(items, answer.get("context", ""), withheld)
 
     async def record(
         self,
@@ -170,13 +215,25 @@ class Client:
         labels: dict[str, list[str]] | None = None,
         applied: list[Item] | None = None,
         subject: str = "",
+        withheld: list[Item | Ref] | None = None,
+        holdout_key: str = "",
     ) -> str:
-        """Records one output and returns the run id a verdict cites"""
+        """Records one output and returns the run id a verdict cites
+
+        withheld takes the refs knowledge answered for a run the holdout drew or the items themselves
+        Never together with applied
+        holdout_key is the key the run passed to knowledge
+        report effect counts a run without it only as undrawn
+        """
         args: dict[str, Any] = {"producer": producer, "output": output, "labels": labels or {}}
         if applied:
-            args["applied"] = [i.ref for i in applied]
+            args["applied"] = [_ref(i) for i in applied]
+        if withheld:
+            args["withheld"] = [_ref(r) for r in withheld]
         if subject:
             args["subject"] = subject
+        if holdout_key:
+            args["holdout_key"] = holdout_key
         return (await self.call("run", args))["trace_id"]
 
     async def judge(
@@ -197,6 +254,23 @@ class Client:
             args["edited_output"] = edited
         await self.call("feedback", args)
 
+    async def outcome(self, run: str, result: Result, confirmed_cause: str = "", note: str = "", reviewer: str = "") -> None:
+        """Records what a real check found for a run: confirmed, refuted or inconclusive
+
+        A different fact from the verdict on the output
+        confirmed_cause only goes with confirmed
+        """
+        args: dict[str, Any] = {"trace_id": run, "result": result}
+        for key, value in (("confirmed_cause", confirmed_cause), ("note", note), ("reviewer", reviewer)):
+            if value:
+                args[key] = value
+        await self.call("outcome", args)
+
+    async def report(self, name: ReportName) -> Report:
+        """One report of the records as JSON with the start of the window it read"""
+        answer = await self.call("report", {"name": name})
+        return Report(answer["report"], answer.get("since", ""))
+
     async def propose(
         self,
         kind: str,
@@ -225,9 +299,15 @@ class Client:
         answer = await self.call("propose", args)
         return Candidate(answer["id"], answer["version"], answer["status"], answer.get("overlaps") or [])
 
-    async def approve(self, item_id: str, version: int, approver: str) -> Approval:
-        """Approves a candidate on behalf of the named person"""
-        answer = await self.call("approve", {"id": item_id, "version": version, "approver": approver})
+    async def approve(self, item_id: str, version: int, approver: str, *, labels: dict[str, list[str]] | None = None) -> Approval:
+        """Approves a candidate on behalf of the named person
+
+        labels approves it for the runs carrying them instead of the proposed scope and an empty mapping for every run of its producer
+        """
+        args: dict[str, Any] = {"id": item_id, "version": version, "approver": approver}
+        if labels is not None:
+            args["scope"] = {"labels": labels}
+        answer = await self.call("approve", args)
         return Approval(
             answer["id"],
             answer["version"],

{nodloop-0.7.0 → nodloop-0.9.0}/src/nodloop/langchain.py

@@ -13,7 +13,7 @@ from typing import Any
 from langchain.agents.middleware import AgentMiddleware
 from langchain_core.messages import AIMessage, SystemMessage, ToolMessage
 
-from .client import Client, Item, Knowledge
+from .client import Client, Item, Knowledge, Ref
 
 Labels = dict[str, list[str]]
 
@@ -24,12 +24,21 @@ class NodloopMiddleware(AgentMiddleware):
     labels is a mapping or a function of the agent state that returns one
     """
 
-    def __init__(self, client: Client, producer: str, labels: Labels | Callable[[Any], Labels] | None = None) -> None:
+    def __init__(
+        self,
+        client: Client,
+        producer: str,
+        labels: Labels | Callable[[Any], Labels] | None = None,
+        holdout_key: str = "",
+    ) -> None:
         super().__init__()
         self.client = client
         self.producer = producer
         self.labels = labels or {}
+        # Passed to knowledge so a share of runs gets no item and report effect compares them
+        self.holdout_key = holdout_key
         self.applied: list[Item] = []
+        self.withheld: list[Ref] = []
         # The items of the current run, fetched on its first model call and dropped once the run is recorded
         self._knowledge: Knowledge | None = None
         # The run the final answer was recorded as so a verdict can cite it
@@ -40,8 +49,8 @@ class NodloopMiddleware(AgentMiddleware):
 
     async def awrap_model_call(self, request: Any, handler: Callable[[Any], Any]) -> Any:
         if self._knowledge is None:
-            self._knowledge = await self.client.knowledge(self.producer, self._labels(request.state))
-            self.applied = self._knowledge.items
+            self._knowledge = await self.client.knowledge(self.producer, self._labels(request.state), self.holdout_key)
+            self.applied, self.withheld = self._knowledge.items, self._knowledge.withheld
         text = self._knowledge.text
         if not text:
             return await handler(request)
@@ -50,7 +59,14 @@ class NodloopMiddleware(AgentMiddleware):
     async def aafter_agent(self, state: Any, runtime: Any) -> None:
         answers = [m for m in state["messages"] if isinstance(m, AIMessage)]
         if answers:
-            self.run = await self.client.record(self.producer, answers[-1].text, labels=self._labels(state), applied=self.applied)
+            self.run = await self.client.record(
+                self.producer,
+                answers[-1].text,
+                labels=self._labels(state),
+                applied=self.applied,
+                withheld=self.withheld,
+                holdout_key=self.holdout_key,
+            )
         self._knowledge = None
 
     async def awrap_tool_call(self, request: Any, handler: Callable[[Any], Any]) -> Any:

{nodloop-0.7.0 → nodloop-0.9.0}/src/nodloop/openai_agents.py

@@ -13,7 +13,7 @@ from typing import Any
 
 from agents import RunHooks, ToolGuardrailFunctionOutput, tool_input_guardrail
 
-from .client import Client, Item
+from .client import Client, Item, Ref
 
 Labels = dict[str, list[str]]
 
@@ -24,11 +24,14 @@ class NodloopHooks(RunHooks):
     Use one instance per run because it keeps the items and the run id of that run
     """
 
-    def __init__(self, client: Client, producer: str, labels: Labels | None = None) -> None:
+    def __init__(self, client: Client, producer: str, labels: Labels | None = None, holdout_key: str = "") -> None:
         self.client = client
         self.producer = producer
         self.labels = labels or {}
+        # Passed to knowledge so a share of runs gets no item and report effect compares them
+        self.holdout_key = holdout_key
         self.applied: list[Item] = []
+        self.withheld: list[Ref] = []
         # The run the final output was recorded as so a verdict can cite it
         self.run: str = ""
 
@@ -36,15 +39,17 @@ class NodloopHooks(RunHooks):
         """Dynamic instructions: the base and the approved items under it"""
 
         async def build(context: Any, agent: Any) -> str:
-            knowledge = await self.client.knowledge(self.producer, self.labels)
-            self.applied = knowledge.items
+            knowledge = await self.client.knowledge(self.producer, self.labels, self.holdout_key)
+            self.applied, self.withheld = knowledge.items, knowledge.withheld
             return f"{base}\n\n{knowledge.text}" if knowledge.text else base
 
         return build
 
     async def on_agent_end(self, context: Any, agent: Any, output: Any) -> None:
         text = output if isinstance(output, str) else json.dumps(output, default=str)
-        self.run = await self.client.record(self.producer, text, labels=self.labels, applied=self.applied)
+        self.run = await self.client.record(
+            self.producer, text, labels=self.labels, applied=self.applied, withheld=self.withheld, holdout_key=self.holdout_key
+        )
 
 
 def veto_guardrail(client: Client, producer: str = "") -> Any:

{nodloop-0.7.0 → nodloop-0.9.0}/tests/test_adapters.py

@@ -1,11 +1,12 @@
 from dataclasses import dataclass, field, replace
+from pathlib import Path
 from types import SimpleNamespace
 from typing import Any
 
 import pytest
 from langchain_core.messages import AIMessage, HumanMessage, SystemMessage, ToolMessage
 
-from nodloop import Client
+from nodloop import Client, Ref
 from nodloop.claude_agent import NodloopHooks as ClaudeHooks
 from nodloop.langchain import NodloopMiddleware
 from nodloop.openai_agents import NodloopHooks as OpenAIHooks
@@ -159,3 +160,20 @@ async def test_langchain_create_agent(binary: str, records: str) -> None:
     assert mw.run in recorded
     assert len(prompts) == 2 and all(p.startswith("You help\n\nnodloop: corrections") for p in prompts)
     assert [i.id for i in mw.applied] == ["window"]
+
+
+async def test_adapter_records_withheld(binary: str, records: str, home: Path) -> None:
+    """An adapter with a holdout key the holdout draws gets no item and records the run as withheld"""
+    (home / ".nodloop").mkdir(exist_ok=True)
+    (home / ".nodloop" / "config.json").write_text('{"holdout": 0.999}')
+    async with Client(records, binary) as c:
+        await seed(c)
+        hooks = ClaudeHooks(c, "bot", ACME, holdout_key="conversation-1")
+
+        prompt = await hooks.prompt({"prompt": "refund?"}, None, None)
+        await hooks.stop({"last_assistant_message": "The window is 30 days"}, None, None)
+        effect = (await c.call("report", {"name": "effect"}))["report"]
+
+    assert prompt == {}
+    assert hooks.applied == [] and Ref("window", 1) in hooks.withheld
+    assert [(r["runs"], r["judged"], r["undrawn"]) for r in effect if r.get("producer") == "bot" and r["arm"] == "withheld"] == [(1, 0, 0)]

{nodloop-0.7.0 → nodloop-0.9.0}/tests/test_client.py

@@ -1,7 +1,10 @@
+import json
+from pathlib import Path
+
 import pytest
 from mcp.shared.exceptions import MCPError
 
-from nodloop import Client, ToolError
+from nodloop import Client, Ref, ToolError
 
 ACME = {"tenant": ["acme"], "task": ["refund"]}
 NO_RM = {
@@ -96,3 +99,73 @@ async def test_approval_says_a_compaction_is_due(binary: str, records: str) -> N
 
     assert [a.compaction_due for a in approvals] == [False] * 5 + [True]
     assert all(a.folder_error == "" for a in approvals)
+
+
+async def test_approve_in_picked_labels(binary: str, records: str) -> None:
+    async with Client(records, binary) as c:
+        run = await c.record("support-bot", "Here are the refund steps", labels=ACME)
+        await c.record("support-bot", "ok", labels={"tenant": ["globex"]})
+        await c.judge(run, "reject", reason="too long", reason_code="form")
+        proposed = await c.propose("judgment", "Answer in three lines", from_run=run)
+        approved = await c.approve(proposed.id, proposed.version, "ann", labels={})
+        globex = await c.knowledge("support-bot", {"tenant": ["globex"]})
+
+    assert approved.status == "approved"
+    assert [i.id for i in globex.items] == [proposed.id]
+
+
+async def test_holdout_key_withholds_and_record_joins_the_withheld_arm(binary: str, records: str, home: Path) -> None:
+    (home / ".nodloop").mkdir(exist_ok=True)
+    (home / ".nodloop" / "config.json").write_text('{"holdout": 0.5}')
+    async with Client(records, binary) as c:
+        run = await c.record("stream-bot", "lag rose", labels=ACME)
+        await c.judge(run, "reject", reason="name the partition")
+        proposed = await c.propose("judgment", "Name the partition", from_run=run, item_id="partition")
+        await c.approve(proposed.id, proposed.version, "ann")
+        drawn = [await c.knowledge("stream-bot", ACME, holdout_key=f"window-{n}") for n in range(200)]
+        unkeyed = await c.knowledge("stream-bot", ACME)
+        n, withheld = next((n, k) for n, k in enumerate(drawn) if k.withheld)
+        await c.record("stream-bot", "lag rose again", labels=ACME, withheld=withheld.withheld, holdout_key=f"window-{n}")
+        effect = (await c.call("report", {"name": "effect"}))["report"]
+
+    assert 60 <= sum(1 for k in drawn if k.withheld) <= 140
+    assert all((k.items == [] and k.text == "") for k in drawn if k.withheld)
+    assert withheld.withheld == [Ref("partition", 1)]
+    assert unkeyed.withheld == [] and [i.id for i in unkeyed.items] == ["partition"]
+    rows = [r for r in effect if r.get("producer") == "stream-bot" and r["arm"] == "withheld"]
+    assert [(r["runs"], r["judged"], r["undrawn"]) for r in rows] == [(1, 0, 0)]
+
+
+async def test_record_takes_items_as_withheld(binary: str, records: str) -> None:
+    """Items a producer passes as withheld are recorded by id and version like applied ones"""
+    async with Client(records, binary) as c:
+        run = await c.record("stream-bot", "lag rose", labels=ACME)
+        await c.judge(run, "reject", reason="name the partition")
+        proposed = await c.propose("judgment", "Name the partition", from_run=run, item_id="partition")
+        await c.approve(proposed.id, proposed.version, "ann")
+        items = (await c.knowledge("stream-bot", ACME)).items
+
+        recorded = await c.record("stream-bot", "lag rose again", labels=ACME, withheld=items)
+        effect = (await c.report("effect")).report
+
+    runs = [json.loads(line) for line in (Path(records) / "traces.jsonl").read_text().splitlines() if line.strip()]
+    run_input = next(t["input"] for t in runs if t["id"] == recorded)
+    assert run_input["applied"] == [] and run_input["withheld"] == [{"id": "partition", "version": 1}]
+    # Recorded without a holdout key the run counts in no arm and only as undrawn
+    assert [(r["runs"], r["undrawn"]) for r in effect if r.get("producer") == "stream-bot"] == [(0, 1), (0, 1)]
+
+
+async def test_outcome_and_report(binary: str, records: str) -> None:
+    # A prune another surface ran sets the window every report reads
+    Path(records).mkdir()
+    (Path(records) / "prune.json").write_text('{"at":"2020-01-02T00:00:00Z","cutoff":"2020-01-01T00:00:00Z"}')
+    async with Client(records, binary) as c:
+        run = await c.record("stream-bot", "lag comes from partition 7", labels=ACME)
+        await c.outcome(run, "confirmed", confirmed_cause="partition 7 was hot", note="checked the broker")
+        health = await c.report("health")
+        loop = await c.report("loop")
+        with pytest.raises(ToolError):
+            await c.outcome(run, "refuted", confirmed_cause="partition 7 was hot")
+
+    assert isinstance(health.report, (list, dict)) and loop.report["totals"]["runs"] == 1
+    assert loop.since == "2020-01-01T00:00:00Z"

{nodloop-0.7.0 → nodloop-0.9.0}/uv.lock

@@ -337,7 +337,7 @@ wheels = [
 
 [[package]]
 name = "claude-agent-sdk"
-version = "0.2.161"
+version = "0.2.162"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "anyio" },
@@ -346,12 +346,12 @@ dependencies = [
     { name = "sniffio" },
     { name = "typing-extensions", marker = "python_full_version < '3.11'" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/8b/e1/8c94ff7f200eabcee9a9e57825dfac596bb6fbdeeafa594f49bc7e7f89b5/claude_agent_sdk-0.2.161.tar.gz", hash = "sha256:284a1372ec3af133578009ab2b821a238272dbf17d687a58be9df918d7b3ae89", size = 365168, upload-time = "2026-09-28T18:53:39.197Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/78/a3/d31886d0723593228797c96eb54d07a19e561931a2771ed40b4a105bedc9/claude_agent_sdk-0.2.162.tar.gz", hash = "sha256:e4cf92f04f934d3b87d42be9ecbeb75c1a3d487a8dc8a299f7fed1ad9d0b476f", size = 365167, upload-time = "2026-09-29T20:06:54.593Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/e8/be/528e3ec2bb30deff4fcc9484dd47d1022786ad3659d31110bd0ea66ea548/claude_agent_sdk-0.2.161-py3-none-macosx_11_0_arm64.whl", hash = "sha256:127c18a790dc9f77cc2d648f932e11b06cd3325c4b3de7a73d3bcf3d3b74b8bc", size = 93872213, upload-time = "2026-09-28T18:53:42.956Z" },
-    { url = "https://files.pythonhosted.org/packages/17/1c/a0dcb628ac2f25bc92d858857a399202723098057828a909ca4f9893b721/claude_agent_sdk-0.2.161-py3-none-macosx_11_0_x86_64.whl", hash = "sha256:599e1c8ccb4d48ed57999b48e80937d6a6aefbc52b0c0cda1291aea21df29e3e", size = 98316495, upload-time = "2026-09-28T18:53:47.239Z" },
-    { url = "https://files.pythonhosted.org/packages/73/04/3c0caee46c27eee75642b52ee11857c0d3b9d6917ecfd916da2ffc2c2d3f/claude_agent_sdk-0.2.161-py3-none-manylinux_2_17_aarch64.whl", hash = "sha256:18081d3277c07a71891bd15d1496387d9efc5f04a6b482ae3a03e7a0b8067069", size = 103331951, upload-time = "2026-09-28T18:53:51.367Z" },
-    { url = "https://files.pythonhosted.org/packages/09/70/2c7c4160a19c6edd42e317bdc9cf810e91b6a535c04377fb09d4ee090769/claude_agent_sdk-0.2.161-py3-none-manylinux_2_17_x86_64.whl", hash = "sha256:b854e46d2889a51e1a77978711a05c757e63064bd5d8f89bcf74bb507babfc2c", size = 103684983, upload-time = "2026-09-28T18:53:55.496Z" },
+    { url = "https://files.pythonhosted.org/packages/6d/b6/ec37e32cb0782977e80cd00338f5073f150d6ecfa568e9fa1a1dd5c9eed3/claude_agent_sdk-0.2.162-py3-none-macosx_11_0_arm64.whl", hash = "sha256:a0d5ad6185a8bcaaff05f5648036e2484cab28bd1c2ccc474fc8969555b386f7", size = 92700771, upload-time = "2026-09-29T20:06:59.661Z" },
+    { url = "https://files.pythonhosted.org/packages/92/c4/1165688243cbf90d3532f509ea65bc337c3ba7f8229e26d7297e00a6ec2b/claude_agent_sdk-0.2.162-py3-none-macosx_11_0_x86_64.whl", hash = "sha256:faab97b03f04f9c3deb58ad85a0410404d40b6efb5547c9defc4fa986642fced", size = 97130408, upload-time = "2026-09-29T20:07:06.243Z" },
+    { url = "https://files.pythonhosted.org/packages/05/41/f0addb831c3aefbc02c811ef7415e89e889d1bc2ae976eb935c508c91c0d/claude_agent_sdk-0.2.162-py3-none-manylinux_2_17_aarch64.whl", hash = "sha256:332cfb5eb9190fc8de1ad427753417d7c345264e9d2bd2ffa312afcc2f5cf766", size = 102183342, upload-time = "2026-09-29T20:07:14.279Z" },
+    { url = "https://files.pythonhosted.org/packages/d2/33/8a7acd75405d0cfde96c26219dbbac050aa23416ceefd5f62cf7aecfc22d/claude_agent_sdk-0.2.162-py3-none-manylinux_2_17_x86_64.whl", hash = "sha256:673598c9916f5f4d03ff49ab3128f856f0c6508135e8e258838fc3b6f7aafa49", size = 102525631, upload-time = "2026-09-29T20:07:23.411Z" },
 ]
 
 [[package]]
@@ -899,7 +899,7 @@ wheels = [
 
 [[package]]
 name = "nodloop"
-version = "0.7.0"
+version = "0.9.0"
 source = { editable = "." }
 dependencies = [
     { name = "mcp" },