nodloop 0.6.6__tar.gz → 0.8.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- nodloop-0.8.0/PKG-INFO +104 -0
- nodloop-0.8.0/README.md +86 -0
- {nodloop-0.6.6 → nodloop-0.8.0}/pyproject.toml +7 -1
- nodloop-0.8.0/src/nodloop/__init__.py +5 -0
- {nodloop-0.6.6 → nodloop-0.8.0}/src/nodloop/claude_agent.py +1 -1
- {nodloop-0.6.6 → nodloop-0.8.0}/src/nodloop/client.py +56 -8
- {nodloop-0.6.6 → nodloop-0.8.0}/src/nodloop/langchain.py +25 -9
- {nodloop-0.6.6 → nodloop-0.8.0}/src/nodloop/openai_agents.py +1 -1
- {nodloop-0.6.6 → nodloop-0.8.0}/tests/conftest.py +11 -3
- {nodloop-0.6.6 → nodloop-0.8.0}/tests/test_adapters.py +19 -4
- {nodloop-0.6.6 → nodloop-0.8.0}/tests/test_client.py +43 -9
- {nodloop-0.6.6 → nodloop-0.8.0}/uv.lock +7 -7
- nodloop-0.6.6/PKG-INFO +0 -13
- nodloop-0.6.6/src/nodloop/__init__.py +0 -5
- {nodloop-0.6.6 → nodloop-0.8.0}/.gitignore +0 -0
- {nodloop-0.6.6 → nodloop-0.8.0}/src/nodloop/binary.py +0 -0
nodloop-0.8.0/PKG-INFO
ADDED
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: nodloop
|
|
3
|
+
Version: 0.8.0
|
|
4
|
+
Summary: Record AI runs and the verdicts people give on them, and hand the knowledge they approve to the next run in the same place
|
|
5
|
+
Project-URL: Homepage, https://github.com/jeon-jihyeon/nodloop
|
|
6
|
+
Project-URL: Repository, https://github.com/jeon-jihyeon/nodloop
|
|
7
|
+
License-Expression: MIT
|
|
8
|
+
Keywords: agents,ai,feedback,human-in-the-loop,langchain,mcp,memory
|
|
9
|
+
Requires-Python: >=3.10
|
|
10
|
+
Requires-Dist: mcp>=2.0
|
|
11
|
+
Provides-Extra: claude-agent
|
|
12
|
+
Requires-Dist: claude-agent-sdk>=0.1; extra == 'claude-agent'
|
|
13
|
+
Provides-Extra: langchain
|
|
14
|
+
Requires-Dist: langchain>=1.0; extra == 'langchain'
|
|
15
|
+
Provides-Extra: openai-agents
|
|
16
|
+
Requires-Dist: openai-agents>=0.3.3; extra == 'openai-agents'
|
|
17
|
+
Description-Content-Type: text/markdown
|
|
18
|
+
|
|
19
|
+
# nodloop for Python
|
|
20
|
+
|
|
21
|
+
Record what your agent answered, take a person's verdict on it, and hand the corrections they approve to the agent's next run in the same situation. This is the Python client of [nodloop](https://github.com/jeon-jihyeon/nodloop).
|
|
22
|
+
|
|
23
|
+
```
|
|
24
|
+
pip install nodloop
|
|
25
|
+
pip install "nodloop[langchain]" # or openai-agents, claude-agent
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
The client starts a local `nodloop mcp` over stdio, so no server runs. It uses `NODLOOP_BIN`, then the binary the Claude Code plugin keeps under `~/.nodloop/bin`, then `nodloop` on PATH, and otherwise downloads the release of its own version from GitHub and checks it against the release checksums. It requires Python 3.10 or newer and is tested on Linux with Python 3.12.
|
|
29
|
+
|
|
30
|
+
## The loop
|
|
31
|
+
|
|
32
|
+
```python
|
|
33
|
+
from nodloop import Client
|
|
34
|
+
|
|
35
|
+
acme = {"tenant": ["acme"], "task": ["refund"]}
|
|
36
|
+
|
|
37
|
+
async with Client() as c:
|
|
38
|
+
knowledge = await c.knowledge("support-bot", acme)
|
|
39
|
+
prompt = f"{system_prompt}\n\n{knowledge.text}" # empty text when nothing is approved
|
|
40
|
+
answer = await my_agent(prompt)
|
|
41
|
+
run = await c.record("support-bot", answer, labels=acme, applied=knowledge.items)
|
|
42
|
+
|
|
43
|
+
# later, when a person reads the answer
|
|
44
|
+
await c.judge(run, "reject", reason="the refund window was missing", reason_code="scope")
|
|
45
|
+
candidate = await c.propose("judgment", "Quote the refund window before the steps", from_run=run)
|
|
46
|
+
approval = await c.approve(candidate.id, candidate.version, approver="ann")
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
From then on every `knowledge("support-bot", acme)` carries the item, and a run of another tenant never does. `judge` takes `approve`, `edit` with `edited=` holding the corrected output, `reject` with what was wrong, or `withdraw` to take back the verdict before it.
|
|
50
|
+
|
|
51
|
+
| Method | Returns |
|
|
52
|
+
|---|---|
|
|
53
|
+
| `knowledge(producer, labels)` | `Knowledge` with `items` and the `text` to put in the prompt |
|
|
54
|
+
| `record(producer, output, *, labels, applied, subject)` | the run id a verdict cites |
|
|
55
|
+
| `judge(run, verdict, reason, reason_code, edited, reviewer)` | nothing |
|
|
56
|
+
| `propose(kind, content, from_run, producer, labels, new_labels, item_id, veto)` | `Candidate` with `id`, `version`, `status` and `overlaps`, the current items of the same kind one run may carry with it |
|
|
57
|
+
| `approve(item_id, version, approver, labels=)` | Approves as proposed, or for the runs carrying `labels` when given, and `{}` for every run of the producer. `Approval` with `id`, `version`, `status`, `approver`, `veto`, `compaction_due` when one run would carry more than five items, and `export_error` and `folder_error` when a step after the approval failed |
|
|
58
|
+
| `check_call(tool, arguments, producer)` | `Decision` with `action` of allow, block or ask, `veto` and `reason` |
|
|
59
|
+
|
|
60
|
+
A tool that refuses, such as a proposal from a run nobody corrected, raises `ToolError` with the reason.
|
|
61
|
+
|
|
62
|
+
## Adapters
|
|
63
|
+
|
|
64
|
+
LangChain `create_agent` middleware fetches the items once per run and puts them in the system message, records the final answer and refuses a vetoed tool call:
|
|
65
|
+
|
|
66
|
+
```python
|
|
67
|
+
from nodloop.langchain import NodloopMiddleware
|
|
68
|
+
|
|
69
|
+
mw = NodloopMiddleware(c, "support-bot", acme)
|
|
70
|
+
agent = create_agent(model, tools, middleware=[mw])
|
|
71
|
+
await agent.ainvoke({"messages": [...]})
|
|
72
|
+
await c.judge(mw.run, "approve")
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
OpenAI Agents SDK hooks and a tool input guardrail:
|
|
76
|
+
|
|
77
|
+
```python
|
|
78
|
+
from nodloop.openai_agents import NodloopHooks, veto_guardrail
|
|
79
|
+
|
|
80
|
+
hooks = NodloopHooks(c, "support-bot", acme)
|
|
81
|
+
agent = Agent(name="support", instructions=hooks.instructions("You help with refunds"), tools=[...])
|
|
82
|
+
await Runner.run(agent, "refund?", hooks=hooks)
|
|
83
|
+
```
|
|
84
|
+
|
|
85
|
+
Pass `veto_guardrail(c, "support-bot")` in a tool's `tool_input_guardrails`.
|
|
86
|
+
|
|
87
|
+
Claude Agent SDK hooks, the same loop the Claude Code plugin runs:
|
|
88
|
+
|
|
89
|
+
```python
|
|
90
|
+
from nodloop.claude_agent import NodloopHooks
|
|
91
|
+
|
|
92
|
+
hooks = NodloopHooks(c, "support-bot", acme)
|
|
93
|
+
options = ClaudeAgentOptions(hooks=hooks.hooks())
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
Use one adapter instance per run or conversation, since it keeps the items it handed out and the id of the run it recorded.
|
|
97
|
+
|
|
98
|
+
## A shared server
|
|
99
|
+
|
|
100
|
+
`Client(url="https://nodloop.example/mcp", key="nl_...")` connects to `nodloop-server serve` instead of a local binary. The key decides the tenant and what the client may do, and an approval through it is recorded under the key's name.
|
|
101
|
+
|
|
102
|
+
## More
|
|
103
|
+
|
|
104
|
+
The [nodloop README](https://github.com/jeon-jihyeon/nodloop#readme) covers labels, vetoes, reports and the server. MIT licensed.
|
nodloop-0.8.0/README.md
ADDED
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
# nodloop for Python
|
|
2
|
+
|
|
3
|
+
Record what your agent answered, take a person's verdict on it, and hand the corrections they approve to the agent's next run in the same situation. This is the Python client of [nodloop](https://github.com/jeon-jihyeon/nodloop).
|
|
4
|
+
|
|
5
|
+
```
|
|
6
|
+
pip install nodloop
|
|
7
|
+
pip install "nodloop[langchain]" # or openai-agents, claude-agent
|
|
8
|
+
```
|
|
9
|
+
|
|
10
|
+
The client starts a local `nodloop mcp` over stdio, so no server runs. It uses `NODLOOP_BIN`, then the binary the Claude Code plugin keeps under `~/.nodloop/bin`, then `nodloop` on PATH, and otherwise downloads the release of its own version from GitHub and checks it against the release checksums. It requires Python 3.10 or newer and is tested on Linux with Python 3.12.
|
|
11
|
+
|
|
12
|
+
## The loop
|
|
13
|
+
|
|
14
|
+
```python
|
|
15
|
+
from nodloop import Client
|
|
16
|
+
|
|
17
|
+
acme = {"tenant": ["acme"], "task": ["refund"]}
|
|
18
|
+
|
|
19
|
+
async with Client() as c:
|
|
20
|
+
knowledge = await c.knowledge("support-bot", acme)
|
|
21
|
+
prompt = f"{system_prompt}\n\n{knowledge.text}" # empty text when nothing is approved
|
|
22
|
+
answer = await my_agent(prompt)
|
|
23
|
+
run = await c.record("support-bot", answer, labels=acme, applied=knowledge.items)
|
|
24
|
+
|
|
25
|
+
# later, when a person reads the answer
|
|
26
|
+
await c.judge(run, "reject", reason="the refund window was missing", reason_code="scope")
|
|
27
|
+
candidate = await c.propose("judgment", "Quote the refund window before the steps", from_run=run)
|
|
28
|
+
approval = await c.approve(candidate.id, candidate.version, approver="ann")
|
|
29
|
+
```
|
|
30
|
+
|
|
31
|
+
From then on every `knowledge("support-bot", acme)` carries the item, and a run of another tenant never does. `judge` takes `approve`, `edit` with `edited=` holding the corrected output, `reject` with what was wrong, or `withdraw` to take back the verdict before it.
|
|
32
|
+
|
|
33
|
+
| Method | Returns |
|
|
34
|
+
|---|---|
|
|
35
|
+
| `knowledge(producer, labels)` | `Knowledge` with `items` and the `text` to put in the prompt |
|
|
36
|
+
| `record(producer, output, *, labels, applied, subject)` | the run id a verdict cites |
|
|
37
|
+
| `judge(run, verdict, reason, reason_code, edited, reviewer)` | nothing |
|
|
38
|
+
| `propose(kind, content, from_run, producer, labels, new_labels, item_id, veto)` | `Candidate` with `id`, `version`, `status` and `overlaps`, the current items of the same kind one run may carry with it |
|
|
39
|
+
| `approve(item_id, version, approver, labels=)` | Approves as proposed, or for the runs carrying `labels` when given, and `{}` for every run of the producer. `Approval` with `id`, `version`, `status`, `approver`, `veto`, `compaction_due` when one run would carry more than five items, and `export_error` and `folder_error` when a step after the approval failed |
|
|
40
|
+
| `check_call(tool, arguments, producer)` | `Decision` with `action` of allow, block or ask, `veto` and `reason` |
|
|
41
|
+
|
|
42
|
+
A tool that refuses, such as a proposal from a run nobody corrected, raises `ToolError` with the reason.
|
|
43
|
+
|
|
44
|
+
## Adapters
|
|
45
|
+
|
|
46
|
+
LangChain `create_agent` middleware fetches the items once per run and puts them in the system message, records the final answer and refuses a vetoed tool call:
|
|
47
|
+
|
|
48
|
+
```python
|
|
49
|
+
from nodloop.langchain import NodloopMiddleware
|
|
50
|
+
|
|
51
|
+
mw = NodloopMiddleware(c, "support-bot", acme)
|
|
52
|
+
agent = create_agent(model, tools, middleware=[mw])
|
|
53
|
+
await agent.ainvoke({"messages": [...]})
|
|
54
|
+
await c.judge(mw.run, "approve")
|
|
55
|
+
```
|
|
56
|
+
|
|
57
|
+
OpenAI Agents SDK hooks and a tool input guardrail:
|
|
58
|
+
|
|
59
|
+
```python
|
|
60
|
+
from nodloop.openai_agents import NodloopHooks, veto_guardrail
|
|
61
|
+
|
|
62
|
+
hooks = NodloopHooks(c, "support-bot", acme)
|
|
63
|
+
agent = Agent(name="support", instructions=hooks.instructions("You help with refunds"), tools=[...])
|
|
64
|
+
await Runner.run(agent, "refund?", hooks=hooks)
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
Pass `veto_guardrail(c, "support-bot")` in a tool's `tool_input_guardrails`.
|
|
68
|
+
|
|
69
|
+
Claude Agent SDK hooks, the same loop the Claude Code plugin runs:
|
|
70
|
+
|
|
71
|
+
```python
|
|
72
|
+
from nodloop.claude_agent import NodloopHooks
|
|
73
|
+
|
|
74
|
+
hooks = NodloopHooks(c, "support-bot", acme)
|
|
75
|
+
options = ClaudeAgentOptions(hooks=hooks.hooks())
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
Use one adapter instance per run or conversation, since it keeps the items it handed out and the id of the run it recorded.
|
|
79
|
+
|
|
80
|
+
## A shared server
|
|
81
|
+
|
|
82
|
+
`Client(url="https://nodloop.example/mcp", key="nl_...")` connects to `nodloop-server serve` instead of a local binary. The key decides the tenant and what the client may do, and an approval through it is recorded under the key's name.
|
|
83
|
+
|
|
84
|
+
## More
|
|
85
|
+
|
|
86
|
+
The [nodloop README](https://github.com/jeon-jihyeon/nodloop#readme) covers labels, vetoes, reports and the server. MIT licensed.
|
|
@@ -1,11 +1,17 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "nodloop"
|
|
3
|
-
version = "0.
|
|
3
|
+
version = "0.8.0"
|
|
4
4
|
description = "Record AI runs and the verdicts people give on them, and hand the knowledge they approve to the next run in the same place"
|
|
5
|
+
readme = "README.md"
|
|
5
6
|
license = "MIT"
|
|
7
|
+
keywords = ["ai", "agents", "feedback", "human-in-the-loop", "langchain", "mcp", "memory"]
|
|
6
8
|
requires-python = ">=3.10"
|
|
7
9
|
dependencies = ["mcp>=2.0"]
|
|
8
10
|
|
|
11
|
+
[project.urls]
|
|
12
|
+
Homepage = "https://github.com/jeon-jihyeon/nodloop"
|
|
13
|
+
Repository = "https://github.com/jeon-jihyeon/nodloop"
|
|
14
|
+
|
|
9
15
|
[project.optional-dependencies]
|
|
10
16
|
langchain = ["langchain>=1.0"]
|
|
11
17
|
openai-agents = ["openai-agents>=0.3.3"]
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
"""Record AI runs and the verdicts people give on them, and hand the knowledge they approve to the next run in the same place"""
|
|
2
|
+
|
|
3
|
+
from .client import Approval, Candidate, Client, Decision, Item, Knowledge, ToolError
|
|
4
|
+
|
|
5
|
+
__all__ = ["Approval", "Candidate", "Client", "Decision", "Item", "Knowledge", "ToolError"]
|
|
@@ -49,7 +49,7 @@ class NodloopHooks:
|
|
|
49
49
|
# A turn without one records nothing
|
|
50
50
|
answer = input_data.get("last_assistant_message") or ""
|
|
51
51
|
if answer.strip():
|
|
52
|
-
self.run = await self.client.record(self.producer, answer, self.labels, self.applied)
|
|
52
|
+
self.run = await self.client.record(self.producer, answer, labels=self.labels, applied=self.applied)
|
|
53
53
|
return {}
|
|
54
54
|
|
|
55
55
|
async def pre_tool_use(self, input_data: dict[str, Any], tool_use_id: str | None, context: Any) -> dict[str, Any]:
|
|
@@ -14,10 +14,10 @@ from mcp.client.streamable_http import create_mcp_http_client, streamable_http_c
|
|
|
14
14
|
|
|
15
15
|
from . import binary
|
|
16
16
|
|
|
17
|
-
__all__ = ["Client", "Decision", "Item", "Knowledge", "ToolError"]
|
|
17
|
+
__all__ = ["Approval", "Candidate", "Client", "Decision", "Item", "Knowledge", "ToolError"]
|
|
18
18
|
|
|
19
19
|
# The binary release the client fetches when no nodloop is installed
|
|
20
|
-
VERSION = "0.
|
|
20
|
+
VERSION = "0.8.0"
|
|
21
21
|
|
|
22
22
|
|
|
23
23
|
class ToolError(RuntimeError):
|
|
@@ -52,6 +52,36 @@ class Decision:
|
|
|
52
52
|
return self.action == "allow"
|
|
53
53
|
|
|
54
54
|
|
|
55
|
+
@dataclass(frozen=True)
|
|
56
|
+
class Candidate:
|
|
57
|
+
"""A proposed knowledge version that waits for a person to approve it"""
|
|
58
|
+
|
|
59
|
+
id: str
|
|
60
|
+
version: int
|
|
61
|
+
status: str
|
|
62
|
+
# The current items of the same kind that one run may carry with the candidate, each as the ledger records it
|
|
63
|
+
overlaps: list[dict[str, Any]] = field(default_factory=list)
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
@dataclass(frozen=True)
|
|
67
|
+
class Approval:
|
|
68
|
+
"""An approved knowledge version"""
|
|
69
|
+
|
|
70
|
+
id: str
|
|
71
|
+
version: int
|
|
72
|
+
status: str
|
|
73
|
+
# The approver the records keep, which a server sets to the key's name
|
|
74
|
+
approver: str
|
|
75
|
+
# Whether the approved version carries a veto the guard now applies
|
|
76
|
+
veto: bool = False
|
|
77
|
+
# Whether one run would carry more than five items so a compaction is due
|
|
78
|
+
compaction_due: bool = False
|
|
79
|
+
# Why approved.md or the veto file could not be written, while the approval itself stands
|
|
80
|
+
export_error: str = ""
|
|
81
|
+
# Why the items one run carries with it could not be read, while the approval itself stands
|
|
82
|
+
folder_error: str = ""
|
|
83
|
+
|
|
84
|
+
|
|
55
85
|
@dataclass(frozen=True)
|
|
56
86
|
class Knowledge:
|
|
57
87
|
"""The approved items a run receives and the text that carries them into its prompt"""
|
|
@@ -66,7 +96,7 @@ class Client:
|
|
|
66
96
|
|
|
67
97
|
1. by default it starts `nodloop mcp` over stdio, so nothing runs as a server
|
|
68
98
|
2. record_dir names the records and else NODLOOP_RECORD_DIR, config.json or ~/.nodloop/records decide as in the CLI
|
|
69
|
-
3. url and key connect to `nodloop
|
|
99
|
+
3. url and key connect to `nodloop-server serve` instead, where the key decides the tenant and the role
|
|
70
100
|
4. use it as an async context manager so the process or the connection ends with the block
|
|
71
101
|
"""
|
|
72
102
|
|
|
@@ -136,6 +166,7 @@ class Client:
|
|
|
136
166
|
self,
|
|
137
167
|
producer: str,
|
|
138
168
|
output: Any,
|
|
169
|
+
*,
|
|
139
170
|
labels: dict[str, list[str]] | None = None,
|
|
140
171
|
applied: list[Item] | None = None,
|
|
141
172
|
subject: str = "",
|
|
@@ -176,7 +207,7 @@ class Client:
|
|
|
176
207
|
new_labels: bool = False,
|
|
177
208
|
item_id: str = "",
|
|
178
209
|
veto: dict[str, Any] | None = None,
|
|
179
|
-
) ->
|
|
210
|
+
) -> Candidate:
|
|
180
211
|
"""Proposes a candidate a person approves later, from a corrected run or for a producer and labels
|
|
181
212
|
|
|
182
213
|
veto names a tool call the judgment forbids as tool, when and example, as the MCP tool propose takes it
|
|
@@ -191,11 +222,28 @@ class Client:
|
|
|
191
222
|
args["labels"] = labels
|
|
192
223
|
if new_labels:
|
|
193
224
|
args["new_labels"] = True
|
|
194
|
-
|
|
225
|
+
answer = await self.call("propose", args)
|
|
226
|
+
return Candidate(answer["id"], answer["version"], answer["status"], answer.get("overlaps") or [])
|
|
227
|
+
|
|
228
|
+
async def approve(self, item_id: str, version: int, approver: str, *, labels: dict[str, list[str]] | None = None) -> Approval:
|
|
229
|
+
"""Approves a candidate on behalf of the named person
|
|
195
230
|
|
|
196
|
-
|
|
197
|
-
"""
|
|
198
|
-
|
|
231
|
+
labels approves it for the runs carrying them instead of the proposed scope and an empty mapping for every run of its producer
|
|
232
|
+
"""
|
|
233
|
+
args: dict[str, Any] = {"id": item_id, "version": version, "approver": approver}
|
|
234
|
+
if labels is not None:
|
|
235
|
+
args["scope"] = {"labels": labels}
|
|
236
|
+
answer = await self.call("approve", args)
|
|
237
|
+
return Approval(
|
|
238
|
+
answer["id"],
|
|
239
|
+
answer["version"],
|
|
240
|
+
answer["status"],
|
|
241
|
+
answer["approver"],
|
|
242
|
+
veto=bool(answer.get("veto")),
|
|
243
|
+
compaction_due=bool((answer.get("folder") or {}).get("compaction_due")),
|
|
244
|
+
export_error=answer.get("export_error", ""),
|
|
245
|
+
folder_error=answer.get("folder_error", ""),
|
|
246
|
+
)
|
|
199
247
|
|
|
200
248
|
async def check_call(self, tool: str, arguments: dict[str, Any], producer: str = "") -> Decision:
|
|
201
249
|
"""Whether a veto approved for the producer blocks or asks about a tool call before the agent runs it
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"""LangChain agents middleware for nodloop
|
|
2
2
|
|
|
3
|
-
1. every model call gets the approved items of the place in its system message
|
|
3
|
+
1. every model call of a run gets the approved items of the place in its system message, fetched once per run
|
|
4
4
|
2. the final answer of the agent is recorded as a run naming the items it got
|
|
5
5
|
3. every tool call is checked against the approved vetoes and a blocked one answers the model with the reason
|
|
6
6
|
"""
|
|
@@ -13,7 +13,7 @@ from typing import Any
|
|
|
13
13
|
from langchain.agents.middleware import AgentMiddleware
|
|
14
14
|
from langchain_core.messages import AIMessage, SystemMessage, ToolMessage
|
|
15
15
|
|
|
16
|
-
from .client import Client, Item
|
|
16
|
+
from .client import Client, Item, Knowledge
|
|
17
17
|
|
|
18
18
|
Labels = dict[str, list[str]]
|
|
19
19
|
|
|
@@ -30,6 +30,8 @@ class NodloopMiddleware(AgentMiddleware):
|
|
|
30
30
|
self.producer = producer
|
|
31
31
|
self.labels = labels or {}
|
|
32
32
|
self.applied: list[Item] = []
|
|
33
|
+
# The items of the current run, fetched on its first model call and dropped once the run is recorded
|
|
34
|
+
self._knowledge: Knowledge | None = None
|
|
33
35
|
# The run the final answer was recorded as so a verdict can cite it
|
|
34
36
|
self.run: str = ""
|
|
35
37
|
|
|
@@ -37,18 +39,19 @@ class NodloopMiddleware(AgentMiddleware):
|
|
|
37
39
|
return self.labels(state) if callable(self.labels) else self.labels
|
|
38
40
|
|
|
39
41
|
async def awrap_model_call(self, request: Any, handler: Callable[[Any], Any]) -> Any:
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
42
|
+
if self._knowledge is None:
|
|
43
|
+
self._knowledge = await self.client.knowledge(self.producer, self._labels(request.state))
|
|
44
|
+
self.applied = self._knowledge.items
|
|
45
|
+
text = self._knowledge.text
|
|
46
|
+
if not text:
|
|
43
47
|
return await handler(request)
|
|
44
|
-
|
|
45
|
-
system = SystemMessage(content=f"{base}\n\n{knowledge.text}" if base else knowledge.text)
|
|
46
|
-
return await handler(request.override(system_message=system))
|
|
48
|
+
return await handler(request.override(system_message=_with_text(request.system_message, text)))
|
|
47
49
|
|
|
48
50
|
async def aafter_agent(self, state: Any, runtime: Any) -> None:
|
|
49
51
|
answers = [m for m in state["messages"] if isinstance(m, AIMessage)]
|
|
50
52
|
if answers:
|
|
51
|
-
self.run = await self.client.record(self.producer, answers[-1].text, self._labels(state), self.applied)
|
|
53
|
+
self.run = await self.client.record(self.producer, answers[-1].text, labels=self._labels(state), applied=self.applied)
|
|
54
|
+
self._knowledge = None
|
|
52
55
|
|
|
53
56
|
async def awrap_tool_call(self, request: Any, handler: Callable[[Any], Any]) -> Any:
|
|
54
57
|
call = request.tool_call
|
|
@@ -60,3 +63,16 @@ class NodloopMiddleware(AgentMiddleware):
|
|
|
60
63
|
tool_call_id=call["id"],
|
|
61
64
|
status="error",
|
|
62
65
|
)
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def _with_text(system: SystemMessage | None, text: str) -> SystemMessage:
|
|
69
|
+
"""The system message with the text after its content, keeping content blocks as they are
|
|
70
|
+
|
|
71
|
+
The text of blocks joins them as they are, so the new text leads with the blank line
|
|
72
|
+
"""
|
|
73
|
+
if system is None:
|
|
74
|
+
return SystemMessage(content=text)
|
|
75
|
+
after = f"\n\n{text}" if system.text else text
|
|
76
|
+
if isinstance(system.content, str):
|
|
77
|
+
return system.model_copy(update={"content": system.content + after})
|
|
78
|
+
return system.model_copy(update={"content": [*system.content, {"type": "text", "text": after}]})
|
|
@@ -44,7 +44,7 @@ class NodloopHooks(RunHooks):
|
|
|
44
44
|
|
|
45
45
|
async def on_agent_end(self, context: Any, agent: Any, output: Any) -> None:
|
|
46
46
|
text = output if isinstance(output, str) else json.dumps(output, default=str)
|
|
47
|
-
self.run = await self.client.record(self.producer, text, self.labels, self.applied)
|
|
47
|
+
self.run = await self.client.record(self.producer, text, labels=self.labels, applied=self.applied)
|
|
48
48
|
|
|
49
49
|
|
|
50
50
|
def veto_guardrail(client: Client, producer: str = "") -> Any:
|
|
@@ -17,6 +17,14 @@ def binary(tmp_path_factory: pytest.TempPathFactory) -> str:
|
|
|
17
17
|
return str(out)
|
|
18
18
|
|
|
19
19
|
|
|
20
|
+
@pytest.fixture(scope="session")
|
|
21
|
+
def server_binary(tmp_path_factory: pytest.TempPathFactory) -> str:
|
|
22
|
+
"""The nodloop-server of this checkout built once for the session"""
|
|
23
|
+
out = tmp_path_factory.mktemp("bin") / "nodloop-server"
|
|
24
|
+
subprocess.run(["go", "build", "-o", str(out), "./cmd/nodloop-server"], cwd=ROOT / "server", check=True)
|
|
25
|
+
return str(out)
|
|
26
|
+
|
|
27
|
+
|
|
20
28
|
@pytest.fixture(autouse=True)
|
|
21
29
|
def home(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path:
|
|
22
30
|
"""A home of the test so an approval never writes a veto file under the real one"""
|
|
@@ -32,15 +40,15 @@ def records(tmp_path: Path) -> str:
|
|
|
32
40
|
|
|
33
41
|
|
|
34
42
|
@pytest.fixture
|
|
35
|
-
def server(
|
|
43
|
+
def server(server_binary: str, tmp_path: Path):
|
|
36
44
|
"""A nodloop server on a free port with an approver key of tenant acme, yielding its url and key"""
|
|
37
45
|
env = dict(os.environ)
|
|
38
|
-
add = [
|
|
46
|
+
add = [server_binary, "key", "add", "ann", "--tenant", "acme", "--role", "approver"]
|
|
39
47
|
key = subprocess.run(add, env=env, check=True, capture_output=True, text=True).stdout.strip()
|
|
40
48
|
with socket.socket() as s:
|
|
41
49
|
s.bind(("127.0.0.1", 0))
|
|
42
50
|
port = s.getsockname()[1]
|
|
43
|
-
serve = [
|
|
51
|
+
serve = [server_binary, "serve", "--addr", f"127.0.0.1:{port}", "--record-dir", str(tmp_path / "records")]
|
|
44
52
|
proc = subprocess.Popen(serve, env=env, stderr=subprocess.DEVNULL)
|
|
45
53
|
for _ in range(100):
|
|
46
54
|
try:
|
|
@@ -17,11 +17,11 @@ NO_RM = {"tool": "Bash", "when": [{"field": "commands", "match": "(?m)^rm "}], "
|
|
|
17
17
|
|
|
18
18
|
async def seed(c: Client) -> None:
|
|
19
19
|
"""An approved item for acme and an approved veto on rm"""
|
|
20
|
-
run = await c.record("bot", "answer", ACME)
|
|
20
|
+
run = await c.record("bot", "answer", labels=ACME)
|
|
21
21
|
await c.judge(run, "reject", reason="missing the window")
|
|
22
22
|
for item_id, content, veto in (("window", "Quote the refund window", None), ("no-rm", "Never run rm", NO_RM)):
|
|
23
23
|
p = await c.propose("judgment", content, from_run=run, item_id=item_id, veto=veto)
|
|
24
|
-
await c.approve(p
|
|
24
|
+
await c.approve(p.id, p.version, "ann")
|
|
25
25
|
|
|
26
26
|
|
|
27
27
|
async def runs(c: Client) -> list[str]:
|
|
@@ -54,6 +54,10 @@ async def test_langchain_middleware(binary: str, records: str) -> None:
|
|
|
54
54
|
return ToolMessage("ran", tool_call_id=request.tool_call["id"])
|
|
55
55
|
|
|
56
56
|
await mw.awrap_model_call(ModelRequest(system_message=SystemMessage("You help with refunds")), model)
|
|
57
|
+
first = mw.applied
|
|
58
|
+
blocks = [{"type": "text", "text": "You help"}, {"type": "text", "text": "with refunds"}]
|
|
59
|
+
await mw.awrap_model_call(ModelRequest(system_message=SystemMessage(content=blocks)), model)
|
|
60
|
+
fetched_once = mw.applied is first
|
|
57
61
|
blocked = await mw.awrap_tool_call(SimpleNamespace(tool_call={"id": "t1", "name": "Bash", "args": {"command": "rm -rf /x"}}), tool)
|
|
58
62
|
allowed = await mw.awrap_tool_call(SimpleNamespace(tool_call={"id": "t2", "name": "Bash", "args": {"command": "ls"}}), tool)
|
|
59
63
|
await mw.aafter_agent({"messages": [HumanMessage("refund?"), AIMessage("The window is 30 days")]}, None)
|
|
@@ -61,6 +65,9 @@ async def test_langchain_middleware(binary: str, records: str) -> None:
|
|
|
61
65
|
|
|
62
66
|
assert seen[0].system_message.content.startswith("You help with refunds\n\nnodloop: corrections")
|
|
63
67
|
assert "[window v1 judgment] Quote the refund window" in seen[0].system_message.content
|
|
68
|
+
assert fetched_once and [i.id for i in first] == ["window"]
|
|
69
|
+
assert seen[1].system_message.content[:2] == blocks
|
|
70
|
+
assert seen[1].system_message.content[2]["text"].startswith("\n\nnodloop: corrections")
|
|
64
71
|
assert blocked.status == "error" and "no-rm" in blocked.content
|
|
65
72
|
assert allowed.content == "ran"
|
|
66
73
|
assert mw.run in recorded
|
|
@@ -111,10 +118,16 @@ async def test_langchain_create_agent(binary: str, records: str) -> None:
|
|
|
111
118
|
from langchain_core.language_models.fake_chat_models import GenericFakeChatModel
|
|
112
119
|
from langchain_core.tools import tool
|
|
113
120
|
|
|
121
|
+
prompts: list[str] = []
|
|
122
|
+
|
|
114
123
|
class Scripted(GenericFakeChatModel):
|
|
115
124
|
def bind_tools(self, tools: Any, **kwargs: Any) -> "Scripted":
|
|
116
125
|
return self
|
|
117
126
|
|
|
127
|
+
async def _agenerate(self, messages: Any, *args: Any, **kwargs: Any) -> Any:
|
|
128
|
+
prompts.append(messages[0].text)
|
|
129
|
+
return await super()._agenerate(messages, *args, **kwargs)
|
|
130
|
+
|
|
118
131
|
ran: list[str] = []
|
|
119
132
|
|
|
120
133
|
@tool
|
|
@@ -129,11 +142,11 @@ async def test_langchain_create_agent(binary: str, records: str) -> None:
|
|
|
129
142
|
async with Client(records, binary) as c:
|
|
130
143
|
await seed(c)
|
|
131
144
|
# A veto names the tool as this agent calls it
|
|
132
|
-
run = await c.record("bot", "ran rm", ACME)
|
|
145
|
+
run = await c.record("bot", "ran rm", labels=ACME)
|
|
133
146
|
await c.judge(run, "reject", reason="never rm")
|
|
134
147
|
veto = {"tool": "bash", "when": [{"field": "commands", "match": "(?m)^rm "}], "example": {"command": "rm -rf /data"}}
|
|
135
148
|
p = await c.propose("judgment", "Never run rm through bash", from_run=run, item_id="no-rm-bash", veto=veto)
|
|
136
|
-
await c.approve(p
|
|
149
|
+
await c.approve(p.id, p.version, "ann")
|
|
137
150
|
mw = NodloopMiddleware(c, "bot", ACME)
|
|
138
151
|
agent = create_agent(Scripted(messages=script), tools=[bash], system_prompt="You help", middleware=[mw])
|
|
139
152
|
|
|
@@ -144,3 +157,5 @@ async def test_langchain_create_agent(binary: str, records: str) -> None:
|
|
|
144
157
|
assert ran == []
|
|
145
158
|
assert tool_messages and "nodloop veto" in tool_messages[0].content
|
|
146
159
|
assert mw.run in recorded
|
|
160
|
+
assert len(prompts) == 2 and all(p.startswith("You help\n\nnodloop: corrections") for p in prompts)
|
|
161
|
+
assert [i.id for i in mw.applied] == ["window"]
|
|
@@ -13,15 +13,16 @@ NO_RM = {
|
|
|
13
13
|
|
|
14
14
|
async def test_loop(binary: str, records: str) -> None:
|
|
15
15
|
async with Client(records, binary) as c:
|
|
16
|
-
run = await c.record("support-bot", "Here are the refund steps", ACME)
|
|
16
|
+
run = await c.record("support-bot", "Here are the refund steps", labels=ACME)
|
|
17
17
|
await c.judge(run, "reject", reason="the refund window was missing", reason_code="scope")
|
|
18
18
|
proposed = await c.propose("judgment", "Quote the refund window before the steps", from_run=run, item_id="refund-window")
|
|
19
|
-
await c.approve(proposed
|
|
19
|
+
approved = await c.approve(proposed.id, proposed.version, "ann")
|
|
20
20
|
|
|
21
21
|
acme = await c.knowledge("support-bot", ACME)
|
|
22
22
|
globex = await c.knowledge("support-bot", {"tenant": ["globex"], "task": ["refund"]})
|
|
23
|
-
nxt = await c.record("support-bot", "The window is 30 days. Steps follow", ACME, acme.items)
|
|
23
|
+
nxt = await c.record("support-bot", "The window is 30 days. Steps follow", labels=ACME, applied=acme.items)
|
|
24
24
|
|
|
25
|
+
assert (approved.veto, approved.compaction_due, approved.export_error, approved.folder_error) == (False, False, "", "")
|
|
25
26
|
assert [i.id for i in acme.items] == ["refund-window"]
|
|
26
27
|
assert acme.text.startswith("nodloop: corrections a person approved")
|
|
27
28
|
assert "- [refund-window v1 judgment] Quote the refund window before the steps" in acme.text
|
|
@@ -31,7 +32,7 @@ async def test_loop(binary: str, records: str) -> None:
|
|
|
31
32
|
|
|
32
33
|
async def test_refusal_is_a_tool_error(binary: str, records: str) -> None:
|
|
33
34
|
async with Client(records, binary) as c:
|
|
34
|
-
run = await c.record("support-bot", "answer", ACME)
|
|
35
|
+
run = await c.record("support-bot", "answer", labels=ACME)
|
|
35
36
|
with pytest.raises(ToolError, match="needs an edit or a reject"):
|
|
36
37
|
await c.propose("judgment", "x", from_run=run)
|
|
37
38
|
|
|
@@ -46,13 +47,14 @@ async def test_refusal_is_a_tool_error(binary: str, records: str) -> None:
|
|
|
46
47
|
)
|
|
47
48
|
async def test_check_call(binary: str, records: str, tool: str, arguments: dict, action: str) -> None:
|
|
48
49
|
async with Client(records, binary) as c:
|
|
49
|
-
run = await c.record("ops-bot", "ran rm -rf /data", {"env": ["prod"]})
|
|
50
|
+
run = await c.record("ops-bot", "ran rm -rf /data", labels={"env": ["prod"]})
|
|
50
51
|
await c.judge(run, "reject", reason="never delete data")
|
|
51
52
|
proposed = await c.propose("judgment", "Never delete under /data", from_run=run, item_id="no-rm-data", veto=NO_RM)
|
|
52
|
-
await c.approve(proposed
|
|
53
|
+
approved = await c.approve(proposed.id, proposed.version, "ann")
|
|
53
54
|
|
|
54
55
|
decision = await c.check_call(tool, arguments)
|
|
55
56
|
|
|
57
|
+
assert approved.veto
|
|
56
58
|
assert decision.action == action
|
|
57
59
|
assert decision.allowed == (action == "allow")
|
|
58
60
|
|
|
@@ -60,13 +62,14 @@ async def test_check_call(binary: str, records: str, tool: str, arguments: dict,
|
|
|
60
62
|
async def test_server(server: tuple[str, str]) -> None:
|
|
61
63
|
url, key = server
|
|
62
64
|
async with Client(url=url, key=key) as c:
|
|
63
|
-
run = await c.record("support-bot", "Here are the refund steps", ACME)
|
|
65
|
+
run = await c.record("support-bot", "Here are the refund steps", labels=ACME)
|
|
64
66
|
await c.judge(run, "reject", reason="the refund window was missing")
|
|
65
67
|
proposed = await c.propose("judgment", "Quote the refund window", from_run=run, item_id="refund-window")
|
|
66
|
-
approved = await c.approve(proposed
|
|
68
|
+
approved = await c.approve(proposed.id, proposed.version, "mallory")
|
|
67
69
|
acme = await c.knowledge("support-bot", ACME)
|
|
68
70
|
|
|
69
|
-
assert approved
|
|
71
|
+
assert (approved.id, approved.version, approved.status, approved.approver) == ("refund-window", 1, "approved", "ann")
|
|
72
|
+
assert (proposed.id, proposed.status) == ("refund-window", "candidate")
|
|
70
73
|
assert [i.id for i in acme.items] == ["refund-window"]
|
|
71
74
|
|
|
72
75
|
|
|
@@ -75,3 +78,34 @@ async def test_server_refuses_a_wrong_key(server: tuple[str, str]) -> None:
|
|
|
75
78
|
with pytest.raises(MCPError):
|
|
76
79
|
async with Client(url=url, key="nl_wrong") as c:
|
|
77
80
|
await c.knowledge("support-bot", ACME)
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
async def test_record_takes_keywords_after_output() -> None:
|
|
84
|
+
with pytest.raises(TypeError):
|
|
85
|
+
await Client().record("support-bot", "answer", ACME) # type: ignore[misc]
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
async def test_approval_says_a_compaction_is_due(binary: str, records: str) -> None:
|
|
89
|
+
async with Client(records, binary) as c:
|
|
90
|
+
run = await c.record("support-bot", "Here are the refund steps", labels=ACME)
|
|
91
|
+
await c.judge(run, "reject", reason="the refund window was missing")
|
|
92
|
+
approvals = []
|
|
93
|
+
for n in range(6):
|
|
94
|
+
proposed = await c.propose("judgment", f"Refund rule number {n}", from_run=run, item_id=f"rule-{n}")
|
|
95
|
+
approvals.append(await c.approve(proposed.id, proposed.version, "ann"))
|
|
96
|
+
|
|
97
|
+
assert [a.compaction_due for a in approvals] == [False] * 5 + [True]
|
|
98
|
+
assert all(a.folder_error == "" for a in approvals)
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
async def test_approve_in_picked_labels(binary: str, records: str) -> None:
|
|
102
|
+
async with Client(records, binary) as c:
|
|
103
|
+
run = await c.record("support-bot", "Here are the refund steps", labels=ACME)
|
|
104
|
+
await c.record("support-bot", "ok", labels={"tenant": ["globex"]})
|
|
105
|
+
await c.judge(run, "reject", reason="too long", reason_code="form")
|
|
106
|
+
proposed = await c.propose("judgment", "Answer in three lines", from_run=run)
|
|
107
|
+
approved = await c.approve(proposed.id, proposed.version, "ann", labels={})
|
|
108
|
+
globex = await c.knowledge("support-bot", {"tenant": ["globex"]})
|
|
109
|
+
|
|
110
|
+
assert approved.status == "approved"
|
|
111
|
+
assert [i.id for i in globex.items] == [proposed.id]
|
|
@@ -337,7 +337,7 @@ wheels = [
|
|
|
337
337
|
|
|
338
338
|
[[package]]
|
|
339
339
|
name = "claude-agent-sdk"
|
|
340
|
-
version = "0.2.
|
|
340
|
+
version = "0.2.162"
|
|
341
341
|
source = { registry = "https://pypi.org/simple" }
|
|
342
342
|
dependencies = [
|
|
343
343
|
{ name = "anyio" },
|
|
@@ -346,12 +346,12 @@ dependencies = [
|
|
|
346
346
|
{ name = "sniffio" },
|
|
347
347
|
{ name = "typing-extensions", marker = "python_full_version < '3.11'" },
|
|
348
348
|
]
|
|
349
|
-
sdist = { url = "https://files.pythonhosted.org/packages/
|
|
349
|
+
sdist = { url = "https://files.pythonhosted.org/packages/78/a3/d31886d0723593228797c96eb54d07a19e561931a2771ed40b4a105bedc9/claude_agent_sdk-0.2.162.tar.gz", hash = "sha256:e4cf92f04f934d3b87d42be9ecbeb75c1a3d487a8dc8a299f7fed1ad9d0b476f", size = 365167, upload-time = "2026-09-29T20:06:54.593Z" }
|
|
350
350
|
wheels = [
|
|
351
|
-
{ url = "https://files.pythonhosted.org/packages/
|
|
352
|
-
{ url = "https://files.pythonhosted.org/packages/
|
|
353
|
-
{ url = "https://files.pythonhosted.org/packages/
|
|
354
|
-
{ url = "https://files.pythonhosted.org/packages/
|
|
351
|
+
{ url = "https://files.pythonhosted.org/packages/6d/b6/ec37e32cb0782977e80cd00338f5073f150d6ecfa568e9fa1a1dd5c9eed3/claude_agent_sdk-0.2.162-py3-none-macosx_11_0_arm64.whl", hash = "sha256:a0d5ad6185a8bcaaff05f5648036e2484cab28bd1c2ccc474fc8969555b386f7", size = 92700771, upload-time = "2026-09-29T20:06:59.661Z" },
|
|
352
|
+
{ url = "https://files.pythonhosted.org/packages/92/c4/1165688243cbf90d3532f509ea65bc337c3ba7f8229e26d7297e00a6ec2b/claude_agent_sdk-0.2.162-py3-none-macosx_11_0_x86_64.whl", hash = "sha256:faab97b03f04f9c3deb58ad85a0410404d40b6efb5547c9defc4fa986642fced", size = 97130408, upload-time = "2026-09-29T20:07:06.243Z" },
|
|
353
|
+
{ url = "https://files.pythonhosted.org/packages/05/41/f0addb831c3aefbc02c811ef7415e89e889d1bc2ae976eb935c508c91c0d/claude_agent_sdk-0.2.162-py3-none-manylinux_2_17_aarch64.whl", hash = "sha256:332cfb5eb9190fc8de1ad427753417d7c345264e9d2bd2ffa312afcc2f5cf766", size = 102183342, upload-time = "2026-09-29T20:07:14.279Z" },
|
|
354
|
+
{ url = "https://files.pythonhosted.org/packages/d2/33/8a7acd75405d0cfde96c26219dbbac050aa23416ceefd5f62cf7aecfc22d/claude_agent_sdk-0.2.162-py3-none-manylinux_2_17_x86_64.whl", hash = "sha256:673598c9916f5f4d03ff49ab3128f856f0c6508135e8e258838fc3b6f7aafa49", size = 102525631, upload-time = "2026-09-29T20:07:23.411Z" },
|
|
355
355
|
]
|
|
356
356
|
|
|
357
357
|
[[package]]
|
|
@@ -899,7 +899,7 @@ wheels = [
|
|
|
899
899
|
|
|
900
900
|
[[package]]
|
|
901
901
|
name = "nodloop"
|
|
902
|
-
version = "0.
|
|
902
|
+
version = "0.8.0"
|
|
903
903
|
source = { editable = "." }
|
|
904
904
|
dependencies = [
|
|
905
905
|
{ name = "mcp" },
|
nodloop-0.6.6/PKG-INFO
DELETED
|
@@ -1,13 +0,0 @@
|
|
|
1
|
-
Metadata-Version: 2.5
|
|
2
|
-
Name: nodloop
|
|
3
|
-
Version: 0.6.6
|
|
4
|
-
Summary: Record AI runs and the verdicts people give on them, and hand the knowledge they approve to the next run in the same place
|
|
5
|
-
License-Expression: MIT
|
|
6
|
-
Requires-Python: >=3.10
|
|
7
|
-
Requires-Dist: mcp>=2.0
|
|
8
|
-
Provides-Extra: claude-agent
|
|
9
|
-
Requires-Dist: claude-agent-sdk>=0.1; extra == 'claude-agent'
|
|
10
|
-
Provides-Extra: langchain
|
|
11
|
-
Requires-Dist: langchain>=1.0; extra == 'langchain'
|
|
12
|
-
Provides-Extra: openai-agents
|
|
13
|
-
Requires-Dist: openai-agents>=0.3.3; extra == 'openai-agents'
|
|
File without changes
|
|
File without changes
|