undoom 1.0.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- undoom-1.0.1/PKG-INFO +252 -0
- undoom-1.0.1/README.md +218 -0
- undoom-1.0.1/pyproject.toml +110 -0
- undoom-1.0.1/pyproject.toml.orig +86 -0
- undoom-1.0.1/src/undoom/__init__.py +15 -0
- undoom-1.0.1/src/undoom/__main__.py +4 -0
- undoom-1.0.1/src/undoom/api.py +127 -0
- undoom-1.0.1/src/undoom/canonical.py +31 -0
- undoom-1.0.1/src/undoom/capture/__init__.py +7 -0
- undoom-1.0.1/src/undoom/capture/anthropic.py +161 -0
- undoom-1.0.1/src/undoom/capture/context.py +116 -0
- undoom-1.0.1/src/undoom/capture/openai.py +97 -0
- undoom-1.0.1/src/undoom/cli.py +242 -0
- undoom-1.0.1/src/undoom/config.py +83 -0
- undoom-1.0.1/src/undoom/errors.py +21 -0
- undoom-1.0.1/src/undoom/models.py +206 -0
- undoom-1.0.1/src/undoom/openai_messages.py +87 -0
- undoom-1.0.1/src/undoom/otel/__init__.py +5 -0
- undoom-1.0.1/src/undoom/otel/import_trace.py +268 -0
- undoom-1.0.1/src/undoom/present.py +56 -0
- undoom-1.0.1/src/undoom/py.typed +0 -0
- undoom-1.0.1/src/undoom/redact.py +69 -0
- undoom-1.0.1/src/undoom/replay/__init__.py +5 -0
- undoom-1.0.1/src/undoom/replay/cassette.py +37 -0
- undoom-1.0.1/src/undoom/replay/changeset.py +63 -0
- undoom-1.0.1/src/undoom/replay/diff.py +69 -0
- undoom-1.0.1/src/undoom/replay/engine.py +227 -0
- undoom-1.0.1/src/undoom/replay/loading.py +26 -0
- undoom-1.0.1/src/undoom/replay/providers.py +72 -0
- undoom-1.0.1/src/undoom/store/__init__.py +7 -0
- undoom-1.0.1/src/undoom/store/base.py +21 -0
- undoom-1.0.1/src/undoom/store/migrate.py +65 -0
- undoom-1.0.1/src/undoom/store/postgres.py +178 -0
- undoom-1.0.1/src/undoom/store/resolve.py +33 -0
- undoom-1.0.1/src/undoom/store/sqlite.py +170 -0
- undoom-1.0.1/src/undoom/suite/__init__.py +7 -0
- undoom-1.0.1/src/undoom/suite/files.py +34 -0
- undoom-1.0.1/src/undoom/suite/promote.py +55 -0
- undoom-1.0.1/src/undoom/suite/report.py +61 -0
- undoom-1.0.1/src/undoom/suite/runner.py +86 -0
undoom-1.0.1/PKG-INFO
ADDED
|
@@ -0,0 +1,252 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: undoom
|
|
3
|
+
Version: 1.0.1
|
|
4
|
+
Summary: Turn production agent failures into regression tests that run on every change.
|
|
5
|
+
Keywords: agents,replay,regression,llm,tools
|
|
6
|
+
Author: Rupam Das
|
|
7
|
+
Author-email: Rupam Das <rejisterjack@gmail.com>
|
|
8
|
+
License-Expression: MIT
|
|
9
|
+
Classifier: Development Status :: 5 - Production/Stable
|
|
10
|
+
Classifier: Environment :: Console
|
|
11
|
+
Classifier: Intended Audience :: Developers
|
|
12
|
+
Classifier: Operating System :: OS Independent
|
|
13
|
+
Classifier: Programming Language :: Python :: 3
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
17
|
+
Classifier: Topic :: Software Development :: Testing
|
|
18
|
+
Requires-Dist: pydantic>=2.10
|
|
19
|
+
Requires-Dist: typer>=0.15
|
|
20
|
+
Requires-Dist: anthropic>=0.49 ; extra == 'anthropic'
|
|
21
|
+
Requires-Dist: openai>=1.68 ; extra == 'openai'
|
|
22
|
+
Requires-Dist: psycopg[binary]>=3.2 ; extra == 'postgres'
|
|
23
|
+
Requires-Dist: fastapi>=0.115 ; extra == 'ui'
|
|
24
|
+
Requires-Dist: uvicorn>=0.34 ; extra == 'ui'
|
|
25
|
+
Requires-Python: >=3.11
|
|
26
|
+
Project-URL: Homepage, https://undoom.rejisterjack.com
|
|
27
|
+
Project-URL: Repository, https://github.com/rejisterjack/undoom
|
|
28
|
+
Project-URL: Documentation, https://github.com/rejisterjack/undoom/blob/main/ARCHITECTURE.md
|
|
29
|
+
Provides-Extra: anthropic
|
|
30
|
+
Provides-Extra: openai
|
|
31
|
+
Provides-Extra: postgres
|
|
32
|
+
Provides-Extra: ui
|
|
33
|
+
Description-Content-Type: text/markdown
|
|
34
|
+
|
|
35
|
+
# Undoom
|
|
36
|
+
|
|
37
|
+
Turn a production agent failure into a regression test that runs on every change.
|
|
38
|
+
|
|
39
|
+
Observability shows that a tool-calling agent failed. An eval platform scores a dataset. Undoom does the step after that: replay the failed run with one change, mock the tools from the recording, and promote a fixed replay into a test your CI can run. It is not a monitor, and it is not a general benchmark suite.
|
|
40
|
+
|
|
41
|
+
Undoom sits next to [Keywarden](https://github.com/rejisterjack/keywarden) (what an agent is allowed to do) and [Boxwren](https://github.com/rejisterjack/boxwren) (where untrusted code runs). Use those when the question is permission or isolation. Use Undoom when the question is whether a known failure can come back.
|
|
42
|
+
|
|
43
|
+
## The loop
|
|
44
|
+
|
|
45
|
+
1. Record a run: prompts, model calls, tool calls, and tool results.
|
|
46
|
+
2. Mark it failed, and name the tool calls that should not have happened.
|
|
47
|
+
3. Replay with exactly one change: a prompt, a model id, or one tool implementation.
|
|
48
|
+
4. Every other tool returns the recorded result. Undoom does not call it.
|
|
49
|
+
5. Promote a fixed replay. `undoom test` fails if that failure shows up again.
|
|
50
|
+
|
|
51
|
+
Live model output is not bit-identical. Deterministic here means the tool I/O is frozen. Replay calls the model again at temperature 0. A cassette hit is the pair `(tool name, canonical JSON arguments)`. Different key order does not count as a new call. A miss stops the replay. Undoom will not fall through to the live tool.
|
|
52
|
+
|
|
53
|
+
A tool override runs your function, with your privileges, the same way your test suite does. Do not point it at a tool you would not run locally.
|
|
54
|
+
|
|
55
|
+
## Install
|
|
56
|
+
|
|
57
|
+
Python 3.11 or newer.
|
|
58
|
+
|
|
59
|
+
```bash
|
|
60
|
+
pip install undoom
|
|
61
|
+
pip install 'undoom[openai]' # live replay against the OpenAI API
|
|
62
|
+
pip install 'undoom[anthropic]' # Anthropic Messages capture helper
|
|
63
|
+
pip install 'undoom[postgres]' # shared trace log
|
|
64
|
+
pip install 'undoom[ui]' # local diff UI
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
Traces default to `.undoom/traces.sqlite`. Case files live in `.undoom/cases/` and are meant to be committed. Keep the database out of git. A useful `.gitignore`:
|
|
68
|
+
|
|
69
|
+
```gitignore
|
|
70
|
+
.undoom/*
|
|
71
|
+
!.undoom/cases/
|
|
72
|
+
!.undoom/cases/**
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
Traces can contain customer text. Undoom redacts common API-key and `Authorization` shapes before writing. It cannot know every secret. Treat the sqlite file, and any case you commit, as sensitive. See [SECURITY.md](SECURITY.md).
|
|
76
|
+
|
|
77
|
+
## Record a run
|
|
78
|
+
|
|
79
|
+
```python
|
|
80
|
+
import undoom
|
|
81
|
+
from openai import OpenAI
|
|
82
|
+
|
|
83
|
+
undoom.configure(agent="support") # sqlite at .undoom/traces.sqlite
|
|
84
|
+
|
|
85
|
+
def lookup_order(order_id: str) -> dict:
|
|
86
|
+
return {"order_id": order_id, "status": "shipped"}
|
|
87
|
+
|
|
88
|
+
def refund_order(order_id: str) -> dict:
|
|
89
|
+
return {"order_id": order_id, "status": "refunded"}
|
|
90
|
+
|
|
91
|
+
with undoom.run(name="ticket-1042") as recorder:
|
|
92
|
+
client = undoom.wrap_openai(
|
|
93
|
+
OpenAI(),
|
|
94
|
+
recorder,
|
|
95
|
+
tools={"lookup_order": lookup_order, "refund_order": refund_order},
|
|
96
|
+
)
|
|
97
|
+
client.chat.completions.create(
|
|
98
|
+
model="gpt-4.1",
|
|
99
|
+
temperature=0,
|
|
100
|
+
messages=[
|
|
101
|
+
{"role": "system", "content": open("prompts/support.txt").read()},
|
|
102
|
+
{"role": "user", "content": "Where is order 1042?"},
|
|
103
|
+
],
|
|
104
|
+
tools=[...], # the same schema you pass to the API
|
|
105
|
+
)
|
|
106
|
+
if any(step.kind == "tool" and step.name == "refund_order" for step in recorder.run.steps):
|
|
107
|
+
recorder.fail(
|
|
108
|
+
"refunded without an explicit request",
|
|
109
|
+
unexpected_tools=["refund_order"],
|
|
110
|
+
)
|
|
111
|
+
```
|
|
112
|
+
|
|
113
|
+
`unexpected_tools` is the failure signature. Promote turns those names into the check `undoom test` enforces. Without them, Undoom treats tools that disappeared in the fixed replay as the signature.
|
|
114
|
+
|
|
115
|
+
`undoom.wrap_anthropic(client, recorder, tools=...)` does the same for Anthropic's `messages.create` API.
|
|
116
|
+
|
|
117
|
+
You can record a model call and a tool yourself when you are not on those SDKs. `recorder.record_model(...)` and `with recorder.tool(name, arguments) as handle: handle.record(result)` are the contract. Adapters are thin callers of that API.
|
|
118
|
+
|
|
119
|
+
```bash
|
|
120
|
+
undoom runs list
|
|
121
|
+
undoom runs show <run-id>
|
|
122
|
+
undoom runs fail <run-id> --reason "refunded without an explicit request" --unexpected-tool refund_order
|
|
123
|
+
```
|
|
124
|
+
|
|
125
|
+
## Import an OpenTelemetry trace
|
|
126
|
+
|
|
127
|
+
When a stack already exports GenAI spans, you can turn one OTLP/JSON document into a stored run:
|
|
128
|
+
|
|
129
|
+
```bash
|
|
130
|
+
undoom import trace.json --agent support --name ticket-1042
|
|
131
|
+
undoom import trace.json --trace-id <id> # required when the file holds more than one trace
|
|
132
|
+
```
|
|
133
|
+
|
|
134
|
+
The importer only accepts tool spans that include both `gen_ai.tool.call.arguments` and a result or error. GenAI conventions mark those fields as opt-in; a span that omits them cannot fill a cassette, so Undoom refuses the import instead of inventing values. Mark the failure afterward with `undoom runs fail` the same way you would for a native capture.
|
|
135
|
+
|
|
136
|
+
## Replay one change
|
|
137
|
+
|
|
138
|
+
```bash
|
|
139
|
+
undoom replay <run-id> --prompt-file prompts/support.txt
|
|
140
|
+
undoom replay <run-id> --model gpt-4.1-mini
|
|
141
|
+
undoom replay <run-id> --tool lookup_order=myapp.tools:lookup_order
|
|
142
|
+
```
|
|
143
|
+
|
|
144
|
+
A command that sets more than one of prompt, model, and tool is rejected. The replay id and verdict are printed. Verdicts:
|
|
145
|
+
|
|
146
|
+
| Verdict | Meaning |
|
|
147
|
+
| --- | --- |
|
|
148
|
+
| `fixed` | The failure signature did not recur |
|
|
149
|
+
| `still_failing` | The same unexpected tool or tool error came back |
|
|
150
|
+
| `diverged` | The model called a tool the recording cannot answer |
|
|
151
|
+
| `error` | The provider or the override function could not be loaded |
|
|
152
|
+
|
|
153
|
+
Final assistant text is shown for comparison. It is not the pass/fail check.
|
|
154
|
+
|
|
155
|
+
## Promote and test
|
|
156
|
+
|
|
157
|
+
```bash
|
|
158
|
+
undoom promote <replay-id> --name no-unsolicited-refund
|
|
159
|
+
undoom promote <replay-id> --name no-unsolicited-refund --expect-tools
|
|
160
|
+
undoom test
|
|
161
|
+
undoom test --format junit
|
|
162
|
+
```
|
|
163
|
+
|
|
164
|
+
`--expect-tools` also locks the successful tool sequence. The default check is only that the original failure does not recur.
|
|
165
|
+
|
|
166
|
+
Prompt and tool changes point at files in your repo. `undoom test` reads those paths again, so editing the prompt is what CI evaluates. The case file embeds the tool cassette, which means CI does not need the sqlite history.
|
|
167
|
+
|
|
168
|
+
```yaml
|
|
169
|
+
# .github/workflows/undoom.yml
|
|
170
|
+
name: undoom
|
|
171
|
+
on: [push, pull_request]
|
|
172
|
+
jobs:
|
|
173
|
+
regressions:
|
|
174
|
+
runs-on: ubuntu-latest
|
|
175
|
+
steps:
|
|
176
|
+
- uses: actions/checkout@v4
|
|
177
|
+
- uses: actions/setup-python@v5
|
|
178
|
+
with:
|
|
179
|
+
python-version: "3.12"
|
|
180
|
+
- run: pip install undoom
|
|
181
|
+
- run: undoom test --format junit
|
|
182
|
+
env:
|
|
183
|
+
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
|
|
184
|
+
```
|
|
185
|
+
|
|
186
|
+
Set `UNDOOM_MODEL_PROVIDER=openai` (the default) for live replay. A provider can also be `module:factory`, a zero-argument function that returns an object with `complete(...)`.
|
|
187
|
+
|
|
188
|
+
## Offline demo
|
|
189
|
+
|
|
190
|
+
The order-lookup story ships in `examples/order_lookup/`. A buggy prompt refunds order 1042 after looking it up. The fixed prompt refunds only when the user asks. No API key is required.
|
|
191
|
+
|
|
192
|
+
```bash
|
|
193
|
+
mkdir -p /tmp/undoom-demo
|
|
194
|
+
UNDOOM_DEMO_ROOT=/tmp/undoom-demo python examples/order_lookup/demo.py
|
|
195
|
+
cd /tmp/undoom-demo
|
|
196
|
+
PYTHONPATH=/path/to/undoom \
|
|
197
|
+
UNDOOM_MODEL_PROVIDER=examples.order_lookup.fake:provider \
|
|
198
|
+
undoom test
|
|
199
|
+
```
|
|
200
|
+
|
|
201
|
+
Put the refund instruction back into `prompts/fixed.txt` and `undoom test` fails on `refund_order`.
|
|
202
|
+
|
|
203
|
+
Docker runs that story and leaves the case file on a volume:
|
|
204
|
+
|
|
205
|
+
```bash
|
|
206
|
+
docker compose -f docker/compose.yml up --build demo
|
|
207
|
+
```
|
|
208
|
+
|
|
209
|
+
Postgres, for a shared trace log rather than for CI, is the `postgres` profile in the same file. Point the CLI at it with `UNDOOM_STORE=postgresql://undoom:undoom@localhost:5432/undoom`.
|
|
210
|
+
|
|
211
|
+
## Local diff UI
|
|
212
|
+
|
|
213
|
+
```bash
|
|
214
|
+
pip install 'undoom[ui]'
|
|
215
|
+
undoom serve
|
|
216
|
+
```
|
|
217
|
+
|
|
218
|
+
The API listens on `http://127.0.0.1:8787`. From `apps/ui`:
|
|
219
|
+
|
|
220
|
+
```bash
|
|
221
|
+
npm install
|
|
222
|
+
NEXT_PUBLIC_UNDOOM_API=http://127.0.0.1:8787 npm run dev
|
|
223
|
+
```
|
|
224
|
+
|
|
225
|
+
The hosted demo at [undoom.rejisterjack.com](https://undoom.rejisterjack.com) is the same UI built with `NEXT_PUBLIC_DEMO=1`. It reads committed synthetic fixtures only. There is no write API and no customer data on that deployment. On Vercel, set the project root to `apps/ui` and `NEXT_PUBLIC_DEMO` to `1`.
|
|
226
|
+
|
|
227
|
+
## Store
|
|
228
|
+
|
|
229
|
+
| `UNDOOM_STORE` | Store |
|
|
230
|
+
| --- | --- |
|
|
231
|
+
| unset | `.undoom/traces.sqlite` |
|
|
232
|
+
| a filesystem path | SQLite at that path |
|
|
233
|
+
| `sqlite:///relative.sqlite` | SQLite |
|
|
234
|
+
| `postgresql://...` | Postgres (`undoom[postgres]`) |
|
|
235
|
+
|
|
236
|
+
Documents are JSON with `schema_version`. This release reads version 1 and rejects anything else.
|
|
237
|
+
|
|
238
|
+
## Development
|
|
239
|
+
|
|
240
|
+
```bash
|
|
241
|
+
uv sync --all-extras --group dev
|
|
242
|
+
uv run ruff check src tests examples
|
|
243
|
+
uv run ruff format --check src tests examples
|
|
244
|
+
uv run mypy src
|
|
245
|
+
uv run pytest
|
|
246
|
+
```
|
|
247
|
+
|
|
248
|
+
`UNDOOM_TEST_POSTGRES` opts into the Postgres integration test. The default suite never calls the network.
|
|
249
|
+
|
|
250
|
+
## What this is not
|
|
251
|
+
|
|
252
|
+
Undoom does not replace production monitoring. It does not decide which tools an agent may call, and it does not sandbox code. Those are Keywarden and Boxwren. It is not a place to host a general LLM benchmark. The architecture note in [ARCHITECTURE.md](ARCHITECTURE.md) is the longer version of that boundary.
|
undoom-1.0.1/README.md
ADDED
|
@@ -0,0 +1,218 @@
|
|
|
1
|
+
# Undoom
|
|
2
|
+
|
|
3
|
+
Turn a production agent failure into a regression test that runs on every change.
|
|
4
|
+
|
|
5
|
+
Observability shows that a tool-calling agent failed. An eval platform scores a dataset. Undoom does the step after that: replay the failed run with one change, mock the tools from the recording, and promote a fixed replay into a test your CI can run. It is not a monitor, and it is not a general benchmark suite.
|
|
6
|
+
|
|
7
|
+
Undoom sits next to [Keywarden](https://github.com/rejisterjack/keywarden) (what an agent is allowed to do) and [Boxwren](https://github.com/rejisterjack/boxwren) (where untrusted code runs). Use those when the question is permission or isolation. Use Undoom when the question is whether a known failure can come back.
|
|
8
|
+
|
|
9
|
+
## The loop
|
|
10
|
+
|
|
11
|
+
1. Record a run: prompts, model calls, tool calls, and tool results.
|
|
12
|
+
2. Mark it failed, and name the tool calls that should not have happened.
|
|
13
|
+
3. Replay with exactly one change: a prompt, a model id, or one tool implementation.
|
|
14
|
+
4. Every other tool returns the recorded result. Undoom does not call it.
|
|
15
|
+
5. Promote a fixed replay. `undoom test` fails if that failure shows up again.
|
|
16
|
+
|
|
17
|
+
Live model output is not bit-identical. Deterministic here means the tool I/O is frozen. Replay calls the model again at temperature 0. A cassette hit is the pair `(tool name, canonical JSON arguments)`. Different key order does not count as a new call. A miss stops the replay. Undoom will not fall through to the live tool.
|
|
18
|
+
|
|
19
|
+
A tool override runs your function, with your privileges, the same way your test suite does. Do not point it at a tool you would not run locally.
|
|
20
|
+
|
|
21
|
+
## Install
|
|
22
|
+
|
|
23
|
+
Python 3.11 or newer.
|
|
24
|
+
|
|
25
|
+
```bash
|
|
26
|
+
pip install undoom
|
|
27
|
+
pip install 'undoom[openai]' # live replay against the OpenAI API
|
|
28
|
+
pip install 'undoom[anthropic]' # Anthropic Messages capture helper
|
|
29
|
+
pip install 'undoom[postgres]' # shared trace log
|
|
30
|
+
pip install 'undoom[ui]' # local diff UI
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
Traces default to `.undoom/traces.sqlite`. Case files live in `.undoom/cases/` and are meant to be committed. Keep the database out of git. A useful `.gitignore`:
|
|
34
|
+
|
|
35
|
+
```gitignore
|
|
36
|
+
.undoom/*
|
|
37
|
+
!.undoom/cases/
|
|
38
|
+
!.undoom/cases/**
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
Traces can contain customer text. Undoom redacts common API-key and `Authorization` shapes before writing. It cannot know every secret. Treat the sqlite file, and any case you commit, as sensitive. See [SECURITY.md](SECURITY.md).
|
|
42
|
+
|
|
43
|
+
## Record a run
|
|
44
|
+
|
|
45
|
+
```python
|
|
46
|
+
import undoom
|
|
47
|
+
from openai import OpenAI
|
|
48
|
+
|
|
49
|
+
undoom.configure(agent="support") # sqlite at .undoom/traces.sqlite
|
|
50
|
+
|
|
51
|
+
def lookup_order(order_id: str) -> dict:
|
|
52
|
+
return {"order_id": order_id, "status": "shipped"}
|
|
53
|
+
|
|
54
|
+
def refund_order(order_id: str) -> dict:
|
|
55
|
+
return {"order_id": order_id, "status": "refunded"}
|
|
56
|
+
|
|
57
|
+
with undoom.run(name="ticket-1042") as recorder:
|
|
58
|
+
client = undoom.wrap_openai(
|
|
59
|
+
OpenAI(),
|
|
60
|
+
recorder,
|
|
61
|
+
tools={"lookup_order": lookup_order, "refund_order": refund_order},
|
|
62
|
+
)
|
|
63
|
+
client.chat.completions.create(
|
|
64
|
+
model="gpt-4.1",
|
|
65
|
+
temperature=0,
|
|
66
|
+
messages=[
|
|
67
|
+
{"role": "system", "content": open("prompts/support.txt").read()},
|
|
68
|
+
{"role": "user", "content": "Where is order 1042?"},
|
|
69
|
+
],
|
|
70
|
+
tools=[...], # the same schema you pass to the API
|
|
71
|
+
)
|
|
72
|
+
if any(step.kind == "tool" and step.name == "refund_order" for step in recorder.run.steps):
|
|
73
|
+
recorder.fail(
|
|
74
|
+
"refunded without an explicit request",
|
|
75
|
+
unexpected_tools=["refund_order"],
|
|
76
|
+
)
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
`unexpected_tools` is the failure signature. Promote turns those names into the check `undoom test` enforces. Without them, Undoom treats tools that disappeared in the fixed replay as the signature.
|
|
80
|
+
|
|
81
|
+
`undoom.wrap_anthropic(client, recorder, tools=...)` does the same for Anthropic's `messages.create` API.
|
|
82
|
+
|
|
83
|
+
You can record a model call and a tool yourself when you are not on those SDKs. `recorder.record_model(...)` and `with recorder.tool(name, arguments) as handle: handle.record(result)` are the contract. Adapters are thin callers of that API.
|
|
84
|
+
|
|
85
|
+
```bash
|
|
86
|
+
undoom runs list
|
|
87
|
+
undoom runs show <run-id>
|
|
88
|
+
undoom runs fail <run-id> --reason "refunded without an explicit request" --unexpected-tool refund_order
|
|
89
|
+
```
|
|
90
|
+
|
|
91
|
+
## Import an OpenTelemetry trace
|
|
92
|
+
|
|
93
|
+
When a stack already exports GenAI spans, you can turn one OTLP/JSON document into a stored run:
|
|
94
|
+
|
|
95
|
+
```bash
|
|
96
|
+
undoom import trace.json --agent support --name ticket-1042
|
|
97
|
+
undoom import trace.json --trace-id <id> # required when the file holds more than one trace
|
|
98
|
+
```
|
|
99
|
+
|
|
100
|
+
The importer only accepts tool spans that include both `gen_ai.tool.call.arguments` and a result or error. GenAI conventions mark those fields as opt-in; a span that omits them cannot fill a cassette, so Undoom refuses the import instead of inventing values. Mark the failure afterward with `undoom runs fail` the same way you would for a native capture.
|
|
101
|
+
|
|
102
|
+
## Replay one change
|
|
103
|
+
|
|
104
|
+
```bash
|
|
105
|
+
undoom replay <run-id> --prompt-file prompts/support.txt
|
|
106
|
+
undoom replay <run-id> --model gpt-4.1-mini
|
|
107
|
+
undoom replay <run-id> --tool lookup_order=myapp.tools:lookup_order
|
|
108
|
+
```
|
|
109
|
+
|
|
110
|
+
A command that sets more than one of prompt, model, and tool is rejected. The replay id and verdict are printed. Verdicts:
|
|
111
|
+
|
|
112
|
+
| Verdict | Meaning |
|
|
113
|
+
| --- | --- |
|
|
114
|
+
| `fixed` | The failure signature did not recur |
|
|
115
|
+
| `still_failing` | The same unexpected tool or tool error came back |
|
|
116
|
+
| `diverged` | The model called a tool the recording cannot answer |
|
|
117
|
+
| `error` | The provider or the override function could not be loaded |
|
|
118
|
+
|
|
119
|
+
Final assistant text is shown for comparison. It is not the pass/fail check.
|
|
120
|
+
|
|
121
|
+
## Promote and test
|
|
122
|
+
|
|
123
|
+
```bash
|
|
124
|
+
undoom promote <replay-id> --name no-unsolicited-refund
|
|
125
|
+
undoom promote <replay-id> --name no-unsolicited-refund --expect-tools
|
|
126
|
+
undoom test
|
|
127
|
+
undoom test --format junit
|
|
128
|
+
```
|
|
129
|
+
|
|
130
|
+
`--expect-tools` also locks the successful tool sequence. The default check is only that the original failure does not recur.
|
|
131
|
+
|
|
132
|
+
Prompt and tool changes point at files in your repo. `undoom test` reads those paths again, so editing the prompt is what CI evaluates. The case file embeds the tool cassette, which means CI does not need the sqlite history.
|
|
133
|
+
|
|
134
|
+
```yaml
|
|
135
|
+
# .github/workflows/undoom.yml
|
|
136
|
+
name: undoom
|
|
137
|
+
on: [push, pull_request]
|
|
138
|
+
jobs:
|
|
139
|
+
regressions:
|
|
140
|
+
runs-on: ubuntu-latest
|
|
141
|
+
steps:
|
|
142
|
+
- uses: actions/checkout@v4
|
|
143
|
+
- uses: actions/setup-python@v5
|
|
144
|
+
with:
|
|
145
|
+
python-version: "3.12"
|
|
146
|
+
- run: pip install undoom
|
|
147
|
+
- run: undoom test --format junit
|
|
148
|
+
env:
|
|
149
|
+
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
|
|
150
|
+
```
|
|
151
|
+
|
|
152
|
+
Set `UNDOOM_MODEL_PROVIDER=openai` (the default) for live replay. A provider can also be `module:factory`, a zero-argument function that returns an object with `complete(...)`.
|
|
153
|
+
|
|
154
|
+
## Offline demo
|
|
155
|
+
|
|
156
|
+
The order-lookup story ships in `examples/order_lookup/`. A buggy prompt refunds order 1042 after looking it up. The fixed prompt refunds only when the user asks. No API key is required.
|
|
157
|
+
|
|
158
|
+
```bash
|
|
159
|
+
mkdir -p /tmp/undoom-demo
|
|
160
|
+
UNDOOM_DEMO_ROOT=/tmp/undoom-demo python examples/order_lookup/demo.py
|
|
161
|
+
cd /tmp/undoom-demo
|
|
162
|
+
PYTHONPATH=/path/to/undoom \
|
|
163
|
+
UNDOOM_MODEL_PROVIDER=examples.order_lookup.fake:provider \
|
|
164
|
+
undoom test
|
|
165
|
+
```
|
|
166
|
+
|
|
167
|
+
Put the refund instruction back into `prompts/fixed.txt` and `undoom test` fails on `refund_order`.
|
|
168
|
+
|
|
169
|
+
Docker runs that story and leaves the case file on a volume:
|
|
170
|
+
|
|
171
|
+
```bash
|
|
172
|
+
docker compose -f docker/compose.yml up --build demo
|
|
173
|
+
```
|
|
174
|
+
|
|
175
|
+
Postgres, for a shared trace log rather than for CI, is the `postgres` profile in the same file. Point the CLI at it with `UNDOOM_STORE=postgresql://undoom:undoom@localhost:5432/undoom`.
|
|
176
|
+
|
|
177
|
+
## Local diff UI
|
|
178
|
+
|
|
179
|
+
```bash
|
|
180
|
+
pip install 'undoom[ui]'
|
|
181
|
+
undoom serve
|
|
182
|
+
```
|
|
183
|
+
|
|
184
|
+
The API listens on `http://127.0.0.1:8787`. From `apps/ui`:
|
|
185
|
+
|
|
186
|
+
```bash
|
|
187
|
+
npm install
|
|
188
|
+
NEXT_PUBLIC_UNDOOM_API=http://127.0.0.1:8787 npm run dev
|
|
189
|
+
```
|
|
190
|
+
|
|
191
|
+
The hosted demo at [undoom.rejisterjack.com](https://undoom.rejisterjack.com) is the same UI built with `NEXT_PUBLIC_DEMO=1`. It reads committed synthetic fixtures only. There is no write API and no customer data on that deployment. On Vercel, set the project root to `apps/ui` and `NEXT_PUBLIC_DEMO` to `1`.
|
|
192
|
+
|
|
193
|
+
## Store
|
|
194
|
+
|
|
195
|
+
| `UNDOOM_STORE` | Store |
|
|
196
|
+
| --- | --- |
|
|
197
|
+
| unset | `.undoom/traces.sqlite` |
|
|
198
|
+
| a filesystem path | SQLite at that path |
|
|
199
|
+
| `sqlite:///relative.sqlite` | SQLite |
|
|
200
|
+
| `postgresql://...` | Postgres (`undoom[postgres]`) |
|
|
201
|
+
|
|
202
|
+
Documents are JSON with `schema_version`. This release reads version 1 and rejects anything else.
|
|
203
|
+
|
|
204
|
+
## Development
|
|
205
|
+
|
|
206
|
+
```bash
|
|
207
|
+
uv sync --all-extras --group dev
|
|
208
|
+
uv run ruff check src tests examples
|
|
209
|
+
uv run ruff format --check src tests examples
|
|
210
|
+
uv run mypy src
|
|
211
|
+
uv run pytest
|
|
212
|
+
```
|
|
213
|
+
|
|
214
|
+
`UNDOOM_TEST_POSTGRES` opts into the Postgres integration test. The default suite never calls the network.
|
|
215
|
+
|
|
216
|
+
## What this is not
|
|
217
|
+
|
|
218
|
+
Undoom does not replace production monitoring. It does not decide which tools an agent may call, and it does not sandbox code. Those are Keywarden and Boxwren. It is not a place to host a general LLM benchmark. The architecture note in [ARCHITECTURE.md](ARCHITECTURE.md) is the longer version of that boundary.
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "undoom"
|
|
3
|
+
version = "1.0.1"
|
|
4
|
+
description = "Turn production agent failures into regression tests that run on every change."
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
license = "MIT"
|
|
7
|
+
requires-python = ">=3.11"
|
|
8
|
+
dependencies = [
|
|
9
|
+
"pydantic>=2.10",
|
|
10
|
+
"typer>=0.15",
|
|
11
|
+
]
|
|
12
|
+
classifiers = [
|
|
13
|
+
"Development Status :: 5 - Production/Stable",
|
|
14
|
+
"Environment :: Console",
|
|
15
|
+
"Intended Audience :: Developers",
|
|
16
|
+
"Operating System :: OS Independent",
|
|
17
|
+
"Programming Language :: Python :: 3",
|
|
18
|
+
"Programming Language :: Python :: 3.11",
|
|
19
|
+
"Programming Language :: Python :: 3.12",
|
|
20
|
+
"Programming Language :: Python :: 3.13",
|
|
21
|
+
"Topic :: Software Development :: Testing",
|
|
22
|
+
]
|
|
23
|
+
keywords = [
|
|
24
|
+
"agents",
|
|
25
|
+
"replay",
|
|
26
|
+
"regression",
|
|
27
|
+
"llm",
|
|
28
|
+
"tools",
|
|
29
|
+
]
|
|
30
|
+
|
|
31
|
+
[[project.authors]]
|
|
32
|
+
name = "Rupam Das"
|
|
33
|
+
email = "rejisterjack@gmail.com"
|
|
34
|
+
|
|
35
|
+
[project.optional-dependencies]
|
|
36
|
+
openai = ["openai>=1.68"]
|
|
37
|
+
anthropic = ["anthropic>=0.49"]
|
|
38
|
+
postgres = ["psycopg[binary]>=3.2"]
|
|
39
|
+
ui = [
|
|
40
|
+
"fastapi>=0.115",
|
|
41
|
+
"uvicorn>=0.34",
|
|
42
|
+
]
|
|
43
|
+
|
|
44
|
+
[project.urls]
|
|
45
|
+
Homepage = "https://undoom.rejisterjack.com"
|
|
46
|
+
Repository = "https://github.com/rejisterjack/undoom"
|
|
47
|
+
Documentation = "https://github.com/rejisterjack/undoom/blob/main/ARCHITECTURE.md"
|
|
48
|
+
|
|
49
|
+
[project.scripts]
|
|
50
|
+
undoom = "undoom.cli:main"
|
|
51
|
+
|
|
52
|
+
[dependency-groups]
|
|
53
|
+
dev = [
|
|
54
|
+
"pytest>=8.3",
|
|
55
|
+
"ruff>=0.11",
|
|
56
|
+
"mypy>=1.15",
|
|
57
|
+
"httpx>=0.28",
|
|
58
|
+
]
|
|
59
|
+
|
|
60
|
+
[build-system]
|
|
61
|
+
requires = ["uv_build>=0.13.0,<0.14.0"]
|
|
62
|
+
build-backend = "uv_build"
|
|
63
|
+
|
|
64
|
+
[tool.uv]
|
|
65
|
+
package = true
|
|
66
|
+
|
|
67
|
+
[tool.pytest.ini_options]
|
|
68
|
+
testpaths = ["tests"]
|
|
69
|
+
pythonpath = ["."]
|
|
70
|
+
markers = ["postgres: integration test that needs UNDOOM_TEST_POSTGRES"]
|
|
71
|
+
|
|
72
|
+
[tool.ruff]
|
|
73
|
+
line-length = 100
|
|
74
|
+
target-version = "py311"
|
|
75
|
+
src = [
|
|
76
|
+
"src",
|
|
77
|
+
"tests",
|
|
78
|
+
"examples",
|
|
79
|
+
]
|
|
80
|
+
|
|
81
|
+
[tool.ruff.lint]
|
|
82
|
+
select = [
|
|
83
|
+
"E",
|
|
84
|
+
"F",
|
|
85
|
+
"I",
|
|
86
|
+
"UP",
|
|
87
|
+
"B",
|
|
88
|
+
]
|
|
89
|
+
ignore = [
|
|
90
|
+
"B008",
|
|
91
|
+
"E501",
|
|
92
|
+
]
|
|
93
|
+
|
|
94
|
+
[tool.ruff.lint.per-file-ignores]
|
|
95
|
+
"examples/order_lookup/demo.py" = ["E402"]
|
|
96
|
+
|
|
97
|
+
[tool.ruff.lint.isort]
|
|
98
|
+
known-first-party = [
|
|
99
|
+
"undoom",
|
|
100
|
+
"examples",
|
|
101
|
+
]
|
|
102
|
+
|
|
103
|
+
[tool.mypy]
|
|
104
|
+
python_version = "3.11"
|
|
105
|
+
packages = ["undoom"]
|
|
106
|
+
disallow_untyped_defs = true
|
|
107
|
+
warn_redundant_casts = true
|
|
108
|
+
warn_unused_ignores = true
|
|
109
|
+
no_implicit_optional = true
|
|
110
|
+
check_untyped_defs = true
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "undoom"
|
|
3
|
+
version = "1.0.1"
|
|
4
|
+
description = "Turn production agent failures into regression tests that run on every change."
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
license = "MIT"
|
|
7
|
+
authors = [
|
|
8
|
+
{ name = "Rupam Das", email = "rejisterjack@gmail.com" }
|
|
9
|
+
]
|
|
10
|
+
requires-python = ">=3.11"
|
|
11
|
+
dependencies = [
|
|
12
|
+
"pydantic>=2.10",
|
|
13
|
+
"typer>=0.15",
|
|
14
|
+
]
|
|
15
|
+
classifiers = [
|
|
16
|
+
"Development Status :: 5 - Production/Stable",
|
|
17
|
+
"Environment :: Console",
|
|
18
|
+
"Intended Audience :: Developers",
|
|
19
|
+
"Operating System :: OS Independent",
|
|
20
|
+
"Programming Language :: Python :: 3",
|
|
21
|
+
"Programming Language :: Python :: 3.11",
|
|
22
|
+
"Programming Language :: Python :: 3.12",
|
|
23
|
+
"Programming Language :: Python :: 3.13",
|
|
24
|
+
"Topic :: Software Development :: Testing",
|
|
25
|
+
]
|
|
26
|
+
keywords = ["agents", "replay", "regression", "llm", "tools"]
|
|
27
|
+
|
|
28
|
+
[project.optional-dependencies]
|
|
29
|
+
openai = ["openai>=1.68"]
|
|
30
|
+
anthropic = ["anthropic>=0.49"]
|
|
31
|
+
postgres = ["psycopg[binary]>=3.2"]
|
|
32
|
+
ui = ["fastapi>=0.115", "uvicorn>=0.34"]
|
|
33
|
+
|
|
34
|
+
[project.urls]
|
|
35
|
+
Homepage = "https://undoom.rejisterjack.com"
|
|
36
|
+
Repository = "https://github.com/rejisterjack/undoom"
|
|
37
|
+
Documentation = "https://github.com/rejisterjack/undoom/blob/main/ARCHITECTURE.md"
|
|
38
|
+
|
|
39
|
+
[project.scripts]
|
|
40
|
+
undoom = "undoom.cli:main"
|
|
41
|
+
|
|
42
|
+
[dependency-groups]
|
|
43
|
+
dev = [
|
|
44
|
+
"pytest>=8.3",
|
|
45
|
+
"ruff>=0.11",
|
|
46
|
+
"mypy>=1.15",
|
|
47
|
+
"httpx>=0.28",
|
|
48
|
+
]
|
|
49
|
+
|
|
50
|
+
[build-system]
|
|
51
|
+
requires = ["uv_build>=0.13.0,<0.14.0"]
|
|
52
|
+
build-backend = "uv_build"
|
|
53
|
+
|
|
54
|
+
[tool.uv]
|
|
55
|
+
package = true
|
|
56
|
+
|
|
57
|
+
[tool.pytest.ini_options]
|
|
58
|
+
testpaths = ["tests"]
|
|
59
|
+
pythonpath = ["."]
|
|
60
|
+
markers = [
|
|
61
|
+
"postgres: integration test that needs UNDOOM_TEST_POSTGRES",
|
|
62
|
+
]
|
|
63
|
+
|
|
64
|
+
[tool.ruff]
|
|
65
|
+
line-length = 100
|
|
66
|
+
target-version = "py311"
|
|
67
|
+
src = ["src", "tests", "examples"]
|
|
68
|
+
|
|
69
|
+
[tool.ruff.lint]
|
|
70
|
+
select = ["E", "F", "I", "UP", "B"]
|
|
71
|
+
ignore = ["B008", "E501"]
|
|
72
|
+
|
|
73
|
+
[tool.ruff.lint.per-file-ignores]
|
|
74
|
+
"examples/order_lookup/demo.py" = ["E402"]
|
|
75
|
+
|
|
76
|
+
[tool.ruff.lint.isort]
|
|
77
|
+
known-first-party = ["undoom", "examples"]
|
|
78
|
+
|
|
79
|
+
[tool.mypy]
|
|
80
|
+
python_version = "3.11"
|
|
81
|
+
packages = ["undoom"]
|
|
82
|
+
disallow_untyped_defs = true
|
|
83
|
+
warn_redundant_casts = true
|
|
84
|
+
warn_unused_ignores = true
|
|
85
|
+
no_implicit_optional = true
|
|
86
|
+
check_untyped_defs = true
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
"""Turn production agent failures into regression tests."""
|
|
2
|
+
|
|
3
|
+
from importlib.metadata import PackageNotFoundError, version
|
|
4
|
+
|
|
5
|
+
from undoom.capture.anthropic import wrap_anthropic
|
|
6
|
+
from undoom.capture.context import run
|
|
7
|
+
from undoom.capture.openai import wrap_openai
|
|
8
|
+
from undoom.config import configure
|
|
9
|
+
|
|
10
|
+
try:
|
|
11
|
+
__version__ = version("undoom")
|
|
12
|
+
except PackageNotFoundError:
|
|
13
|
+
__version__ = "1.0.0"
|
|
14
|
+
|
|
15
|
+
__all__ = ["__version__", "configure", "run", "wrap_anthropic", "wrap_openai"]
|