undoom 1.0.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
undoom/__init__.py ADDED
@@ -0,0 +1,15 @@
1
+ """Turn production agent failures into regression tests."""
2
+
3
+ from importlib.metadata import PackageNotFoundError, version
4
+
5
+ from undoom.capture.anthropic import wrap_anthropic
6
+ from undoom.capture.context import run
7
+ from undoom.capture.openai import wrap_openai
8
+ from undoom.config import configure
9
+
10
+ try:
11
+ __version__ = version("undoom")
12
+ except PackageNotFoundError:
13
+ __version__ = "1.0.0"
14
+
15
+ __all__ = ["__version__", "configure", "run", "wrap_anthropic", "wrap_openai"]
undoom/__main__.py ADDED
@@ -0,0 +1,4 @@
1
+ from undoom.cli import main
2
+
3
+ if __name__ == "__main__":
4
+ main()
undoom/api.py ADDED
@@ -0,0 +1,127 @@
1
+ """Local HTTP API for the diff UI.
2
+
3
+ The demo deployment does not use this process. It reads committed fixtures.
4
+ Writes are refused when `demo` is set.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ from pathlib import Path
10
+ from typing import Any
11
+
12
+ from fastapi import FastAPI, HTTPException
13
+ from fastapi.middleware.cors import CORSMiddleware
14
+ from pydantic import BaseModel
15
+
16
+ from undoom.errors import PromoteError, ReplayError, UndoomError
17
+ from undoom.replay.changeset import build_changeset
18
+ from undoom.replay.engine import replay_run
19
+ from undoom.replay.providers import resolve_provider
20
+ from undoom.store.base import Store
21
+ from undoom.suite.files import load_cases, write_case
22
+ from undoom.suite.promote import promote_replay
23
+
24
+
25
+ class ReplayIn(BaseModel):
26
+ run_id: str
27
+ prompt_text: str | None = None
28
+ prompt_file: str | None = None
29
+ model: str | None = None
30
+ tool_name: str | None = None
31
+ tool_entrypoint: str | None = None
32
+ provider: str | None = None
33
+
34
+
35
+ class PromoteIn(BaseModel):
36
+ replay_id: str
37
+ name: str
38
+ expect_tools: bool = False
39
+
40
+
41
+ def create_app(store: Store, cases_dir: Path, *, demo: bool = False) -> FastAPI:
42
+ application = FastAPI(title="undoom", docs_url=None, redoc_url=None)
43
+ application.add_middleware(
44
+ CORSMiddleware,
45
+ allow_origin_regex=r"http://(localhost|127\.0\.0\.1)(:\d+)?",
46
+ allow_methods=["*"],
47
+ allow_headers=["*"],
48
+ )
49
+
50
+ @application.get("/api/meta")
51
+ def meta() -> dict[str, Any]:
52
+ return {"demo": demo, "product": "undoom"}
53
+
54
+ @application.get("/api/runs")
55
+ def list_runs() -> dict[str, Any]:
56
+ runs = store.list_runs(limit=200)
57
+ return {"runs": [run.model_dump(mode="json") for run in runs]}
58
+
59
+ @application.get("/api/runs/{run_id}")
60
+ def get_run(run_id: str) -> dict[str, Any]:
61
+ run = store.get_run(run_id)
62
+ if run is None:
63
+ raise HTTPException(status_code=404, detail="run not found")
64
+ return run.model_dump(mode="json")
65
+
66
+ @application.get("/api/replays")
67
+ def list_replays(source_run_id: str | None = None) -> dict[str, Any]:
68
+ replays = store.list_replays(limit=200, source_run_id=source_run_id)
69
+ return {"replays": [replay.model_dump(mode="json") for replay in replays]}
70
+
71
+ @application.get("/api/replays/{replay_id}")
72
+ def get_replay(replay_id: str) -> dict[str, Any]:
73
+ replay = store.get_replay(replay_id)
74
+ if replay is None:
75
+ raise HTTPException(status_code=404, detail="replay not found")
76
+ return replay.model_dump(mode="json")
77
+
78
+ @application.get("/api/cases")
79
+ def get_cases() -> dict[str, Any]:
80
+ cases = load_cases(cases_dir)
81
+ return {"cases": [case.model_dump(mode="json") for case in cases]}
82
+
83
+ @application.post("/api/replay")
84
+ def post_replay(body: ReplayIn) -> dict[str, Any]:
85
+ _require_write()
86
+ try:
87
+ changeset = build_changeset(
88
+ prompt_text=body.prompt_text,
89
+ prompt_file=body.prompt_file,
90
+ model=body.model,
91
+ tool_name=body.tool_name,
92
+ tool_entrypoint=body.tool_entrypoint,
93
+ )
94
+ source = store.get_run(body.run_id)
95
+ if source is None:
96
+ raise HTTPException(status_code=404, detail="run not found")
97
+ replay = replay_run(source, changeset, provider=resolve_provider(body.provider))
98
+ store.save_replay(replay)
99
+ except HTTPException:
100
+ raise
101
+ except (ReplayError, UndoomError) as exc:
102
+ raise HTTPException(status_code=400, detail=str(exc)) from exc
103
+ return replay.model_dump(mode="json")
104
+
105
+ @application.post("/api/promote")
106
+ def post_promote(body: PromoteIn) -> dict[str, Any]:
107
+ _require_write()
108
+ try:
109
+ replay = store.get_replay(body.replay_id)
110
+ if replay is None:
111
+ raise HTTPException(status_code=404, detail="replay not found")
112
+ source = store.get_run(replay.source_run_id)
113
+ if source is None:
114
+ raise HTTPException(status_code=404, detail="source run not found")
115
+ case = promote_replay(replay, source, name=body.name, expect_tools=body.expect_tools)
116
+ path = write_case(case, cases_dir)
117
+ except HTTPException:
118
+ raise
119
+ except (PromoteError, ReplayError, UndoomError) as exc:
120
+ raise HTTPException(status_code=400, detail=str(exc)) from exc
121
+ return {"path": str(path), "case": case.model_dump(mode="json")}
122
+
123
+ def _require_write() -> None:
124
+ if demo:
125
+ raise HTTPException(status_code=403, detail="demo mode does not accept writes")
126
+
127
+ return application
undoom/canonical.py ADDED
@@ -0,0 +1,31 @@
1
+ """Stable JSON keys for cassette lookup.
2
+
3
+ Floats are rounded so 1.0 and 1.0000001 do not look like different tool calls.
4
+ Object keys are sorted so argument order does not matter.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ import json
10
+ from typing import Any
11
+
12
+
13
+ def normalize(value: Any) -> Any:
14
+ if isinstance(value, bool) or value is None or isinstance(value, (str, int)):
15
+ return value
16
+ if isinstance(value, float):
17
+ return round(value, 6)
18
+ if isinstance(value, dict):
19
+ return {str(key): normalize(item) for key, item in value.items()}
20
+ if isinstance(value, (list, tuple)):
21
+ return [normalize(item) for item in value]
22
+ return str(value)
23
+
24
+
25
+ def canonical_json(value: Any) -> str:
26
+ return json.dumps(
27
+ normalize(value),
28
+ sort_keys=True,
29
+ separators=(",", ":"),
30
+ ensure_ascii=False,
31
+ )
@@ -0,0 +1,7 @@
1
+ """Capture helpers."""
2
+
3
+ from undoom.capture.anthropic import wrap_anthropic
4
+ from undoom.capture.context import RunRecorder, run
5
+ from undoom.capture.openai import wrap_openai
6
+
7
+ __all__ = ["RunRecorder", "run", "wrap_anthropic", "wrap_openai"]
@@ -0,0 +1,161 @@
1
+ """Anthropic Messages API capture adapter.
2
+
3
+ Records `client.messages.create` calls. Pass a tool map to run the tool loop
4
+ and record each round-trip. Importing this module does not import the Anthropic
5
+ SDK; the caller supplies a client.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import json
11
+ from collections.abc import Callable
12
+ from typing import Any
13
+
14
+ from undoom.capture.context import RunRecorder
15
+ from undoom.models import ModelResponse, ToolCall
16
+ from undoom.openai_messages import safe_params, usage_dict
17
+
18
+ MAX_TURNS = 8
19
+
20
+
21
+ def wrap_anthropic(
22
+ client: Any,
23
+ recorder: RunRecorder,
24
+ tools: dict[str, Callable[..., Any]] | None = None,
25
+ ) -> Any:
26
+ """Record `client.messages.create` on `recorder`.
27
+
28
+ When `tools` is omitted, one completion is recorded and returned.
29
+ When `tools` is provided, tool_use blocks are executed, recorded, and fed
30
+ back until the model stops or the turn limit is hit.
31
+ """
32
+
33
+ messages = client.messages
34
+ original = messages.create
35
+
36
+ def create(*args: Any, **kwargs: Any) -> Any:
37
+ if args:
38
+ raise TypeError("wrap_anthropic records keyword calls to messages.create")
39
+ if tools is None:
40
+ response = original(**kwargs)
41
+ _record(recorder, kwargs, response)
42
+ return response
43
+ return _tool_loop(original, recorder, tools, kwargs)
44
+
45
+ messages.create = create
46
+ return client
47
+
48
+
49
+ def parse_message(response: Any) -> ModelResponse:
50
+ text_parts: list[str] = []
51
+ calls: list[ToolCall] = []
52
+ for block in getattr(response, "content", None) or []:
53
+ block_type = getattr(block, "type", None) or (block.get("type") if isinstance(block, dict) else None)
54
+ if block_type == "text":
55
+ value = getattr(block, "text", None)
56
+ if value is None and isinstance(block, dict):
57
+ value = block.get("text")
58
+ if value:
59
+ text_parts.append(str(value))
60
+ elif block_type == "tool_use":
61
+ call_id = getattr(block, "id", None) or (block.get("id") if isinstance(block, dict) else None)
62
+ name = getattr(block, "name", None) or (block.get("name") if isinstance(block, dict) else None)
63
+ raw_input = getattr(block, "input", None)
64
+ if raw_input is None and isinstance(block, dict):
65
+ raw_input = block.get("input")
66
+ arguments = raw_input if isinstance(raw_input, dict) else {}
67
+ calls.append(ToolCall(id=str(call_id or ""), name=str(name or ""), arguments=arguments))
68
+ return ModelResponse(text="\n".join(text_parts) if text_parts else None, tool_calls=calls)
69
+
70
+
71
+ def _record(recorder: RunRecorder, kwargs: dict[str, Any], response: Any) -> None:
72
+ parsed = parse_message(response)
73
+ recorder.record_model(
74
+ model=str(kwargs.get("model") or getattr(response, "model", "") or ""),
75
+ messages=list(kwargs.get("messages") or []),
76
+ response_text=parsed.text,
77
+ tool_calls=parsed.tool_calls,
78
+ params=safe_params(kwargs),
79
+ usage=usage_dict(response),
80
+ )
81
+
82
+
83
+ def _assistant_content(response: Any) -> list[Any]:
84
+ content = getattr(response, "content", None)
85
+ if content is None:
86
+ return []
87
+ if hasattr(content, "model_dump"):
88
+ dumped = content.model_dump()
89
+ if isinstance(dumped, list):
90
+ return dumped
91
+ out: list[Any] = []
92
+ for block in content:
93
+ if hasattr(block, "model_dump"):
94
+ out.append(block.model_dump())
95
+ elif isinstance(block, dict):
96
+ out.append(block)
97
+ else:
98
+ out.append(
99
+ {
100
+ "type": getattr(block, "type", None),
101
+ "text": getattr(block, "text", None),
102
+ "id": getattr(block, "id", None),
103
+ "name": getattr(block, "name", None),
104
+ "input": getattr(block, "input", None),
105
+ }
106
+ )
107
+ return out
108
+
109
+
110
+ def _tool_result_message(call_id: str, result: Any, error: str | None) -> dict[str, Any]:
111
+ if error is not None:
112
+ body: Any = {"error": error}
113
+ if result is not None:
114
+ body["result"] = result
115
+ else:
116
+ body = result
117
+ return {
118
+ "role": "user",
119
+ "content": [
120
+ {
121
+ "type": "tool_result",
122
+ "tool_use_id": call_id,
123
+ "content": json.dumps(body, default=str),
124
+ }
125
+ ],
126
+ }
127
+
128
+
129
+ def _tool_loop(
130
+ create: Callable[..., Any],
131
+ recorder: RunRecorder,
132
+ tools: dict[str, Callable[..., Any]],
133
+ kwargs: dict[str, Any],
134
+ ) -> Any:
135
+ messages = list(kwargs.get("messages") or [])
136
+ last = None
137
+ for _ in range(MAX_TURNS):
138
+ pending = dict(kwargs)
139
+ pending["messages"] = messages
140
+ response = create(**pending)
141
+ last = response
142
+ parsed = parse_message(response)
143
+ _record(recorder, pending, response)
144
+ if not parsed.tool_calls:
145
+ return response
146
+ messages.append({"role": "assistant", "content": _assistant_content(response)})
147
+ for call in parsed.tool_calls:
148
+ function = tools.get(call.name)
149
+ if function is None:
150
+ with recorder.tool(call.name, call.arguments, call_id=call.id) as handle:
151
+ handle.record({"error": f"unknown tool {call.name}"})
152
+ recorder.fail(f"unknown tool {call.name}", unexpected_tools=[call.name])
153
+ messages.append(
154
+ _tool_result_message(call.id, {"error": f"unknown tool {call.name}"}, "unknown tool")
155
+ )
156
+ return response
157
+ with recorder.tool(call.name, call.arguments, call_id=call.id) as handle:
158
+ result = function(**call.arguments)
159
+ handle.record(result)
160
+ messages.append(_tool_result_message(call.id, result, None))
161
+ return last
@@ -0,0 +1,116 @@
1
+ """Record a tool-calling run."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import json
6
+ from collections.abc import Iterator
7
+ from contextlib import contextmanager
8
+ from typing import Any
9
+ from uuid import uuid4
10
+
11
+ from undoom.config import get_config
12
+ from undoom.models import ModelStep, Run, RunStatus, ToolCall, ToolStep
13
+
14
+
15
+ class ToolHandle:
16
+ """Receiver for the result of one tool call."""
17
+
18
+ def __init__(self, step: ToolStep) -> None:
19
+ self._step = step
20
+
21
+ def record(self, result: Any) -> None:
22
+ self._step.result = result
23
+
24
+
25
+ class RunRecorder:
26
+ """Open run. Steps are stored when the `run` block exits."""
27
+
28
+ def __init__(self, record: Run) -> None:
29
+ self.run = record
30
+ self.failed = False
31
+
32
+ def fail(self, reason: str, unexpected_tools: list[str] | None = None) -> None:
33
+ """Mark the run as a failure.
34
+
35
+ `unexpected_tools` names the calls that must not return. Promote turns
36
+ those names into the regression expectation.
37
+ """
38
+
39
+ self.failed = True
40
+ if self.run.failure_reason is None:
41
+ self.run.failure_reason = reason
42
+ for name in unexpected_tools or []:
43
+ if name not in self.run.unexpected_tools:
44
+ self.run.unexpected_tools.append(name)
45
+
46
+ def record_model(
47
+ self,
48
+ *,
49
+ model: str,
50
+ messages: list[dict[str, Any]],
51
+ response_text: str | None,
52
+ tool_calls: list[ToolCall],
53
+ params: dict[str, Any] | None = None,
54
+ usage: dict[str, Any] | None = None,
55
+ ) -> None:
56
+ self.run.steps.append(
57
+ ModelStep(
58
+ model=model,
59
+ messages=json.loads(json.dumps(messages, default=str)),
60
+ params=json.loads(json.dumps(params or {}, default=str)),
61
+ response_text=response_text,
62
+ tool_calls=tool_calls,
63
+ usage=usage,
64
+ )
65
+ )
66
+
67
+ @contextmanager
68
+ def tool(
69
+ self,
70
+ name: str,
71
+ arguments: dict[str, Any],
72
+ call_id: str | None = None,
73
+ ) -> Iterator[ToolHandle]:
74
+ step = ToolStep(
75
+ call_id=call_id or uuid4().hex,
76
+ name=name,
77
+ arguments=arguments,
78
+ )
79
+ handle = ToolHandle(step)
80
+ try:
81
+ yield handle
82
+ except Exception as exc:
83
+ step.error = f"{type(exc).__name__}: {exc}"
84
+ self.fail(f"tool {name} failed: {exc}")
85
+ raise
86
+ finally:
87
+ self.run.steps.append(step)
88
+
89
+
90
+ @contextmanager
91
+ def run(agent: str | None = None, name: str | None = None) -> Iterator[RunRecorder]:
92
+ """Capture one agent run and write it when the block exits."""
93
+
94
+ config = get_config()
95
+ record = Run(agent=agent or config.agent, name=name or "run")
96
+ recorder = RunRecorder(record)
97
+ try:
98
+ yield recorder
99
+ except Exception as exc:
100
+ if recorder.failed:
101
+ record.status = RunStatus.failure
102
+ elif record.status == RunStatus.running:
103
+ record.status = RunStatus.error
104
+ record.failure_reason = f"{type(exc).__name__}: {exc}"
105
+ raise
106
+ else:
107
+ if recorder.failed:
108
+ record.status = RunStatus.failure
109
+ elif record.status == RunStatus.running:
110
+ record.status = RunStatus.success
111
+ finally:
112
+ store = config.store()
113
+ store.save_run(recorder.run)
114
+ stored = store.get_run(recorder.run.id)
115
+ if stored is not None:
116
+ recorder.run = stored
@@ -0,0 +1,97 @@
1
+ """OpenAI chat-completions adapter.
2
+
3
+ The adapter records model calls. Pass a tool map and it also runs the tool
4
+ loop, recording each round-trip. Importing this module does not import the
5
+ OpenAI SDK; the caller supplies a client.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ from collections.abc import Callable
11
+ from typing import Any
12
+
13
+ from undoom.capture.context import RunRecorder
14
+ from undoom.openai_messages import (
15
+ assistant_message,
16
+ parse_completion,
17
+ safe_params,
18
+ tool_message,
19
+ usage_dict,
20
+ )
21
+
22
+ MAX_TURNS = 8
23
+
24
+
25
+ def wrap_openai(
26
+ client: Any,
27
+ recorder: RunRecorder,
28
+ tools: dict[str, Callable[..., Any]] | None = None,
29
+ ) -> Any:
30
+ """Record `client.chat.completions.create` on `recorder`.
31
+
32
+ When `tools` is omitted, one completion is recorded and returned.
33
+ When `tools` is provided, tool calls are executed, recorded, and fed back
34
+ to the model until it stops or the turn limit is hit.
35
+ """
36
+
37
+ completions = client.chat.completions
38
+ original = completions.create
39
+
40
+ def create(*args: Any, **kwargs: Any) -> Any:
41
+ if args:
42
+ raise TypeError("wrap_openai records keyword calls to chat.completions.create")
43
+ if tools is None:
44
+ response = original(**kwargs)
45
+ _record(recorder, kwargs, response)
46
+ return response
47
+ return _tool_loop(original, recorder, tools, kwargs)
48
+
49
+ completions.create = create
50
+ return client
51
+
52
+
53
+ def _record(recorder: RunRecorder, kwargs: dict[str, Any], response: Any) -> None:
54
+ parsed = parse_completion(response)
55
+ recorder.record_model(
56
+ model=str(kwargs.get("model") or ""),
57
+ messages=list(kwargs.get("messages") or []),
58
+ response_text=parsed.text,
59
+ tool_calls=parsed.tool_calls,
60
+ params=safe_params(kwargs),
61
+ usage=usage_dict(response),
62
+ )
63
+
64
+
65
+ def _tool_loop(
66
+ create: Callable[..., Any],
67
+ recorder: RunRecorder,
68
+ tools: dict[str, Callable[..., Any]],
69
+ kwargs: dict[str, Any],
70
+ ) -> Any:
71
+ messages = list(kwargs.get("messages") or [])
72
+ last = None
73
+ for _ in range(MAX_TURNS):
74
+ pending = dict(kwargs)
75
+ pending["messages"] = messages
76
+ response = create(**pending)
77
+ last = response
78
+ parsed = parse_completion(response)
79
+ _record(recorder, pending, response)
80
+ if not parsed.tool_calls:
81
+ return response
82
+ messages.append(assistant_message(parsed.text, parsed.tool_calls))
83
+ for call in parsed.tool_calls:
84
+ function = tools.get(call.name)
85
+ if function is None:
86
+ with recorder.tool(call.name, call.arguments, call_id=call.id) as handle:
87
+ handle.record({"error": f"unknown tool {call.name}"})
88
+ recorder.fail(f"unknown tool {call.name}", unexpected_tools=[call.name])
89
+ messages.append(
90
+ tool_message(call.id, {"error": f"unknown tool {call.name}"}, "unknown tool")
91
+ )
92
+ return response
93
+ with recorder.tool(call.name, call.arguments, call_id=call.id) as handle:
94
+ result = function(**call.arguments)
95
+ handle.record(result)
96
+ messages.append(tool_message(call.id, result, None))
97
+ return last