undoom 1.0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- undoom/__init__.py +15 -0
- undoom/__main__.py +4 -0
- undoom/api.py +127 -0
- undoom/canonical.py +31 -0
- undoom/capture/__init__.py +7 -0
- undoom/capture/anthropic.py +161 -0
- undoom/capture/context.py +116 -0
- undoom/capture/openai.py +97 -0
- undoom/cli.py +242 -0
- undoom/config.py +83 -0
- undoom/errors.py +21 -0
- undoom/models.py +206 -0
- undoom/openai_messages.py +87 -0
- undoom/otel/__init__.py +5 -0
- undoom/otel/import_trace.py +268 -0
- undoom/present.py +56 -0
- undoom/py.typed +0 -0
- undoom/redact.py +69 -0
- undoom/replay/__init__.py +5 -0
- undoom/replay/cassette.py +37 -0
- undoom/replay/changeset.py +63 -0
- undoom/replay/diff.py +69 -0
- undoom/replay/engine.py +227 -0
- undoom/replay/loading.py +26 -0
- undoom/replay/providers.py +72 -0
- undoom/store/__init__.py +7 -0
- undoom/store/base.py +21 -0
- undoom/store/migrate.py +65 -0
- undoom/store/postgres.py +178 -0
- undoom/store/resolve.py +33 -0
- undoom/store/sqlite.py +170 -0
- undoom/suite/__init__.py +7 -0
- undoom/suite/files.py +34 -0
- undoom/suite/promote.py +55 -0
- undoom/suite/report.py +61 -0
- undoom/suite/runner.py +86 -0
- undoom-1.0.1.dist-info/METADATA +252 -0
- undoom-1.0.1.dist-info/RECORD +40 -0
- undoom-1.0.1.dist-info/WHEEL +4 -0
- undoom-1.0.1.dist-info/entry_points.txt +3 -0
undoom/__init__.py
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
"""Turn production agent failures into regression tests."""
|
|
2
|
+
|
|
3
|
+
from importlib.metadata import PackageNotFoundError, version
|
|
4
|
+
|
|
5
|
+
from undoom.capture.anthropic import wrap_anthropic
|
|
6
|
+
from undoom.capture.context import run
|
|
7
|
+
from undoom.capture.openai import wrap_openai
|
|
8
|
+
from undoom.config import configure
|
|
9
|
+
|
|
10
|
+
try:
|
|
11
|
+
__version__ = version("undoom")
|
|
12
|
+
except PackageNotFoundError:
|
|
13
|
+
__version__ = "1.0.0"
|
|
14
|
+
|
|
15
|
+
__all__ = ["__version__", "configure", "run", "wrap_anthropic", "wrap_openai"]
|
undoom/__main__.py
ADDED
undoom/api.py
ADDED
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
"""Local HTTP API for the diff UI.
|
|
2
|
+
|
|
3
|
+
The demo deployment does not use this process. It reads committed fixtures.
|
|
4
|
+
Writes are refused when `demo` is set.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
from typing import Any
|
|
11
|
+
|
|
12
|
+
from fastapi import FastAPI, HTTPException
|
|
13
|
+
from fastapi.middleware.cors import CORSMiddleware
|
|
14
|
+
from pydantic import BaseModel
|
|
15
|
+
|
|
16
|
+
from undoom.errors import PromoteError, ReplayError, UndoomError
|
|
17
|
+
from undoom.replay.changeset import build_changeset
|
|
18
|
+
from undoom.replay.engine import replay_run
|
|
19
|
+
from undoom.replay.providers import resolve_provider
|
|
20
|
+
from undoom.store.base import Store
|
|
21
|
+
from undoom.suite.files import load_cases, write_case
|
|
22
|
+
from undoom.suite.promote import promote_replay
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class ReplayIn(BaseModel):
|
|
26
|
+
run_id: str
|
|
27
|
+
prompt_text: str | None = None
|
|
28
|
+
prompt_file: str | None = None
|
|
29
|
+
model: str | None = None
|
|
30
|
+
tool_name: str | None = None
|
|
31
|
+
tool_entrypoint: str | None = None
|
|
32
|
+
provider: str | None = None
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
class PromoteIn(BaseModel):
|
|
36
|
+
replay_id: str
|
|
37
|
+
name: str
|
|
38
|
+
expect_tools: bool = False
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def create_app(store: Store, cases_dir: Path, *, demo: bool = False) -> FastAPI:
|
|
42
|
+
application = FastAPI(title="undoom", docs_url=None, redoc_url=None)
|
|
43
|
+
application.add_middleware(
|
|
44
|
+
CORSMiddleware,
|
|
45
|
+
allow_origin_regex=r"http://(localhost|127\.0\.0\.1)(:\d+)?",
|
|
46
|
+
allow_methods=["*"],
|
|
47
|
+
allow_headers=["*"],
|
|
48
|
+
)
|
|
49
|
+
|
|
50
|
+
@application.get("/api/meta")
|
|
51
|
+
def meta() -> dict[str, Any]:
|
|
52
|
+
return {"demo": demo, "product": "undoom"}
|
|
53
|
+
|
|
54
|
+
@application.get("/api/runs")
|
|
55
|
+
def list_runs() -> dict[str, Any]:
|
|
56
|
+
runs = store.list_runs(limit=200)
|
|
57
|
+
return {"runs": [run.model_dump(mode="json") for run in runs]}
|
|
58
|
+
|
|
59
|
+
@application.get("/api/runs/{run_id}")
|
|
60
|
+
def get_run(run_id: str) -> dict[str, Any]:
|
|
61
|
+
run = store.get_run(run_id)
|
|
62
|
+
if run is None:
|
|
63
|
+
raise HTTPException(status_code=404, detail="run not found")
|
|
64
|
+
return run.model_dump(mode="json")
|
|
65
|
+
|
|
66
|
+
@application.get("/api/replays")
|
|
67
|
+
def list_replays(source_run_id: str | None = None) -> dict[str, Any]:
|
|
68
|
+
replays = store.list_replays(limit=200, source_run_id=source_run_id)
|
|
69
|
+
return {"replays": [replay.model_dump(mode="json") for replay in replays]}
|
|
70
|
+
|
|
71
|
+
@application.get("/api/replays/{replay_id}")
|
|
72
|
+
def get_replay(replay_id: str) -> dict[str, Any]:
|
|
73
|
+
replay = store.get_replay(replay_id)
|
|
74
|
+
if replay is None:
|
|
75
|
+
raise HTTPException(status_code=404, detail="replay not found")
|
|
76
|
+
return replay.model_dump(mode="json")
|
|
77
|
+
|
|
78
|
+
@application.get("/api/cases")
|
|
79
|
+
def get_cases() -> dict[str, Any]:
|
|
80
|
+
cases = load_cases(cases_dir)
|
|
81
|
+
return {"cases": [case.model_dump(mode="json") for case in cases]}
|
|
82
|
+
|
|
83
|
+
@application.post("/api/replay")
|
|
84
|
+
def post_replay(body: ReplayIn) -> dict[str, Any]:
|
|
85
|
+
_require_write()
|
|
86
|
+
try:
|
|
87
|
+
changeset = build_changeset(
|
|
88
|
+
prompt_text=body.prompt_text,
|
|
89
|
+
prompt_file=body.prompt_file,
|
|
90
|
+
model=body.model,
|
|
91
|
+
tool_name=body.tool_name,
|
|
92
|
+
tool_entrypoint=body.tool_entrypoint,
|
|
93
|
+
)
|
|
94
|
+
source = store.get_run(body.run_id)
|
|
95
|
+
if source is None:
|
|
96
|
+
raise HTTPException(status_code=404, detail="run not found")
|
|
97
|
+
replay = replay_run(source, changeset, provider=resolve_provider(body.provider))
|
|
98
|
+
store.save_replay(replay)
|
|
99
|
+
except HTTPException:
|
|
100
|
+
raise
|
|
101
|
+
except (ReplayError, UndoomError) as exc:
|
|
102
|
+
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
|
103
|
+
return replay.model_dump(mode="json")
|
|
104
|
+
|
|
105
|
+
@application.post("/api/promote")
|
|
106
|
+
def post_promote(body: PromoteIn) -> dict[str, Any]:
|
|
107
|
+
_require_write()
|
|
108
|
+
try:
|
|
109
|
+
replay = store.get_replay(body.replay_id)
|
|
110
|
+
if replay is None:
|
|
111
|
+
raise HTTPException(status_code=404, detail="replay not found")
|
|
112
|
+
source = store.get_run(replay.source_run_id)
|
|
113
|
+
if source is None:
|
|
114
|
+
raise HTTPException(status_code=404, detail="source run not found")
|
|
115
|
+
case = promote_replay(replay, source, name=body.name, expect_tools=body.expect_tools)
|
|
116
|
+
path = write_case(case, cases_dir)
|
|
117
|
+
except HTTPException:
|
|
118
|
+
raise
|
|
119
|
+
except (PromoteError, ReplayError, UndoomError) as exc:
|
|
120
|
+
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
|
121
|
+
return {"path": str(path), "case": case.model_dump(mode="json")}
|
|
122
|
+
|
|
123
|
+
def _require_write() -> None:
|
|
124
|
+
if demo:
|
|
125
|
+
raise HTTPException(status_code=403, detail="demo mode does not accept writes")
|
|
126
|
+
|
|
127
|
+
return application
|
undoom/canonical.py
ADDED
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
"""Stable JSON keys for cassette lookup.
|
|
2
|
+
|
|
3
|
+
Floats are rounded so 1.0 and 1.0000001 do not look like different tool calls.
|
|
4
|
+
Object keys are sorted so argument order does not matter.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import json
|
|
10
|
+
from typing import Any
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def normalize(value: Any) -> Any:
|
|
14
|
+
if isinstance(value, bool) or value is None or isinstance(value, (str, int)):
|
|
15
|
+
return value
|
|
16
|
+
if isinstance(value, float):
|
|
17
|
+
return round(value, 6)
|
|
18
|
+
if isinstance(value, dict):
|
|
19
|
+
return {str(key): normalize(item) for key, item in value.items()}
|
|
20
|
+
if isinstance(value, (list, tuple)):
|
|
21
|
+
return [normalize(item) for item in value]
|
|
22
|
+
return str(value)
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def canonical_json(value: Any) -> str:
|
|
26
|
+
return json.dumps(
|
|
27
|
+
normalize(value),
|
|
28
|
+
sort_keys=True,
|
|
29
|
+
separators=(",", ":"),
|
|
30
|
+
ensure_ascii=False,
|
|
31
|
+
)
|
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
"""Anthropic Messages API capture adapter.
|
|
2
|
+
|
|
3
|
+
Records `client.messages.create` calls. Pass a tool map to run the tool loop
|
|
4
|
+
and record each round-trip. Importing this module does not import the Anthropic
|
|
5
|
+
SDK; the caller supplies a client.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import json
|
|
11
|
+
from collections.abc import Callable
|
|
12
|
+
from typing import Any
|
|
13
|
+
|
|
14
|
+
from undoom.capture.context import RunRecorder
|
|
15
|
+
from undoom.models import ModelResponse, ToolCall
|
|
16
|
+
from undoom.openai_messages import safe_params, usage_dict
|
|
17
|
+
|
|
18
|
+
MAX_TURNS = 8
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def wrap_anthropic(
|
|
22
|
+
client: Any,
|
|
23
|
+
recorder: RunRecorder,
|
|
24
|
+
tools: dict[str, Callable[..., Any]] | None = None,
|
|
25
|
+
) -> Any:
|
|
26
|
+
"""Record `client.messages.create` on `recorder`.
|
|
27
|
+
|
|
28
|
+
When `tools` is omitted, one completion is recorded and returned.
|
|
29
|
+
When `tools` is provided, tool_use blocks are executed, recorded, and fed
|
|
30
|
+
back until the model stops or the turn limit is hit.
|
|
31
|
+
"""
|
|
32
|
+
|
|
33
|
+
messages = client.messages
|
|
34
|
+
original = messages.create
|
|
35
|
+
|
|
36
|
+
def create(*args: Any, **kwargs: Any) -> Any:
|
|
37
|
+
if args:
|
|
38
|
+
raise TypeError("wrap_anthropic records keyword calls to messages.create")
|
|
39
|
+
if tools is None:
|
|
40
|
+
response = original(**kwargs)
|
|
41
|
+
_record(recorder, kwargs, response)
|
|
42
|
+
return response
|
|
43
|
+
return _tool_loop(original, recorder, tools, kwargs)
|
|
44
|
+
|
|
45
|
+
messages.create = create
|
|
46
|
+
return client
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def parse_message(response: Any) -> ModelResponse:
|
|
50
|
+
text_parts: list[str] = []
|
|
51
|
+
calls: list[ToolCall] = []
|
|
52
|
+
for block in getattr(response, "content", None) or []:
|
|
53
|
+
block_type = getattr(block, "type", None) or (block.get("type") if isinstance(block, dict) else None)
|
|
54
|
+
if block_type == "text":
|
|
55
|
+
value = getattr(block, "text", None)
|
|
56
|
+
if value is None and isinstance(block, dict):
|
|
57
|
+
value = block.get("text")
|
|
58
|
+
if value:
|
|
59
|
+
text_parts.append(str(value))
|
|
60
|
+
elif block_type == "tool_use":
|
|
61
|
+
call_id = getattr(block, "id", None) or (block.get("id") if isinstance(block, dict) else None)
|
|
62
|
+
name = getattr(block, "name", None) or (block.get("name") if isinstance(block, dict) else None)
|
|
63
|
+
raw_input = getattr(block, "input", None)
|
|
64
|
+
if raw_input is None and isinstance(block, dict):
|
|
65
|
+
raw_input = block.get("input")
|
|
66
|
+
arguments = raw_input if isinstance(raw_input, dict) else {}
|
|
67
|
+
calls.append(ToolCall(id=str(call_id or ""), name=str(name or ""), arguments=arguments))
|
|
68
|
+
return ModelResponse(text="\n".join(text_parts) if text_parts else None, tool_calls=calls)
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def _record(recorder: RunRecorder, kwargs: dict[str, Any], response: Any) -> None:
|
|
72
|
+
parsed = parse_message(response)
|
|
73
|
+
recorder.record_model(
|
|
74
|
+
model=str(kwargs.get("model") or getattr(response, "model", "") or ""),
|
|
75
|
+
messages=list(kwargs.get("messages") or []),
|
|
76
|
+
response_text=parsed.text,
|
|
77
|
+
tool_calls=parsed.tool_calls,
|
|
78
|
+
params=safe_params(kwargs),
|
|
79
|
+
usage=usage_dict(response),
|
|
80
|
+
)
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def _assistant_content(response: Any) -> list[Any]:
|
|
84
|
+
content = getattr(response, "content", None)
|
|
85
|
+
if content is None:
|
|
86
|
+
return []
|
|
87
|
+
if hasattr(content, "model_dump"):
|
|
88
|
+
dumped = content.model_dump()
|
|
89
|
+
if isinstance(dumped, list):
|
|
90
|
+
return dumped
|
|
91
|
+
out: list[Any] = []
|
|
92
|
+
for block in content:
|
|
93
|
+
if hasattr(block, "model_dump"):
|
|
94
|
+
out.append(block.model_dump())
|
|
95
|
+
elif isinstance(block, dict):
|
|
96
|
+
out.append(block)
|
|
97
|
+
else:
|
|
98
|
+
out.append(
|
|
99
|
+
{
|
|
100
|
+
"type": getattr(block, "type", None),
|
|
101
|
+
"text": getattr(block, "text", None),
|
|
102
|
+
"id": getattr(block, "id", None),
|
|
103
|
+
"name": getattr(block, "name", None),
|
|
104
|
+
"input": getattr(block, "input", None),
|
|
105
|
+
}
|
|
106
|
+
)
|
|
107
|
+
return out
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def _tool_result_message(call_id: str, result: Any, error: str | None) -> dict[str, Any]:
|
|
111
|
+
if error is not None:
|
|
112
|
+
body: Any = {"error": error}
|
|
113
|
+
if result is not None:
|
|
114
|
+
body["result"] = result
|
|
115
|
+
else:
|
|
116
|
+
body = result
|
|
117
|
+
return {
|
|
118
|
+
"role": "user",
|
|
119
|
+
"content": [
|
|
120
|
+
{
|
|
121
|
+
"type": "tool_result",
|
|
122
|
+
"tool_use_id": call_id,
|
|
123
|
+
"content": json.dumps(body, default=str),
|
|
124
|
+
}
|
|
125
|
+
],
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def _tool_loop(
|
|
130
|
+
create: Callable[..., Any],
|
|
131
|
+
recorder: RunRecorder,
|
|
132
|
+
tools: dict[str, Callable[..., Any]],
|
|
133
|
+
kwargs: dict[str, Any],
|
|
134
|
+
) -> Any:
|
|
135
|
+
messages = list(kwargs.get("messages") or [])
|
|
136
|
+
last = None
|
|
137
|
+
for _ in range(MAX_TURNS):
|
|
138
|
+
pending = dict(kwargs)
|
|
139
|
+
pending["messages"] = messages
|
|
140
|
+
response = create(**pending)
|
|
141
|
+
last = response
|
|
142
|
+
parsed = parse_message(response)
|
|
143
|
+
_record(recorder, pending, response)
|
|
144
|
+
if not parsed.tool_calls:
|
|
145
|
+
return response
|
|
146
|
+
messages.append({"role": "assistant", "content": _assistant_content(response)})
|
|
147
|
+
for call in parsed.tool_calls:
|
|
148
|
+
function = tools.get(call.name)
|
|
149
|
+
if function is None:
|
|
150
|
+
with recorder.tool(call.name, call.arguments, call_id=call.id) as handle:
|
|
151
|
+
handle.record({"error": f"unknown tool {call.name}"})
|
|
152
|
+
recorder.fail(f"unknown tool {call.name}", unexpected_tools=[call.name])
|
|
153
|
+
messages.append(
|
|
154
|
+
_tool_result_message(call.id, {"error": f"unknown tool {call.name}"}, "unknown tool")
|
|
155
|
+
)
|
|
156
|
+
return response
|
|
157
|
+
with recorder.tool(call.name, call.arguments, call_id=call.id) as handle:
|
|
158
|
+
result = function(**call.arguments)
|
|
159
|
+
handle.record(result)
|
|
160
|
+
messages.append(_tool_result_message(call.id, result, None))
|
|
161
|
+
return last
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
"""Record a tool-calling run."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
from collections.abc import Iterator
|
|
7
|
+
from contextlib import contextmanager
|
|
8
|
+
from typing import Any
|
|
9
|
+
from uuid import uuid4
|
|
10
|
+
|
|
11
|
+
from undoom.config import get_config
|
|
12
|
+
from undoom.models import ModelStep, Run, RunStatus, ToolCall, ToolStep
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class ToolHandle:
|
|
16
|
+
"""Receiver for the result of one tool call."""
|
|
17
|
+
|
|
18
|
+
def __init__(self, step: ToolStep) -> None:
|
|
19
|
+
self._step = step
|
|
20
|
+
|
|
21
|
+
def record(self, result: Any) -> None:
|
|
22
|
+
self._step.result = result
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class RunRecorder:
|
|
26
|
+
"""Open run. Steps are stored when the `run` block exits."""
|
|
27
|
+
|
|
28
|
+
def __init__(self, record: Run) -> None:
|
|
29
|
+
self.run = record
|
|
30
|
+
self.failed = False
|
|
31
|
+
|
|
32
|
+
def fail(self, reason: str, unexpected_tools: list[str] | None = None) -> None:
|
|
33
|
+
"""Mark the run as a failure.
|
|
34
|
+
|
|
35
|
+
`unexpected_tools` names the calls that must not return. Promote turns
|
|
36
|
+
those names into the regression expectation.
|
|
37
|
+
"""
|
|
38
|
+
|
|
39
|
+
self.failed = True
|
|
40
|
+
if self.run.failure_reason is None:
|
|
41
|
+
self.run.failure_reason = reason
|
|
42
|
+
for name in unexpected_tools or []:
|
|
43
|
+
if name not in self.run.unexpected_tools:
|
|
44
|
+
self.run.unexpected_tools.append(name)
|
|
45
|
+
|
|
46
|
+
def record_model(
|
|
47
|
+
self,
|
|
48
|
+
*,
|
|
49
|
+
model: str,
|
|
50
|
+
messages: list[dict[str, Any]],
|
|
51
|
+
response_text: str | None,
|
|
52
|
+
tool_calls: list[ToolCall],
|
|
53
|
+
params: dict[str, Any] | None = None,
|
|
54
|
+
usage: dict[str, Any] | None = None,
|
|
55
|
+
) -> None:
|
|
56
|
+
self.run.steps.append(
|
|
57
|
+
ModelStep(
|
|
58
|
+
model=model,
|
|
59
|
+
messages=json.loads(json.dumps(messages, default=str)),
|
|
60
|
+
params=json.loads(json.dumps(params or {}, default=str)),
|
|
61
|
+
response_text=response_text,
|
|
62
|
+
tool_calls=tool_calls,
|
|
63
|
+
usage=usage,
|
|
64
|
+
)
|
|
65
|
+
)
|
|
66
|
+
|
|
67
|
+
@contextmanager
|
|
68
|
+
def tool(
|
|
69
|
+
self,
|
|
70
|
+
name: str,
|
|
71
|
+
arguments: dict[str, Any],
|
|
72
|
+
call_id: str | None = None,
|
|
73
|
+
) -> Iterator[ToolHandle]:
|
|
74
|
+
step = ToolStep(
|
|
75
|
+
call_id=call_id or uuid4().hex,
|
|
76
|
+
name=name,
|
|
77
|
+
arguments=arguments,
|
|
78
|
+
)
|
|
79
|
+
handle = ToolHandle(step)
|
|
80
|
+
try:
|
|
81
|
+
yield handle
|
|
82
|
+
except Exception as exc:
|
|
83
|
+
step.error = f"{type(exc).__name__}: {exc}"
|
|
84
|
+
self.fail(f"tool {name} failed: {exc}")
|
|
85
|
+
raise
|
|
86
|
+
finally:
|
|
87
|
+
self.run.steps.append(step)
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
@contextmanager
|
|
91
|
+
def run(agent: str | None = None, name: str | None = None) -> Iterator[RunRecorder]:
|
|
92
|
+
"""Capture one agent run and write it when the block exits."""
|
|
93
|
+
|
|
94
|
+
config = get_config()
|
|
95
|
+
record = Run(agent=agent or config.agent, name=name or "run")
|
|
96
|
+
recorder = RunRecorder(record)
|
|
97
|
+
try:
|
|
98
|
+
yield recorder
|
|
99
|
+
except Exception as exc:
|
|
100
|
+
if recorder.failed:
|
|
101
|
+
record.status = RunStatus.failure
|
|
102
|
+
elif record.status == RunStatus.running:
|
|
103
|
+
record.status = RunStatus.error
|
|
104
|
+
record.failure_reason = f"{type(exc).__name__}: {exc}"
|
|
105
|
+
raise
|
|
106
|
+
else:
|
|
107
|
+
if recorder.failed:
|
|
108
|
+
record.status = RunStatus.failure
|
|
109
|
+
elif record.status == RunStatus.running:
|
|
110
|
+
record.status = RunStatus.success
|
|
111
|
+
finally:
|
|
112
|
+
store = config.store()
|
|
113
|
+
store.save_run(recorder.run)
|
|
114
|
+
stored = store.get_run(recorder.run.id)
|
|
115
|
+
if stored is not None:
|
|
116
|
+
recorder.run = stored
|
undoom/capture/openai.py
ADDED
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
"""OpenAI chat-completions adapter.
|
|
2
|
+
|
|
3
|
+
The adapter records model calls. Pass a tool map and it also runs the tool
|
|
4
|
+
loop, recording each round-trip. Importing this module does not import the
|
|
5
|
+
OpenAI SDK; the caller supplies a client.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
from collections.abc import Callable
|
|
11
|
+
from typing import Any
|
|
12
|
+
|
|
13
|
+
from undoom.capture.context import RunRecorder
|
|
14
|
+
from undoom.openai_messages import (
|
|
15
|
+
assistant_message,
|
|
16
|
+
parse_completion,
|
|
17
|
+
safe_params,
|
|
18
|
+
tool_message,
|
|
19
|
+
usage_dict,
|
|
20
|
+
)
|
|
21
|
+
|
|
22
|
+
MAX_TURNS = 8
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def wrap_openai(
|
|
26
|
+
client: Any,
|
|
27
|
+
recorder: RunRecorder,
|
|
28
|
+
tools: dict[str, Callable[..., Any]] | None = None,
|
|
29
|
+
) -> Any:
|
|
30
|
+
"""Record `client.chat.completions.create` on `recorder`.
|
|
31
|
+
|
|
32
|
+
When `tools` is omitted, one completion is recorded and returned.
|
|
33
|
+
When `tools` is provided, tool calls are executed, recorded, and fed back
|
|
34
|
+
to the model until it stops or the turn limit is hit.
|
|
35
|
+
"""
|
|
36
|
+
|
|
37
|
+
completions = client.chat.completions
|
|
38
|
+
original = completions.create
|
|
39
|
+
|
|
40
|
+
def create(*args: Any, **kwargs: Any) -> Any:
|
|
41
|
+
if args:
|
|
42
|
+
raise TypeError("wrap_openai records keyword calls to chat.completions.create")
|
|
43
|
+
if tools is None:
|
|
44
|
+
response = original(**kwargs)
|
|
45
|
+
_record(recorder, kwargs, response)
|
|
46
|
+
return response
|
|
47
|
+
return _tool_loop(original, recorder, tools, kwargs)
|
|
48
|
+
|
|
49
|
+
completions.create = create
|
|
50
|
+
return client
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def _record(recorder: RunRecorder, kwargs: dict[str, Any], response: Any) -> None:
|
|
54
|
+
parsed = parse_completion(response)
|
|
55
|
+
recorder.record_model(
|
|
56
|
+
model=str(kwargs.get("model") or ""),
|
|
57
|
+
messages=list(kwargs.get("messages") or []),
|
|
58
|
+
response_text=parsed.text,
|
|
59
|
+
tool_calls=parsed.tool_calls,
|
|
60
|
+
params=safe_params(kwargs),
|
|
61
|
+
usage=usage_dict(response),
|
|
62
|
+
)
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def _tool_loop(
|
|
66
|
+
create: Callable[..., Any],
|
|
67
|
+
recorder: RunRecorder,
|
|
68
|
+
tools: dict[str, Callable[..., Any]],
|
|
69
|
+
kwargs: dict[str, Any],
|
|
70
|
+
) -> Any:
|
|
71
|
+
messages = list(kwargs.get("messages") or [])
|
|
72
|
+
last = None
|
|
73
|
+
for _ in range(MAX_TURNS):
|
|
74
|
+
pending = dict(kwargs)
|
|
75
|
+
pending["messages"] = messages
|
|
76
|
+
response = create(**pending)
|
|
77
|
+
last = response
|
|
78
|
+
parsed = parse_completion(response)
|
|
79
|
+
_record(recorder, pending, response)
|
|
80
|
+
if not parsed.tool_calls:
|
|
81
|
+
return response
|
|
82
|
+
messages.append(assistant_message(parsed.text, parsed.tool_calls))
|
|
83
|
+
for call in parsed.tool_calls:
|
|
84
|
+
function = tools.get(call.name)
|
|
85
|
+
if function is None:
|
|
86
|
+
with recorder.tool(call.name, call.arguments, call_id=call.id) as handle:
|
|
87
|
+
handle.record({"error": f"unknown tool {call.name}"})
|
|
88
|
+
recorder.fail(f"unknown tool {call.name}", unexpected_tools=[call.name])
|
|
89
|
+
messages.append(
|
|
90
|
+
tool_message(call.id, {"error": f"unknown tool {call.name}"}, "unknown tool")
|
|
91
|
+
)
|
|
92
|
+
return response
|
|
93
|
+
with recorder.tool(call.name, call.arguments, call_id=call.id) as handle:
|
|
94
|
+
result = function(**call.arguments)
|
|
95
|
+
handle.record(result)
|
|
96
|
+
messages.append(tool_message(call.id, result, None))
|
|
97
|
+
return last
|