first-domino 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- first_domino-0.1.0/.gitignore +18 -0
- first_domino-0.1.0/LICENSE +21 -0
- first_domino-0.1.0/PKG-INFO +243 -0
- first_domino-0.1.0/README.md +212 -0
- first_domino-0.1.0/pyproject.toml +50 -0
- first_domino-0.1.0/src/first_domino/__init__.py +62 -0
- first_domino-0.1.0/src/first_domino/_generated/__init__.py +1 -0
- first_domino-0.1.0/src/first_domino/_generated/models.py +1322 -0
- first_domino-0.1.0/src/first_domino/capture.py +565 -0
- first_domino-0.1.0/src/first_domino/cli.py +1 -0
- first_domino-0.1.0/src/first_domino/client.py +179 -0
- first_domino-0.1.0/src/first_domino/diagnose.py +42 -0
- first_domino-0.1.0/src/first_domino/errors.py +30 -0
- first_domino-0.1.0/src/first_domino/instrument.py +488 -0
- first_domino-0.1.0/src/first_domino/openai_agents.py +306 -0
- first_domino-0.1.0/src/first_domino/openai_format.py +81 -0
- first_domino-0.1.0/src/first_domino/providers.py +538 -0
- first_domino-0.1.0/src/first_domino/py.typed +0 -0
- first_domino-0.1.0/src/first_domino/recorder.py +207 -0
- first_domino-0.1.0/src/first_domino/responses_format.py +94 -0
- first_domino-0.1.0/src/first_domino/settings.py +34 -0
- first_domino-0.1.0/src/first_domino/testing.py +234 -0
- first_domino-0.1.0/src/first_domino/worker.py +226 -0
- first_domino-0.1.0/src/first_domino/wrap.py +136 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Jaini Sai Abhiram
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,243 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: first-domino
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Find the step where a failed AI agent run went wrong, and prove it with a replay.
|
|
5
|
+
Project-URL: Homepage, https://first-domino.abhiramtech.in
|
|
6
|
+
Project-URL: Repository, https://github.com/saiabhiramjaini/first-domino
|
|
7
|
+
Project-URL: Documentation, https://github.com/saiabhiramjaini/first-domino/tree/main/server/packages/first-domino#readme
|
|
8
|
+
Project-URL: Issues, https://github.com/saiabhiramjaini/first-domino/issues
|
|
9
|
+
Author: Jaini Sai Abhiram
|
|
10
|
+
License-Expression: MIT
|
|
11
|
+
License-File: LICENSE
|
|
12
|
+
Keywords: agents,ai,anthropic,debugging,evals,langchain,llm,observability,openai
|
|
13
|
+
Classifier: Development Status :: 4 - Beta
|
|
14
|
+
Classifier: Intended Audience :: Developers
|
|
15
|
+
Classifier: Operating System :: OS Independent
|
|
16
|
+
Classifier: Programming Language :: Python :: 3
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
19
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
20
|
+
Classifier: Topic :: Software Development :: Debuggers
|
|
21
|
+
Classifier: Typing :: Typed
|
|
22
|
+
Requires-Python: >=3.12
|
|
23
|
+
Requires-Dist: httpx>=0.27
|
|
24
|
+
Requires-Dist: pydantic-settings>=2.6
|
|
25
|
+
Requires-Dist: pydantic>=2.9
|
|
26
|
+
Provides-Extra: agents
|
|
27
|
+
Requires-Dist: openai-agents>=0.25.0; extra == 'agents'
|
|
28
|
+
Provides-Extra: openai
|
|
29
|
+
Requires-Dist: openai>=1.55; extra == 'openai'
|
|
30
|
+
Description-Content-Type: text/markdown
|
|
31
|
+
|
|
32
|
+
# first-domino
|
|
33
|
+
|
|
34
|
+
Find the step where a failed AI agent run went wrong, and prove it.
|
|
35
|
+
|
|
36
|
+
First Domino records every run of your agent, live. When a run fails, it replays the run from its own save points to find the first step after which the run always fails (the first domino), then rewrites that one step and replays again to prove the fix. Replays run in a worker on your side, so your code, tools and secrets never leave your machines.
|
|
37
|
+
|
|
38
|
+
Works with any agent that calls a model through the OpenAI or Anthropic SDK, or a framework built on them: LangChain, the OpenAI Agents SDK, LlamaIndex, CrewAI, or a plain loop. Sync or async, streaming or not, and any OpenAI-compatible API.
|
|
39
|
+
|
|
40
|
+
## Install
|
|
41
|
+
|
|
42
|
+
```bash
|
|
43
|
+
pip install first-domino
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
Python 3.12+. Set the key from your project's Settings page:
|
|
47
|
+
|
|
48
|
+
```bash
|
|
49
|
+
export FIRST_DOMINO_API_KEY=fd_live_...
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
Without a key the SDK does nothing, so the same code runs in tests and in production.
|
|
53
|
+
|
|
54
|
+
## 1. Record your agent: two lines
|
|
55
|
+
|
|
56
|
+
Mark the function that runs your agent. Each call becomes one run, and every model call inside it becomes a step, as it happens.
|
|
57
|
+
|
|
58
|
+
```python
|
|
59
|
+
import first_domino as fd
|
|
60
|
+
|
|
61
|
+
@fd.entrypoint("support-bot", check=lambda question, answer: "refund" in answer)
|
|
62
|
+
def answer(question: str) -> str:
|
|
63
|
+
... # your agent, unchanged
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
- `check(output)` or `check(input, output)` says whether the run passed. A run that raises is recorded as `errored`, and your exception continues.
|
|
67
|
+
- Already know the answer inside your code? Call `fd.set_outcome(passed=False, reason="...")`; it wins over `check`.
|
|
68
|
+
- The function's arguments are saved with the run so it can be replayed. Use JSON-friendly types: text, numbers, lists, dicts, dataclasses or pydantic models.
|
|
69
|
+
- Recording never raises into your agent. If First Domino is unreachable, your code behaves exactly as without it.
|
|
70
|
+
- Failed runs are diagnosed on their own.
|
|
71
|
+
|
|
72
|
+
### Tools with side effects
|
|
73
|
+
|
|
74
|
+
During a replay, the steps before the one being tested come from the recording. Mark a tool that must not run twice (sends an email, moves money) so it returns its recorded result there instead:
|
|
75
|
+
|
|
76
|
+
```python
|
|
77
|
+
@fd.tool
|
|
78
|
+
def send_refund(order_id: str, amount: float) -> str:
|
|
79
|
+
...
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
Put it under your framework's own decorator (`@function_tool`, `@tool`). Read-only tools need no marking.
|
|
83
|
+
|
|
84
|
+
## 2. Start the replay worker
|
|
85
|
+
|
|
86
|
+
Run this next to your agent, with the same code and keys:
|
|
87
|
+
|
|
88
|
+
```python
|
|
89
|
+
import first_domino as fd
|
|
90
|
+
import my_agent # the module with your @fd.entrypoint
|
|
91
|
+
|
|
92
|
+
fd.serve()
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
The worker only connects outward. When First Domino needs a replay, the worker calls your function again: model calls up to the save point are answered from the recording, the next one with the fixed step when there is one, and the rest by the model. Your `check` decides whether the replay passed.
|
|
96
|
+
|
|
97
|
+
Async code can run `await fd.worker().run()` instead.
|
|
98
|
+
|
|
99
|
+
## Advanced
|
|
100
|
+
|
|
101
|
+
The two lines above cover most agents. These lower-level APIs remain for full control.
|
|
102
|
+
|
|
103
|
+
### Record runs by hand
|
|
104
|
+
|
|
105
|
+
Wrap your OpenAI client inside `fd.record`. Each completion becomes a step (the model turn plus the tool results it led to), and the tools you pass become the run's tool list. This is a complete call assistant loop:
|
|
106
|
+
|
|
107
|
+
```python
|
|
108
|
+
import first_domino as fd
|
|
109
|
+
from openai import AsyncOpenAI
|
|
110
|
+
|
|
111
|
+
openai = AsyncOpenAI()
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
async def answer_call(call_text: str, task_id: str) -> None:
|
|
115
|
+
task = fd.TaskInput(prompt=call_text, metadata={"task_id": task_id})
|
|
116
|
+
async with fd.record("call-assistant", task) as run:
|
|
117
|
+
client = fd.wrap(openai, run)
|
|
118
|
+
messages = [{"role": "system", "content": SYSTEM}, {"role": "user", "content": call_text}]
|
|
119
|
+
while True:
|
|
120
|
+
response = await client.chat.completions.create(
|
|
121
|
+
model=MODEL, messages=messages, tools=TOOLS
|
|
122
|
+
)
|
|
123
|
+
reply = response.choices[0].message
|
|
124
|
+
messages.append(reply.model_dump(exclude_none=True))
|
|
125
|
+
if not reply.tool_calls:
|
|
126
|
+
break
|
|
127
|
+
for call in reply.tool_calls:
|
|
128
|
+
result = run_tool(call)
|
|
129
|
+
messages.append({"role": "tool", "tool_call_id": call.id, "content": result})
|
|
130
|
+
outcome = fd.RunOutcome.passed if call_handled(task_id) else fd.RunOutcome.failed
|
|
131
|
+
await run.finish(outcome, final_output=reply.content)
|
|
132
|
+
```
|
|
133
|
+
|
|
134
|
+
- Recording never raises into your agent. If First Domino is unreachable, your code behaves exactly as without it.
|
|
135
|
+
- If the block raises, the run is finished as `errored` and your exception continues.
|
|
136
|
+
- Put what your success check needs (here `task_id`) in `TaskInput.metadata`. Replays get it back.
|
|
137
|
+
- A run finished as `failed` is diagnosed automatically.
|
|
138
|
+
- Not using OpenAI's client directly? Call `await run.step(model=..., output=..., tool_results=..., messages=...)` once per model turn. `fd.message_from_openai` converts OpenAI message dicts.
|
|
139
|
+
|
|
140
|
+
### Agents built with the OpenAI Agents SDK
|
|
141
|
+
|
|
142
|
+
Install `first-domino[agents]` and run the agent through the adapter. Every model turn is recorded as a step, and a replay resumes `Runner.run` from the save point:
|
|
143
|
+
|
|
144
|
+
```python
|
|
145
|
+
from first_domino import openai_agents as fd_agents
|
|
146
|
+
|
|
147
|
+
async with fd.record("call-assistant", task) as run:
|
|
148
|
+
result = await fd_agents.run_recorded(agent, call_text, run)
|
|
149
|
+
await run.finish(fd.RunOutcome.passed if handled(result) else fd.RunOutcome.failed)
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
@worker.agent("call-assistant")
|
|
153
|
+
async def replay(ctx: fd.ReplayContext) -> RunResult:
|
|
154
|
+
return await fd_agents.replay(ctx, agent) # replacement output and model override included
|
|
155
|
+
```
|
|
156
|
+
|
|
157
|
+
Replays resume from the recorded messages; your tools only see new calls, so restore any state they need for the save point first. Streamed turns (`Runner.run_streamed`) are not recorded.
|
|
158
|
+
|
|
159
|
+
### Write the replay worker by hand
|
|
160
|
+
|
|
161
|
+
The worker only calls out. It leases replay jobs, runs your agent from the save point it is given, runs your success check, and reports the outcome.
|
|
162
|
+
|
|
163
|
+
```python
|
|
164
|
+
worker = fd.Worker(agent_names=["call-assistant"])
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
@worker.agent("call-assistant")
|
|
168
|
+
async def replay(ctx: fd.ReplayContext) -> str:
|
|
169
|
+
# Continue from ctx.messages (from_step steps already done). When ctx.replacement_output
|
|
170
|
+
# is set, use it as the next model turn instead of calling the model. When
|
|
171
|
+
# ctx.model_override is set, use that model. Record each step with ctx.run.step(...).
|
|
172
|
+
task_id = ctx.task_input.metadata["task_id"]
|
|
173
|
+
await continue_call(fd.messages_to_openai(ctx.messages), ctx)
|
|
174
|
+
return task_id
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
@worker.success_check("call-assistant")
|
|
178
|
+
def check(task_input: fd.TaskInput, task_id: str) -> bool:
|
|
179
|
+
return call_handled(task_id)
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
await worker.run() # leases forever, several jobs at a time
|
|
183
|
+
```
|
|
184
|
+
|
|
185
|
+
- One failed job never stops the worker: a crash in your agent or check is reported as `errored` and the loop goes on.
|
|
186
|
+
- The worker refuses to start without an API key or without both handlers for each agent name.
|
|
187
|
+
- `FIRST_DOMINO_WORKER_CONCURRENCY` (default 4) sets how many replays run at once.
|
|
188
|
+
|
|
189
|
+
## Diagnose from code
|
|
190
|
+
|
|
191
|
+
Failed runs are diagnosed automatically. To start one yourself and wait for the verdict:
|
|
192
|
+
|
|
193
|
+
```python
|
|
194
|
+
diagnosis = await fd.diagnose(run_id)
|
|
195
|
+
async with asyncio.timeout(600):
|
|
196
|
+
final = await fd.wait_for_diagnosis(diagnosis.id)
|
|
197
|
+
print(final.verdict_kind, final.first_domino_step, final.explanation.title)
|
|
198
|
+
# step_verified 6 Converted AM appointment to PM
|
|
199
|
+
```
|
|
200
|
+
|
|
201
|
+
## Test without a server
|
|
202
|
+
|
|
203
|
+
`first_domino.testing` is an in-memory API, so you can test your worker with no network:
|
|
204
|
+
|
|
205
|
+
```python
|
|
206
|
+
from first_domino.testing import FakeApi, fake_client, make_job
|
|
207
|
+
|
|
208
|
+
api = FakeApi()
|
|
209
|
+
api.jobs.append(make_job(agent_name="call-assistant", from_step=2))
|
|
210
|
+
worker = fd.Worker(agent_names=["call-assistant"], client=fake_client(api))
|
|
211
|
+
# register @worker.agent and @worker.success_check as above
|
|
212
|
+
assert await worker.run_once() == fd.ReplayOutcome.passed
|
|
213
|
+
```
|
|
214
|
+
|
|
215
|
+
## Reference
|
|
216
|
+
|
|
217
|
+
| Setting | Meaning | Default |
|
|
218
|
+
| --- | --- | --- |
|
|
219
|
+
| `FIRST_DOMINO_API_KEY` | Project API key. Without it, recording does nothing. | none |
|
|
220
|
+
| `FIRST_DOMINO_BASE_URL` | The First Domino API. Point it at your own server if you self-host. | `https://api-first-domino.abhiramtech.in` |
|
|
221
|
+
| `FIRST_DOMINO_WORKER_CONCURRENCY` | Replays one worker runs at once. | `4` |
|
|
222
|
+
| `FIRST_DOMINO_LEASE_WAIT_SECONDS` | How long one lease call waits for a job (0 to 20). | `10` |
|
|
223
|
+
| `FIRST_DOMINO_TIMEOUT_SECONDS` | Timeout of one API call. | `30` |
|
|
224
|
+
|
|
225
|
+
Settings can also live in a `.env` file in the working directory.
|
|
226
|
+
|
|
227
|
+
| Module | Job |
|
|
228
|
+
| --- | --- |
|
|
229
|
+
| `instrument` | `entrypoint()`, `tool()`, `set_outcome()`, `serve()`, `worker()` and `init()`: the two-line integration. |
|
|
230
|
+
| `capture` | Records model calls at the HTTP layer (`httpx` and `httpx2`) and answers them during a replay. |
|
|
231
|
+
| `providers` | The OpenAI Chat Completions, OpenAI Responses and Anthropic Messages wire formats. |
|
|
232
|
+
| `recorder` | `record()`, `start_recording()` and `RecordedRun`: send each step without ever raising into the agent. |
|
|
233
|
+
| `wrap` | `wrap()`: record every completion made through an async OpenAI client. |
|
|
234
|
+
| `worker` | `Worker` and `ReplayContext`: lease, replay, check, submit. |
|
|
235
|
+
| `diagnose` | `diagnose()` and `wait_for_diagnosis()`. |
|
|
236
|
+
| `openai_format` | Convert between OpenAI message params and First Domino messages. |
|
|
237
|
+
| `openai_agents` | Record and replay OpenAI Agents SDK agents (`first-domino[agents]`). |
|
|
238
|
+
| `responses_format` | Convert between Responses input items and First Domino messages. |
|
|
239
|
+
| `client` | `FirstDominoClient`, the typed async HTTP client. |
|
|
240
|
+
| `settings` | `SdkSettings`, read with the `FIRST_DOMINO_` prefix. |
|
|
241
|
+
| `errors` | `ConfigurationError` and `ApiRequestError` (with the API's stable `code`). |
|
|
242
|
+
| `testing` | An in-memory API for tests. |
|
|
243
|
+
| `_generated/` | Wire models generated from `api/openapi.json` by `make sdk-models`. Do not edit. |
|
|
@@ -0,0 +1,212 @@
|
|
|
1
|
+
# first-domino
|
|
2
|
+
|
|
3
|
+
Find the step where a failed AI agent run went wrong, and prove it.
|
|
4
|
+
|
|
5
|
+
First Domino records every run of your agent, live. When a run fails, it replays the run from its own save points to find the first step after which the run always fails (the first domino), then rewrites that one step and replays again to prove the fix. Replays run in a worker on your side, so your code, tools and secrets never leave your machines.
|
|
6
|
+
|
|
7
|
+
Works with any agent that calls a model through the OpenAI or Anthropic SDK, or a framework built on them: LangChain, the OpenAI Agents SDK, LlamaIndex, CrewAI, or a plain loop. Sync or async, streaming or not, and any OpenAI-compatible API.
|
|
8
|
+
|
|
9
|
+
## Install
|
|
10
|
+
|
|
11
|
+
```bash
|
|
12
|
+
pip install first-domino
|
|
13
|
+
```
|
|
14
|
+
|
|
15
|
+
Python 3.12+. Set the key from your project's Settings page:
|
|
16
|
+
|
|
17
|
+
```bash
|
|
18
|
+
export FIRST_DOMINO_API_KEY=fd_live_...
|
|
19
|
+
```
|
|
20
|
+
|
|
21
|
+
Without a key the SDK does nothing, so the same code runs in tests and in production.
|
|
22
|
+
|
|
23
|
+
## 1. Record your agent: two lines
|
|
24
|
+
|
|
25
|
+
Mark the function that runs your agent. Each call becomes one run, and every model call inside it becomes a step, as it happens.
|
|
26
|
+
|
|
27
|
+
```python
|
|
28
|
+
import first_domino as fd
|
|
29
|
+
|
|
30
|
+
@fd.entrypoint("support-bot", check=lambda question, answer: "refund" in answer)
|
|
31
|
+
def answer(question: str) -> str:
|
|
32
|
+
... # your agent, unchanged
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
- `check(output)` or `check(input, output)` says whether the run passed. A run that raises is recorded as `errored`, and your exception continues.
|
|
36
|
+
- Already know the answer inside your code? Call `fd.set_outcome(passed=False, reason="...")`; it wins over `check`.
|
|
37
|
+
- The function's arguments are saved with the run so it can be replayed. Use JSON-friendly types: text, numbers, lists, dicts, dataclasses or pydantic models.
|
|
38
|
+
- Recording never raises into your agent. If First Domino is unreachable, your code behaves exactly as without it.
|
|
39
|
+
- Failed runs are diagnosed on their own.
|
|
40
|
+
|
|
41
|
+
### Tools with side effects
|
|
42
|
+
|
|
43
|
+
During a replay, the steps before the one being tested come from the recording. Mark a tool that must not run twice (sends an email, moves money) so it returns its recorded result there instead:
|
|
44
|
+
|
|
45
|
+
```python
|
|
46
|
+
@fd.tool
|
|
47
|
+
def send_refund(order_id: str, amount: float) -> str:
|
|
48
|
+
...
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
Put it under your framework's own decorator (`@function_tool`, `@tool`). Read-only tools need no marking.
|
|
52
|
+
|
|
53
|
+
## 2. Start the replay worker
|
|
54
|
+
|
|
55
|
+
Run this next to your agent, with the same code and keys:
|
|
56
|
+
|
|
57
|
+
```python
|
|
58
|
+
import first_domino as fd
|
|
59
|
+
import my_agent # the module with your @fd.entrypoint
|
|
60
|
+
|
|
61
|
+
fd.serve()
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
The worker only connects outward. When First Domino needs a replay, the worker calls your function again: model calls up to the save point are answered from the recording, the next one with the fixed step when there is one, and the rest by the model. Your `check` decides whether the replay passed.
|
|
65
|
+
|
|
66
|
+
Async code can run `await fd.worker().run()` instead.
|
|
67
|
+
|
|
68
|
+
## Advanced
|
|
69
|
+
|
|
70
|
+
The two lines above cover most agents. These lower-level APIs remain for full control.
|
|
71
|
+
|
|
72
|
+
### Record runs by hand
|
|
73
|
+
|
|
74
|
+
Wrap your OpenAI client inside `fd.record`. Each completion becomes a step (the model turn plus the tool results it led to), and the tools you pass become the run's tool list. This is a complete call assistant loop:
|
|
75
|
+
|
|
76
|
+
```python
|
|
77
|
+
import first_domino as fd
|
|
78
|
+
from openai import AsyncOpenAI
|
|
79
|
+
|
|
80
|
+
openai = AsyncOpenAI()
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
async def answer_call(call_text: str, task_id: str) -> None:
|
|
84
|
+
task = fd.TaskInput(prompt=call_text, metadata={"task_id": task_id})
|
|
85
|
+
async with fd.record("call-assistant", task) as run:
|
|
86
|
+
client = fd.wrap(openai, run)
|
|
87
|
+
messages = [{"role": "system", "content": SYSTEM}, {"role": "user", "content": call_text}]
|
|
88
|
+
while True:
|
|
89
|
+
response = await client.chat.completions.create(
|
|
90
|
+
model=MODEL, messages=messages, tools=TOOLS
|
|
91
|
+
)
|
|
92
|
+
reply = response.choices[0].message
|
|
93
|
+
messages.append(reply.model_dump(exclude_none=True))
|
|
94
|
+
if not reply.tool_calls:
|
|
95
|
+
break
|
|
96
|
+
for call in reply.tool_calls:
|
|
97
|
+
result = run_tool(call)
|
|
98
|
+
messages.append({"role": "tool", "tool_call_id": call.id, "content": result})
|
|
99
|
+
outcome = fd.RunOutcome.passed if call_handled(task_id) else fd.RunOutcome.failed
|
|
100
|
+
await run.finish(outcome, final_output=reply.content)
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
- Recording never raises into your agent. If First Domino is unreachable, your code behaves exactly as without it.
|
|
104
|
+
- If the block raises, the run is finished as `errored` and your exception continues.
|
|
105
|
+
- Put what your success check needs (here `task_id`) in `TaskInput.metadata`. Replays get it back.
|
|
106
|
+
- A run finished as `failed` is diagnosed automatically.
|
|
107
|
+
- Not using OpenAI's client directly? Call `await run.step(model=..., output=..., tool_results=..., messages=...)` once per model turn. `fd.message_from_openai` converts OpenAI message dicts.
|
|
108
|
+
|
|
109
|
+
### Agents built with the OpenAI Agents SDK
|
|
110
|
+
|
|
111
|
+
Install `first-domino[agents]` and run the agent through the adapter. Every model turn is recorded as a step, and a replay resumes `Runner.run` from the save point:
|
|
112
|
+
|
|
113
|
+
```python
|
|
114
|
+
from first_domino import openai_agents as fd_agents
|
|
115
|
+
|
|
116
|
+
async with fd.record("call-assistant", task) as run:
|
|
117
|
+
result = await fd_agents.run_recorded(agent, call_text, run)
|
|
118
|
+
await run.finish(fd.RunOutcome.passed if handled(result) else fd.RunOutcome.failed)
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
@worker.agent("call-assistant")
|
|
122
|
+
async def replay(ctx: fd.ReplayContext) -> RunResult:
|
|
123
|
+
return await fd_agents.replay(ctx, agent) # replacement output and model override included
|
|
124
|
+
```
|
|
125
|
+
|
|
126
|
+
Replays resume from the recorded messages; your tools only see new calls, so restore any state they need for the save point first. Streamed turns (`Runner.run_streamed`) are not recorded.
|
|
127
|
+
|
|
128
|
+
### Write the replay worker by hand
|
|
129
|
+
|
|
130
|
+
The worker only calls out. It leases replay jobs, runs your agent from the save point it is given, runs your success check, and reports the outcome.
|
|
131
|
+
|
|
132
|
+
```python
|
|
133
|
+
worker = fd.Worker(agent_names=["call-assistant"])
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
@worker.agent("call-assistant")
|
|
137
|
+
async def replay(ctx: fd.ReplayContext) -> str:
|
|
138
|
+
# Continue from ctx.messages (from_step steps already done). When ctx.replacement_output
|
|
139
|
+
# is set, use it as the next model turn instead of calling the model. When
|
|
140
|
+
# ctx.model_override is set, use that model. Record each step with ctx.run.step(...).
|
|
141
|
+
task_id = ctx.task_input.metadata["task_id"]
|
|
142
|
+
await continue_call(fd.messages_to_openai(ctx.messages), ctx)
|
|
143
|
+
return task_id
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
@worker.success_check("call-assistant")
|
|
147
|
+
def check(task_input: fd.TaskInput, task_id: str) -> bool:
|
|
148
|
+
return call_handled(task_id)
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
await worker.run() # leases forever, several jobs at a time
|
|
152
|
+
```
|
|
153
|
+
|
|
154
|
+
- One failed job never stops the worker: a crash in your agent or check is reported as `errored` and the loop goes on.
|
|
155
|
+
- The worker refuses to start without an API key or without both handlers for each agent name.
|
|
156
|
+
- `FIRST_DOMINO_WORKER_CONCURRENCY` (default 4) sets how many replays run at once.
|
|
157
|
+
|
|
158
|
+
## Diagnose from code
|
|
159
|
+
|
|
160
|
+
Failed runs are diagnosed automatically. To start one yourself and wait for the verdict:
|
|
161
|
+
|
|
162
|
+
```python
|
|
163
|
+
diagnosis = await fd.diagnose(run_id)
|
|
164
|
+
async with asyncio.timeout(600):
|
|
165
|
+
final = await fd.wait_for_diagnosis(diagnosis.id)
|
|
166
|
+
print(final.verdict_kind, final.first_domino_step, final.explanation.title)
|
|
167
|
+
# step_verified 6 Converted AM appointment to PM
|
|
168
|
+
```
|
|
169
|
+
|
|
170
|
+
## Test without a server
|
|
171
|
+
|
|
172
|
+
`first_domino.testing` is an in-memory API, so you can test your worker with no network:
|
|
173
|
+
|
|
174
|
+
```python
|
|
175
|
+
from first_domino.testing import FakeApi, fake_client, make_job
|
|
176
|
+
|
|
177
|
+
api = FakeApi()
|
|
178
|
+
api.jobs.append(make_job(agent_name="call-assistant", from_step=2))
|
|
179
|
+
worker = fd.Worker(agent_names=["call-assistant"], client=fake_client(api))
|
|
180
|
+
# register @worker.agent and @worker.success_check as above
|
|
181
|
+
assert await worker.run_once() == fd.ReplayOutcome.passed
|
|
182
|
+
```
|
|
183
|
+
|
|
184
|
+
## Reference
|
|
185
|
+
|
|
186
|
+
| Setting | Meaning | Default |
|
|
187
|
+
| --- | --- | --- |
|
|
188
|
+
| `FIRST_DOMINO_API_KEY` | Project API key. Without it, recording does nothing. | none |
|
|
189
|
+
| `FIRST_DOMINO_BASE_URL` | The First Domino API. Point it at your own server if you self-host. | `https://api-first-domino.abhiramtech.in` |
|
|
190
|
+
| `FIRST_DOMINO_WORKER_CONCURRENCY` | Replays one worker runs at once. | `4` |
|
|
191
|
+
| `FIRST_DOMINO_LEASE_WAIT_SECONDS` | How long one lease call waits for a job (0 to 20). | `10` |
|
|
192
|
+
| `FIRST_DOMINO_TIMEOUT_SECONDS` | Timeout of one API call. | `30` |
|
|
193
|
+
|
|
194
|
+
Settings can also live in a `.env` file in the working directory.
|
|
195
|
+
|
|
196
|
+
| Module | Job |
|
|
197
|
+
| --- | --- |
|
|
198
|
+
| `instrument` | `entrypoint()`, `tool()`, `set_outcome()`, `serve()`, `worker()` and `init()`: the two-line integration. |
|
|
199
|
+
| `capture` | Records model calls at the HTTP layer (`httpx` and `httpx2`) and answers them during a replay. |
|
|
200
|
+
| `providers` | The OpenAI Chat Completions, OpenAI Responses and Anthropic Messages wire formats. |
|
|
201
|
+
| `recorder` | `record()`, `start_recording()` and `RecordedRun`: send each step without ever raising into the agent. |
|
|
202
|
+
| `wrap` | `wrap()`: record every completion made through an async OpenAI client. |
|
|
203
|
+
| `worker` | `Worker` and `ReplayContext`: lease, replay, check, submit. |
|
|
204
|
+
| `diagnose` | `diagnose()` and `wait_for_diagnosis()`. |
|
|
205
|
+
| `openai_format` | Convert between OpenAI message params and First Domino messages. |
|
|
206
|
+
| `openai_agents` | Record and replay OpenAI Agents SDK agents (`first-domino[agents]`). |
|
|
207
|
+
| `responses_format` | Convert between Responses input items and First Domino messages. |
|
|
208
|
+
| `client` | `FirstDominoClient`, the typed async HTTP client. |
|
|
209
|
+
| `settings` | `SdkSettings`, read with the `FIRST_DOMINO_` prefix. |
|
|
210
|
+
| `errors` | `ConfigurationError` and `ApiRequestError` (with the API's stable `code`). |
|
|
211
|
+
| `testing` | An in-memory API for tests. |
|
|
212
|
+
| `_generated/` | Wire models generated from `api/openapi.json` by `make sdk-models`. Do not edit. |
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "first-domino"
|
|
3
|
+
version = "0.1.0"
|
|
4
|
+
description = "Find the step where a failed AI agent run went wrong, and prove it with a replay."
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
license = "MIT"
|
|
7
|
+
license-files = ["LICENSE"]
|
|
8
|
+
authors = [{ name = "Jaini Sai Abhiram" }]
|
|
9
|
+
requires-python = ">=3.12"
|
|
10
|
+
keywords = ["ai", "agents", "llm", "debugging", "observability", "evals", "openai", "anthropic", "langchain"]
|
|
11
|
+
classifiers = [
|
|
12
|
+
"Development Status :: 4 - Beta",
|
|
13
|
+
"Intended Audience :: Developers",
|
|
14
|
+
"Operating System :: OS Independent",
|
|
15
|
+
"Programming Language :: Python :: 3",
|
|
16
|
+
"Programming Language :: Python :: 3.12",
|
|
17
|
+
"Programming Language :: Python :: 3.13",
|
|
18
|
+
"Topic :: Scientific/Engineering :: Artificial Intelligence",
|
|
19
|
+
"Topic :: Software Development :: Debuggers",
|
|
20
|
+
"Typing :: Typed",
|
|
21
|
+
]
|
|
22
|
+
dependencies = [
|
|
23
|
+
"httpx>=0.27",
|
|
24
|
+
"pydantic>=2.9",
|
|
25
|
+
"pydantic-settings>=2.6",
|
|
26
|
+
]
|
|
27
|
+
|
|
28
|
+
[project.optional-dependencies]
|
|
29
|
+
openai = [
|
|
30
|
+
"openai>=1.55",
|
|
31
|
+
]
|
|
32
|
+
agents = [
|
|
33
|
+
"openai-agents>=0.25.0",
|
|
34
|
+
]
|
|
35
|
+
|
|
36
|
+
[project.urls]
|
|
37
|
+
Homepage = "https://first-domino.abhiramtech.in"
|
|
38
|
+
Repository = "https://github.com/saiabhiramjaini/first-domino"
|
|
39
|
+
Documentation = "https://github.com/saiabhiramjaini/first-domino/tree/main/server/packages/first-domino#readme"
|
|
40
|
+
Issues = "https://github.com/saiabhiramjaini/first-domino/issues"
|
|
41
|
+
|
|
42
|
+
[build-system]
|
|
43
|
+
requires = ["hatchling>=1.27"]
|
|
44
|
+
build-backend = "hatchling.build"
|
|
45
|
+
|
|
46
|
+
[tool.hatch.build.targets.wheel]
|
|
47
|
+
packages = ["src/first_domino"]
|
|
48
|
+
|
|
49
|
+
[tool.hatch.build.targets.sdist]
|
|
50
|
+
include = ["src/first_domino", "README.md", "LICENSE", "pyproject.toml"]
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
"""First Domino SDK: record agent runs and replay them on command.
|
|
2
|
+
|
|
3
|
+
Mark the function that runs your agent, and every model call inside it is recorded,
|
|
4
|
+
whatever framework makes it. Then `fd.serve()` lets First Domino replay it.
|
|
5
|
+
|
|
6
|
+
Example:
|
|
7
|
+
>>> import first_domino as fd
|
|
8
|
+
>>> @fd.entrypoint("support-bot", check=lambda question, answer: "refund" in answer)
|
|
9
|
+
... def answer(question: str) -> str: ...
|
|
10
|
+
>>> fd.serve() # in a separate process: replays runs when First Domino asks
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from first_domino._generated.models import (
|
|
14
|
+
Diagnosis,
|
|
15
|
+
Message,
|
|
16
|
+
MessageRole,
|
|
17
|
+
ReplayOutcome,
|
|
18
|
+
RunOutcome,
|
|
19
|
+
TaskInput,
|
|
20
|
+
ToolCall,
|
|
21
|
+
)
|
|
22
|
+
from first_domino.client import FirstDominoClient
|
|
23
|
+
from first_domino.diagnose import diagnose, wait_for_diagnosis
|
|
24
|
+
from first_domino.errors import ApiRequestError, ConfigurationError, FirstDominoError
|
|
25
|
+
from first_domino.instrument import entrypoint, init, serve, set_outcome, tool, worker
|
|
26
|
+
from first_domino.openai_format import message_from_openai, message_to_openai, messages_to_openai
|
|
27
|
+
from first_domino.recorder import RecordedRun, record, start_recording
|
|
28
|
+
from first_domino.settings import SdkSettings
|
|
29
|
+
from first_domino.worker import ReplayContext, Worker
|
|
30
|
+
from first_domino.wrap import wrap
|
|
31
|
+
|
|
32
|
+
__all__ = [
|
|
33
|
+
"ApiRequestError",
|
|
34
|
+
"ConfigurationError",
|
|
35
|
+
"Diagnosis",
|
|
36
|
+
"FirstDominoClient",
|
|
37
|
+
"FirstDominoError",
|
|
38
|
+
"Message",
|
|
39
|
+
"MessageRole",
|
|
40
|
+
"RecordedRun",
|
|
41
|
+
"ReplayContext",
|
|
42
|
+
"ReplayOutcome",
|
|
43
|
+
"RunOutcome",
|
|
44
|
+
"SdkSettings",
|
|
45
|
+
"TaskInput",
|
|
46
|
+
"ToolCall",
|
|
47
|
+
"Worker",
|
|
48
|
+
"diagnose",
|
|
49
|
+
"entrypoint",
|
|
50
|
+
"init",
|
|
51
|
+
"message_from_openai",
|
|
52
|
+
"message_to_openai",
|
|
53
|
+
"messages_to_openai",
|
|
54
|
+
"record",
|
|
55
|
+
"serve",
|
|
56
|
+
"set_outcome",
|
|
57
|
+
"start_recording",
|
|
58
|
+
"tool",
|
|
59
|
+
"wait_for_diagnosis",
|
|
60
|
+
"worker",
|
|
61
|
+
"wrap",
|
|
62
|
+
]
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Wire models generated from api/openapi.json by `make sdk-models`. Do not edit."""
|