stepline 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- stepline-0.1.0/.gitignore +20 -0
- stepline-0.1.0/CHANGELOG.md +35 -0
- stepline-0.1.0/LICENSE +21 -0
- stepline-0.1.0/PKG-INFO +274 -0
- stepline-0.1.0/README.md +242 -0
- stepline-0.1.0/examples/weather_agent.py +158 -0
- stepline-0.1.0/frontend/README.md +21 -0
- stepline-0.1.0/pyproject.toml +104 -0
- stepline-0.1.0/src/stepline/__init__.py +28 -0
- stepline-0.1.0/src/stepline/_util.py +98 -0
- stepline-0.1.0/src/stepline/cli/__init__.py +375 -0
- stepline-0.1.0/src/stepline/cli/render.py +511 -0
- stepline-0.1.0/src/stepline/config.py +86 -0
- stepline-0.1.0/src/stepline/demo/__init__.py +19 -0
- stepline-0.1.0/src/stepline/demo/mock_api.py +238 -0
- stepline-0.1.0/src/stepline/py.typed +0 -0
- stepline-0.1.0/src/stepline/recorder/__init__.py +54 -0
- stepline-0.1.0/src/stepline/recorder/anthropic_adapter.py +517 -0
- stepline-0.1.0/src/stepline/recorder/base.py +167 -0
- stepline-0.1.0/src/stepline/recorder/pricing.py +246 -0
- stepline-0.1.0/src/stepline/recorder/recorder.py +423 -0
- stepline-0.1.0/src/stepline/recorder/redact.py +66 -0
- stepline-0.1.0/src/stepline/recorder/streaming.py +121 -0
- stepline-0.1.0/src/stepline/replay.py +496 -0
- stepline-0.1.0/src/stepline/server/__init__.py +5 -0
- stepline-0.1.0/src/stepline/server/app.py +401 -0
- stepline-0.1.0/src/stepline/server/schemas.py +272 -0
- stepline-0.1.0/src/stepline/server/security.py +185 -0
- stepline-0.1.0/src/stepline/server/static/assets/index-BWeF02Jm.js +49 -0
- stepline-0.1.0/src/stepline/server/static/assets/index-CCBg99Nd.css +2 -0
- stepline-0.1.0/src/stepline/server/static/assets/inter-cyrillic-ext-wght-normal-BOeWTOD4.woff2 +0 -0
- stepline-0.1.0/src/stepline/server/static/assets/inter-cyrillic-wght-normal-DqGufNeO.woff2 +0 -0
- stepline-0.1.0/src/stepline/server/static/assets/inter-greek-ext-wght-normal-DlzME5K_.woff2 +0 -0
- stepline-0.1.0/src/stepline/server/static/assets/inter-greek-wght-normal-CkhJZR-_.woff2 +0 -0
- stepline-0.1.0/src/stepline/server/static/assets/inter-latin-ext-wght-normal-DO1Apj_S.woff2 +0 -0
- stepline-0.1.0/src/stepline/server/static/assets/inter-latin-wght-normal-Dx4kXJAl.woff2 +0 -0
- stepline-0.1.0/src/stepline/server/static/assets/inter-vietnamese-wght-normal-CBcvBZtf.woff2 +0 -0
- stepline-0.1.0/src/stepline/server/static/assets/jetbrains-mono-cyrillic-ext-wght-normal-EocZY2iu.woff2 +0 -0
- stepline-0.1.0/src/stepline/server/static/assets/jetbrains-mono-cyrillic-wght-normal-D73BlboJ.woff2 +0 -0
- stepline-0.1.0/src/stepline/server/static/assets/jetbrains-mono-greek-wght-normal-Bw9x6K1M.woff2 +0 -0
- stepline-0.1.0/src/stepline/server/static/assets/jetbrains-mono-latin-ext-wght-normal-DBQx-q_a.woff2 +0 -0
- stepline-0.1.0/src/stepline/server/static/assets/jetbrains-mono-latin-wght-normal-B9CIFXIH.woff2 +0 -0
- stepline-0.1.0/src/stepline/server/static/assets/jetbrains-mono-vietnamese-wght-normal-Bt-aOZkq.woff2 +0 -0
- stepline-0.1.0/src/stepline/server/static/favicon.svg +1 -0
- stepline-0.1.0/src/stepline/server/static/index.html +15 -0
- stepline-0.1.0/src/stepline/storage/__init__.py +13 -0
- stepline-0.1.0/src/stepline/storage/db.py +149 -0
- stepline-0.1.0/src/stepline/storage/digest.py +121 -0
- stepline-0.1.0/src/stepline/storage/models.py +142 -0
- stepline-0.1.0/src/stepline/storage/store.py +575 -0
- stepline-0.1.0/tests/conftest.py +62 -0
- stepline-0.1.0/tests/fakes.py +231 -0
- stepline-0.1.0/tests/test_cli.py +240 -0
- stepline-0.1.0/tests/test_demo.py +105 -0
- stepline-0.1.0/tests/test_pricing_redact.py +194 -0
- stepline-0.1.0/tests/test_recorder_async.py +87 -0
- stepline-0.1.0/tests/test_recorder_sync.py +470 -0
- stepline-0.1.0/tests/test_replay.py +530 -0
- stepline-0.1.0/tests/test_sdk_contract.py +116 -0
- stepline-0.1.0/tests/test_server_api.py +444 -0
- stepline-0.1.0/tests/test_server_security.py +310 -0
- stepline-0.1.0/tests/test_storage.py +264 -0
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
__pycache__/
|
|
2
|
+
*.py[cod]
|
|
3
|
+
.venv/
|
|
4
|
+
dist/
|
|
5
|
+
build/
|
|
6
|
+
*.egg-info/
|
|
7
|
+
.pytest_cache/
|
|
8
|
+
.mypy_cache/
|
|
9
|
+
.ruff_cache/
|
|
10
|
+
.coverage
|
|
11
|
+
.stepline/
|
|
12
|
+
node_modules/
|
|
13
|
+
src/stepline/server/static/
|
|
14
|
+
!src/stepline/server/static/.gitkeep
|
|
15
|
+
.DS_Store
|
|
16
|
+
frontend/test-results/
|
|
17
|
+
frontend/playwright-report/
|
|
18
|
+
frontend/node_modules/.tmp/
|
|
19
|
+
frontend/*.tsbuildinfo
|
|
20
|
+
.venv-oldest/
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to this project are documented here. The format follows
|
|
4
|
+
[Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and the project uses
|
|
5
|
+
[Semantic Versioning](https://semver.org/).
|
|
6
|
+
|
|
7
|
+
## [0.1.0] - Unreleased
|
|
8
|
+
|
|
9
|
+
First release. The PyPI package, the import and the CLI command are all `stepline`.
|
|
10
|
+
|
|
11
|
+
### Added
|
|
12
|
+
|
|
13
|
+
- **Recorder:** `stepline.wrap(client)` for the Anthropic Python SDK (`anthropic>=0.40,<3`),
|
|
14
|
+
sync and async. Records `messages.create` (including `stream=True`), `messages.stream` and
|
|
15
|
+
`messages.parse` on `client.messages` and `client.beta.messages`: request, response, tool
|
|
16
|
+
calls with their linked results, tokens (including cache), estimated cost, latency, time to
|
|
17
|
+
first token, and errors. It uses a background writer and flushes at exit, never raises into
|
|
18
|
+
your code, and redacts key-like strings without ever storing headers.
|
|
19
|
+
- `stepline.run(name)` context manager/decorator for named runs; `stepline.configure()`;
|
|
20
|
+
`STEPLINE_DISABLED`, `STEPLINE_DB` and `STEPLINE_DIR`.
|
|
21
|
+
- **Storage:** SQLite (`./.stepline/traces.db`, WAL) with versioned migrations; runs, and steps
|
|
22
|
+
linked into a tree by `parent_step_id`.
|
|
23
|
+
- **Cost estimation** for Claude models, with overrides in `.stepline/prices.toml`.
|
|
24
|
+
- **CLI:** `stepline ls`, `show`, `rm`, `prices`, `demo`, `ui`, `openapi`.
|
|
25
|
+
- **Local web UI:** run list, a virtualized timeline with per-step tokens, cost, latency and
|
|
26
|
+
a waterfall, a step panel (messages, tool calls, request, raw JSON), run diffs, dark and
|
|
27
|
+
light themes, keyboard navigation.
|
|
28
|
+
- **Replay:** edit a step (model, system prompt, messages, tools, parameters), review a token
|
|
29
|
+
and cost estimate, and send one call. The result is saved as a forked run; API errors are
|
|
30
|
+
kept as failed steps. `stepline ui --mock` replays against an offline mock.
|
|
31
|
+
- **Security model** for the UI server: loopback bind, Host and Origin checks, no CORS, a
|
|
32
|
+
custom header required for writes, strict CSP with Trusted Types. See `SECURITY.md`.
|
|
33
|
+
- **Example agent** (`examples/weather_agent.py`) with an offline mock mode; `stepline demo`.
|
|
34
|
+
|
|
35
|
+
[0.1.0]: https://github.com/dapdap777/stepline/releases/tag/v0.1.0
|
stepline-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Stepline contributors
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
stepline-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,274 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: stepline
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Time-travel debugger for AI agents: record every LLM call, inspect it in a local web UI, and replay from any step.
|
|
5
|
+
Project-URL: Homepage, https://github.com/dapdap777/stepline
|
|
6
|
+
Project-URL: Issues, https://github.com/dapdap777/stepline/issues
|
|
7
|
+
Project-URL: Changelog, https://github.com/dapdap777/stepline/blob/main/CHANGELOG.md
|
|
8
|
+
Project-URL: Security, https://github.com/dapdap777/stepline/blob/main/SECURITY.md
|
|
9
|
+
Author: Stepline contributors
|
|
10
|
+
License-Expression: MIT
|
|
11
|
+
License-File: LICENSE
|
|
12
|
+
Keywords: agents,anthropic,claude,debugger,llm,observability,tracing
|
|
13
|
+
Classifier: Development Status :: 3 - Alpha
|
|
14
|
+
Classifier: Intended Audience :: Developers
|
|
15
|
+
Classifier: Operating System :: OS Independent
|
|
16
|
+
Classifier: Programming Language :: Python :: 3
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
21
|
+
Classifier: Topic :: Software Development :: Debuggers
|
|
22
|
+
Classifier: Typing :: Typed
|
|
23
|
+
Requires-Python: >=3.10
|
|
24
|
+
Requires-Dist: click>=8.1
|
|
25
|
+
Requires-Dist: fastapi>=0.110
|
|
26
|
+
Requires-Dist: rich>=13.0
|
|
27
|
+
Requires-Dist: tomli>=1.1; python_version < '3.11'
|
|
28
|
+
Requires-Dist: uvicorn>=0.27
|
|
29
|
+
Provides-Extra: anthropic
|
|
30
|
+
Requires-Dist: anthropic<3,>=0.40; extra == 'anthropic'
|
|
31
|
+
Description-Content-Type: text/markdown
|
|
32
|
+
|
|
33
|
+
# Stepline
|
|
34
|
+
|
|
35
|
+
**A time-travel debugger for AI agents: record every LLM call, inspect each step in a local web UI, then edit any step and replay it as a new branch.**
|
|
36
|
+
|
|
37
|
+
[](https://github.com/dapdap777/stepline/actions/workflows/ci.yml)
|
|
38
|
+
[](LICENSE)
|
|
39
|
+
|
|
40
|
+
<img src="docs/assets/demo-placeholder.svg" alt="Demo GIF placeholder: record an agent run, inspect every step, edit and replay one step" width="100%">
|
|
41
|
+
|
|
42
|
+
## Quickstart (30 seconds, no API key)
|
|
43
|
+
|
|
44
|
+
```bash
|
|
45
|
+
pip install "stepline[anthropic]"
|
|
46
|
+
stepline demo # runs an example agent against an offline mock API and records it
|
|
47
|
+
stepline ui # opens http://127.0.0.1:7777
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
Then record your own agent. These are the only two lines you add:
|
|
51
|
+
|
|
52
|
+
```python
|
|
53
|
+
import stepline
|
|
54
|
+
client = stepline.wrap(Anthropic()) # use `client` exactly as before
|
|
55
|
+
```
|
|
56
|
+
|
|
57
|
+
Run your agent as usual, then `stepline ui` (or `stepline show` in the terminal).
|
|
58
|
+
|
|
59
|
+
> **Not on PyPI yet.** Stepline hasn't had its first release. Until then,
|
|
60
|
+
> [install from source](#install-from-source).
|
|
61
|
+
|
|
62
|
+
---
|
|
63
|
+
|
|
64
|
+
## What you get
|
|
65
|
+
|
|
66
|
+
**A timeline of every call:** tokens, cost and latency per step, plus a waterfall showing when
|
|
67
|
+
each call ran. The gaps between bars are time spent in your own code, such as running tools.
|
|
68
|
+
Tool calls appear next to the results your code returned, and errors stand out.
|
|
69
|
+
|
|
70
|
+

|
|
71
|
+
|
|
72
|
+
**Replay any step with edits.** Change the system prompt, messages, tools, parameters or the
|
|
73
|
+
model, check the estimated cost, and send. The result is saved as a fork of the original run.
|
|
74
|
+
|
|
75
|
+

|
|
76
|
+
|
|
77
|
+
**Diff a fork against its parent:** step-by-step alignment plus word-level diffs of prompts,
|
|
78
|
+
parameters and responses.
|
|
79
|
+
|
|
80
|
+

|
|
81
|
+
|
|
82
|
+
**Or stay in the terminal:**
|
|
83
|
+
|
|
84
|
+

|
|
85
|
+
|
|
86
|
+
<sub>Screenshots come from `stepline demo` (the example agent below), viewed with
|
|
87
|
+
`stepline ui --mock` so the replay also ran offline.</sub>
|
|
88
|
+
|
|
89
|
+
## Features
|
|
90
|
+
|
|
91
|
+
Everything listed here exists and is covered by the test suite.
|
|
92
|
+
|
|
93
|
+
- **Two-line recording** for the Anthropic Python SDK, sync (`Anthropic`) and async
|
|
94
|
+
(`AsyncAnthropic`). It records `messages.create` (including `stream=True`),
|
|
95
|
+
`messages.stream` and `messages.parse`, on both `client.messages` and
|
|
96
|
+
`client.beta.messages`. `wrap()` returns the same client object.
|
|
97
|
+
- **What's captured per call:** the exact request (messages, system prompt, tools, parameters),
|
|
98
|
+
the response, tool calls linked to the tool results your code sent back, token usage
|
|
99
|
+
(including prompt-cache reads and writes), estimated cost, latency, time to first token for
|
|
100
|
+
streams, and errors.
|
|
101
|
+
- **Never breaks your app.** Recording happens on a background thread and adds well under a
|
|
102
|
+
millisecond per call. Errors in the recorder are logged and swallowed, and API exceptions
|
|
103
|
+
reach your code unchanged. Pending writes are flushed when the process exits.
|
|
104
|
+
- **Local storage:** SQLite at `./.stepline/traces.db` (WAL mode), plus a `.gitignore` written
|
|
105
|
+
into `.stepline/` so traces aren't committed by accident.
|
|
106
|
+
- **Web UI:** run list with search and filters; a virtualized timeline, tested with 3,000 steps;
|
|
107
|
+
a step panel showing full messages and tool calls; run-to-run diffs; dark (default) and light
|
|
108
|
+
themes; keyboard navigation (`j`/`k`, `g`/`G`, `r`, `?`).
|
|
109
|
+
- **Replay:** edit a step and re-send that one call, after a cost estimate and an explicit
|
|
110
|
+
confirmation. The result is saved as a new forked run, and the original run is never
|
|
111
|
+
modified. API errors are kept as failed steps. `stepline ui --mock` lets you try replay
|
|
112
|
+
offline.
|
|
113
|
+
- **CLI:** `stepline ls`, `show [RUN] [--step N]`, `rm`, `prices`, `demo`, `ui`, `openapi`.
|
|
114
|
+
Runs can be referred to by id, unique id prefix, or name.
|
|
115
|
+
- **Cost estimates** from a built-in price table for Claude models. You can override or extend
|
|
116
|
+
it in `.stepline/prices.toml`.
|
|
117
|
+
|
|
118
|
+
## Limits
|
|
119
|
+
|
|
120
|
+
- **Anthropic Python SDK only, for now** (`anthropic>=0.40,<3`; CI tests both ends of that
|
|
121
|
+
range). OpenAI is next on the roadmap. Other SDKs, frameworks and languages aren't recorded.
|
|
122
|
+
Bedrock, Vertex and Foundry clients from the `anthropic` package use the same interface, but
|
|
123
|
+
they aren't tested yet.
|
|
124
|
+
- **Replay re-sends one LLM call. It does not re-run your tools or your agent.** A fork
|
|
125
|
+
therefore ends at the replayed step, even if the new response asks for a tool.
|
|
126
|
+
- **Costs are estimates** based on first-party list prices. They don't account for batch
|
|
127
|
+
discounts, partner-platform pricing or negotiated rates.
|
|
128
|
+
- **Single machine, single user.** It's a local development tool, not production monitoring,
|
|
129
|
+
and there's no sharing or team view.
|
|
130
|
+
- **Traces contain your prompts and model outputs.** Strings that look like API keys are
|
|
131
|
+
redacted, and request headers are never stored, but redaction is best-effort. Treat
|
|
132
|
+
`.stepline/` as sensitive. See [SECURITY.md](SECURITY.md).
|
|
133
|
+
|
|
134
|
+
## Why Stepline
|
|
135
|
+
|
|
136
|
+
Hosted LLM tracing and observability platforms are the right tool for production. They offer
|
|
137
|
+
team dashboards, monitoring, evaluations and SDKs for many languages, and several are open
|
|
138
|
+
source and can be self-hosted. Stepline is deliberately narrower: a debugger for the edit-run-
|
|
139
|
+
inspect loop on your own machine.
|
|
140
|
+
|
|
141
|
+
| | Stepline | Hosted tracing platforms (typical) |
|
|
142
|
+
|---|---|---|
|
|
143
|
+
| Setup | `pip install`, two lines of code | Account, project and API key, then an SDK or proxy |
|
|
144
|
+
| Where traces live | A SQLite file in your project | The vendor's cloud, or your own deployment |
|
|
145
|
+
| Telemetry | None. The UI makes no requests outside localhost | Varies by vendor |
|
|
146
|
+
| Re-running a step | Edit any recorded call and replay it as a forked branch, then diff against the original | Many offer a playground to re-run a prompt |
|
|
147
|
+
| Scope | One developer, one machine, Anthropic Python SDK | Teams, production traffic, many SDKs and frameworks |
|
|
148
|
+
|
|
149
|
+
### Similar tools for recording and replaying agent runs
|
|
150
|
+
|
|
151
|
+
Several open-source projects also record agent runs and let you replay or fork them. The table
|
|
152
|
+
summarizes what each project's own README says it does. It isn't an independent evaluation, and
|
|
153
|
+
it may be out of date.
|
|
154
|
+
|
|
155
|
+
| Project | How it records | What replay means there | Main focus |
|
|
156
|
+
|---|---|---|---|
|
|
157
|
+
| [Rewind](https://github.com/llm-rewind/rewind) (`llm-rewind`) | Local HTTPS proxy (mitmproxy, needs a local CA certificate) for OpenAI, Anthropic and Gemini, in any language | Deterministic cassette replay of a whole run, at zero LLM cost | Finding *why* two runs diverged (`rewind bisect`) and mutation-testing agents (`rewind mutate`), plus pytest |
|
|
158
|
+
| [AgentTape](https://github.com/MITHRAN-BALACHANDER/AgentTape) | LLM calls (automatic OpenAI interception) and tools wrapped with `@agenttape.tool`, saved as YAML cassettes | Deterministic offline replay; tools never re-execute. Partial replay can run the LLM live against a new prompt with tools frozen | Fast, safe agent tests in CI, plus a CLI with an ASCII waterfall, diff and an HTML viewer |
|
|
159
|
+
| [Stepfork](https://github.com/utsab345/stepfork) | Instrumented tool calls (`@trace_tool`) and LLM requests; adapters for OpenAI chat completions (non-streaming) and LangGraph | Frozen replay of your entrypoint; hybrid replay re-runs selected LLM calls; fork-at-step with output overrides | Turning a failed run into a pytest regression test, with behavioral diffs and failure minimization |
|
|
160
|
+
| AgentRewind ([JWang-Langer](https://github.com/JWang-Langer/agentrewind)) | Request fingerprints for OpenAI-compatible upstreams; side effects go through an idempotent channel | Deterministic replay; `arw rewind` re-runs live from step N with changed settings | A CLI that pinpoints the first diverging step and explains the request-level differences; no visual dashboard |
|
|
161
|
+
| AgentRewind ([Akshatkasera](https://github.com/Akshatkasera/codex-community-hackathon-del-AgentRewind), a hackathon project) | Imports runs from common agent frameworks | Patch a broken step and replay the downstream branch | A FastAPI + React app that diagnoses failed multi-agent runs and turns fixes into regression tests |
|
|
162
|
+
|
|
163
|
+
What Stepline does differently, and what it doesn't do:
|
|
164
|
+
|
|
165
|
+
- **Recording is an in-process wrapper:** `stepline.wrap(client)` around the Anthropic Python
|
|
166
|
+
SDK. There's no proxy or certificate to install and no decorators on your tools. Streaming,
|
|
167
|
+
async and tool-result linking work automatically. The flip side is that only that SDK is
|
|
168
|
+
supported. Rewind's proxy covers more providers and any language.
|
|
169
|
+
- **Replay is a live, edited re-send of one call,** not a deterministic re-execution of your
|
|
170
|
+
program. You change the prompt, messages, tools, parameters or model, see the estimated cost,
|
|
171
|
+
and get a new branch you can diff against the original. Stepline doesn't capture or re-run
|
|
172
|
+
your tools, so a fork ends at that step. AgentTape, Stepfork and AgentRewind (JWang-Langer)
|
|
173
|
+
record tool calls or side effects and serve them back on replay. Both AgentRewind projects
|
|
174
|
+
can continue a run past the point you changed.
|
|
175
|
+
- **It's an interactive debugger, not a test harness.** The main interface is a local web UI
|
|
176
|
+
with a timeline of every step (tokens, cost, latency) and a step-by-step diff view. There's
|
|
177
|
+
no pytest export, cassette replay in CI, divergence-cause inference or mutation testing. If you
|
|
178
|
+
need those, the tools above are built for them.
|
|
179
|
+
- **It's local and needs no account,** like most of the open-source tools above.
|
|
180
|
+
|
|
181
|
+
### Other tools in this space
|
|
182
|
+
|
|
183
|
+
- [Retrace](https://retraceai.tech)
|
|
184
|
+
|
|
185
|
+
## How it works
|
|
186
|
+
|
|
187
|
+
```mermaid
|
|
188
|
+
flowchart LR
|
|
189
|
+
subgraph app["Your agent process"]
|
|
190
|
+
code["agent code"] -->|"client.messages.create / .stream"| wrapped["stepline.wrap(client)"]
|
|
191
|
+
wrapped -->|unchanged call| sdk["Anthropic SDK"]
|
|
192
|
+
wrapped -. "snapshot + timing<br/>(non-blocking queue)" .-> writer["background writer thread"]
|
|
193
|
+
end
|
|
194
|
+
sdk <--> anthropic[("Anthropic API")]
|
|
195
|
+
writer --> db[("./.stepline/traces.db<br/>SQLite: runs → steps")]
|
|
196
|
+
db --> cli["stepline ls / show"]
|
|
197
|
+
db --> server["stepline ui<br/>FastAPI on 127.0.0.1"]
|
|
198
|
+
server --> browser["Web UI<br/>(bundled React app)"]
|
|
199
|
+
browser -- "replay (explicit, one call)" --> server
|
|
200
|
+
server -- "edited request" --> anthropic
|
|
201
|
+
server -- "new forked run" --> db
|
|
202
|
+
```
|
|
203
|
+
|
|
204
|
+
- **Recorder.** `wrap()` patches the SDK's `messages` resources in place. A call snapshots its
|
|
205
|
+
request on your thread and passes straight through to the SDK. Streams are observed chunk by
|
|
206
|
+
chunk without being buffered. Normalization, redaction and SQL writes happen on a daemon
|
|
207
|
+
thread. SDK support sits behind a small adapter interface, so other SDKs can be added.
|
|
208
|
+
- **Storage.** A run has many steps. Each step points at the step before it
|
|
209
|
+
(`parent_step_id`), and a replay creates a new run whose step branches off the original's
|
|
210
|
+
parent. Together the runs form a tree.
|
|
211
|
+
- **Server and UI.** A FastAPI app that binds to loopback, checks Host and Origin on every
|
|
212
|
+
request, and serves a strict Content-Security-Policy. The UI is built into the Python
|
|
213
|
+
package, so nothing is fetched from a CDN.
|
|
214
|
+
|
|
215
|
+
## The example agent
|
|
216
|
+
|
|
217
|
+
[`examples/weather_agent.py`](examples/weather_agent.py) is a ~150-line tool-using agent with
|
|
218
|
+
three toy tools that return canned data. It runs streamed turns, makes parallel tool calls,
|
|
219
|
+
and handles a tool failure.
|
|
220
|
+
|
|
221
|
+
```bash
|
|
222
|
+
python examples/weather_agent.py --mock # offline mock API, no key needed
|
|
223
|
+
python examples/weather_agent.py # real API, needs ANTHROPIC_API_KEY (costs a few cents)
|
|
224
|
+
stepline ui
|
|
225
|
+
```
|
|
226
|
+
|
|
227
|
+
`stepline demo` runs the same file in mock mode.
|
|
228
|
+
|
|
229
|
+
## Configuration
|
|
230
|
+
|
|
231
|
+
Zero configuration is required. The options are in [docs/configuration.md](docs/configuration.md):
|
|
232
|
+
|
|
233
|
+
- where traces are stored (`STEPLINE_DB`, `STEPLINE_DIR`), and how to turn recording off
|
|
234
|
+
(`STEPLINE_DISABLED=1`);
|
|
235
|
+
- named runs: `with stepline.run("triage"): ...`;
|
|
236
|
+
- custom prices in `.stepline/prices.toml`;
|
|
237
|
+
- replay and API keys: the key is read from the environment when you replay and is never stored;
|
|
238
|
+
- the UI server's security model.
|
|
239
|
+
|
|
240
|
+
## Install from source
|
|
241
|
+
|
|
242
|
+
```bash
|
|
243
|
+
git clone https://github.com/dapdap777/stepline && cd stepline
|
|
244
|
+
uv sync # Python deps (Python 3.10+)
|
|
245
|
+
(cd frontend && npm ci && npm run build) # builds the UI into the package (Node 20+)
|
|
246
|
+
uv run stepline demo && uv run stepline ui
|
|
247
|
+
```
|
|
248
|
+
|
|
249
|
+
Without the frontend build, `stepline ui` serves a page explaining how to build it. The CLI
|
|
250
|
+
works either way.
|
|
251
|
+
|
|
252
|
+
## Roadmap
|
|
253
|
+
|
|
254
|
+
1. **OpenAI Python SDK adapter.** The recorder already goes through a `ProviderAdapter`
|
|
255
|
+
interface; Chat Completions and Responses come next.
|
|
256
|
+
2. **Replay that continues:** opt-in registration of your tool functions, so a fork can keep
|
|
257
|
+
running past the replayed step.
|
|
258
|
+
3. **Export and import a run** as a JSON file, to share a trace in a bug report.
|
|
259
|
+
4. **Search inside step content** (messages, tool inputs and results).
|
|
260
|
+
5. **Prompt-cache view:** cache hit rate per step and per run.
|
|
261
|
+
6. **Retention:** `stepline gc --older-than 30d`.
|
|
262
|
+
|
|
263
|
+
Smaller, self-contained items are listed as good first issues in [CONTRIBUTING.md](CONTRIBUTING.md).
|
|
264
|
+
|
|
265
|
+
## Contributing
|
|
266
|
+
|
|
267
|
+
Contributions are welcome: bug reports, adapters for other SDKs, UI improvements. Start with
|
|
268
|
+
[CONTRIBUTING.md](CONTRIBUTING.md), which covers development setup, the test commands
|
|
269
|
+
(`make check`, `npm run e2e`) and good first issues. Please report security issues privately,
|
|
270
|
+
as described in [SECURITY.md](SECURITY.md).
|
|
271
|
+
|
|
272
|
+
## License
|
|
273
|
+
|
|
274
|
+
[MIT](LICENSE)
|
stepline-0.1.0/README.md
ADDED
|
@@ -0,0 +1,242 @@
|
|
|
1
|
+
# Stepline
|
|
2
|
+
|
|
3
|
+
**A time-travel debugger for AI agents: record every LLM call, inspect each step in a local web UI, then edit any step and replay it as a new branch.**
|
|
4
|
+
|
|
5
|
+
[](https://github.com/dapdap777/stepline/actions/workflows/ci.yml)
|
|
6
|
+
[](LICENSE)
|
|
7
|
+
|
|
8
|
+
<img src="docs/assets/demo-placeholder.svg" alt="Demo GIF placeholder: record an agent run, inspect every step, edit and replay one step" width="100%">
|
|
9
|
+
|
|
10
|
+
## Quickstart (30 seconds, no API key)
|
|
11
|
+
|
|
12
|
+
```bash
|
|
13
|
+
pip install "stepline[anthropic]"
|
|
14
|
+
stepline demo # runs an example agent against an offline mock API and records it
|
|
15
|
+
stepline ui # opens http://127.0.0.1:7777
|
|
16
|
+
```
|
|
17
|
+
|
|
18
|
+
Then record your own agent. These are the only two lines you add:
|
|
19
|
+
|
|
20
|
+
```python
|
|
21
|
+
import stepline
|
|
22
|
+
client = stepline.wrap(Anthropic()) # use `client` exactly as before
|
|
23
|
+
```
|
|
24
|
+
|
|
25
|
+
Run your agent as usual, then `stepline ui` (or `stepline show` in the terminal).
|
|
26
|
+
|
|
27
|
+
> **Not on PyPI yet.** Stepline hasn't had its first release. Until then,
|
|
28
|
+
> [install from source](#install-from-source).
|
|
29
|
+
|
|
30
|
+
---
|
|
31
|
+
|
|
32
|
+
## What you get
|
|
33
|
+
|
|
34
|
+
**A timeline of every call:** tokens, cost and latency per step, plus a waterfall showing when
|
|
35
|
+
each call ran. The gaps between bars are time spent in your own code, such as running tools.
|
|
36
|
+
Tool calls appear next to the results your code returned, and errors stand out.
|
|
37
|
+
|
|
38
|
+

|
|
39
|
+
|
|
40
|
+
**Replay any step with edits.** Change the system prompt, messages, tools, parameters or the
|
|
41
|
+
model, check the estimated cost, and send. The result is saved as a fork of the original run.
|
|
42
|
+
|
|
43
|
+

|
|
44
|
+
|
|
45
|
+
**Diff a fork against its parent:** step-by-step alignment plus word-level diffs of prompts,
|
|
46
|
+
parameters and responses.
|
|
47
|
+
|
|
48
|
+

|
|
49
|
+
|
|
50
|
+
**Or stay in the terminal:**
|
|
51
|
+
|
|
52
|
+

|
|
53
|
+
|
|
54
|
+
<sub>Screenshots come from `stepline demo` (the example agent below), viewed with
|
|
55
|
+
`stepline ui --mock` so the replay also ran offline.</sub>
|
|
56
|
+
|
|
57
|
+
## Features
|
|
58
|
+
|
|
59
|
+
Everything listed here exists and is covered by the test suite.
|
|
60
|
+
|
|
61
|
+
- **Two-line recording** for the Anthropic Python SDK, sync (`Anthropic`) and async
|
|
62
|
+
(`AsyncAnthropic`). It records `messages.create` (including `stream=True`),
|
|
63
|
+
`messages.stream` and `messages.parse`, on both `client.messages` and
|
|
64
|
+
`client.beta.messages`. `wrap()` returns the same client object.
|
|
65
|
+
- **What's captured per call:** the exact request (messages, system prompt, tools, parameters),
|
|
66
|
+
the response, tool calls linked to the tool results your code sent back, token usage
|
|
67
|
+
(including prompt-cache reads and writes), estimated cost, latency, time to first token for
|
|
68
|
+
streams, and errors.
|
|
69
|
+
- **Never breaks your app.** Recording happens on a background thread and adds well under a
|
|
70
|
+
millisecond per call. Errors in the recorder are logged and swallowed, and API exceptions
|
|
71
|
+
reach your code unchanged. Pending writes are flushed when the process exits.
|
|
72
|
+
- **Local storage:** SQLite at `./.stepline/traces.db` (WAL mode), plus a `.gitignore` written
|
|
73
|
+
into `.stepline/` so traces aren't committed by accident.
|
|
74
|
+
- **Web UI:** run list with search and filters; a virtualized timeline, tested with 3,000 steps;
|
|
75
|
+
a step panel showing full messages and tool calls; run-to-run diffs; dark (default) and light
|
|
76
|
+
themes; keyboard navigation (`j`/`k`, `g`/`G`, `r`, `?`).
|
|
77
|
+
- **Replay:** edit a step and re-send that one call, after a cost estimate and an explicit
|
|
78
|
+
confirmation. The result is saved as a new forked run, and the original run is never
|
|
79
|
+
modified. API errors are kept as failed steps. `stepline ui --mock` lets you try replay
|
|
80
|
+
offline.
|
|
81
|
+
- **CLI:** `stepline ls`, `show [RUN] [--step N]`, `rm`, `prices`, `demo`, `ui`, `openapi`.
|
|
82
|
+
Runs can be referred to by id, unique id prefix, or name.
|
|
83
|
+
- **Cost estimates** from a built-in price table for Claude models. You can override or extend
|
|
84
|
+
it in `.stepline/prices.toml`.
|
|
85
|
+
|
|
86
|
+
## Limits
|
|
87
|
+
|
|
88
|
+
- **Anthropic Python SDK only, for now** (`anthropic>=0.40,<3`; CI tests both ends of that
|
|
89
|
+
range). OpenAI is next on the roadmap. Other SDKs, frameworks and languages aren't recorded.
|
|
90
|
+
Bedrock, Vertex and Foundry clients from the `anthropic` package use the same interface, but
|
|
91
|
+
they aren't tested yet.
|
|
92
|
+
- **Replay re-sends one LLM call. It does not re-run your tools or your agent.** A fork
|
|
93
|
+
therefore ends at the replayed step, even if the new response asks for a tool.
|
|
94
|
+
- **Costs are estimates** based on first-party list prices. They don't account for batch
|
|
95
|
+
discounts, partner-platform pricing or negotiated rates.
|
|
96
|
+
- **Single machine, single user.** It's a local development tool, not production monitoring,
|
|
97
|
+
and there's no sharing or team view.
|
|
98
|
+
- **Traces contain your prompts and model outputs.** Strings that look like API keys are
|
|
99
|
+
redacted, and request headers are never stored, but redaction is best-effort. Treat
|
|
100
|
+
`.stepline/` as sensitive. See [SECURITY.md](SECURITY.md).
|
|
101
|
+
|
|
102
|
+
## Why Stepline
|
|
103
|
+
|
|
104
|
+
Hosted LLM tracing and observability platforms are the right tool for production. They offer
|
|
105
|
+
team dashboards, monitoring, evaluations and SDKs for many languages, and several are open
|
|
106
|
+
source and can be self-hosted. Stepline is deliberately narrower: a debugger for the edit-run-
|
|
107
|
+
inspect loop on your own machine.
|
|
108
|
+
|
|
109
|
+
| | Stepline | Hosted tracing platforms (typical) |
|
|
110
|
+
|---|---|---|
|
|
111
|
+
| Setup | `pip install`, two lines of code | Account, project and API key, then an SDK or proxy |
|
|
112
|
+
| Where traces live | A SQLite file in your project | The vendor's cloud, or your own deployment |
|
|
113
|
+
| Telemetry | None. The UI makes no requests outside localhost | Varies by vendor |
|
|
114
|
+
| Re-running a step | Edit any recorded call and replay it as a forked branch, then diff against the original | Many offer a playground to re-run a prompt |
|
|
115
|
+
| Scope | One developer, one machine, Anthropic Python SDK | Teams, production traffic, many SDKs and frameworks |
|
|
116
|
+
|
|
117
|
+
### Similar tools for recording and replaying agent runs
|
|
118
|
+
|
|
119
|
+
Several open-source projects also record agent runs and let you replay or fork them. The table
|
|
120
|
+
summarizes what each project's own README says it does. It isn't an independent evaluation, and
|
|
121
|
+
it may be out of date.
|
|
122
|
+
|
|
123
|
+
| Project | How it records | What replay means there | Main focus |
|
|
124
|
+
|---|---|---|---|
|
|
125
|
+
| [Rewind](https://github.com/llm-rewind/rewind) (`llm-rewind`) | Local HTTPS proxy (mitmproxy, needs a local CA certificate) for OpenAI, Anthropic and Gemini, in any language | Deterministic cassette replay of a whole run, at zero LLM cost | Finding *why* two runs diverged (`rewind bisect`) and mutation-testing agents (`rewind mutate`), plus pytest |
|
|
126
|
+
| [AgentTape](https://github.com/MITHRAN-BALACHANDER/AgentTape) | LLM calls (automatic OpenAI interception) and tools wrapped with `@agenttape.tool`, saved as YAML cassettes | Deterministic offline replay; tools never re-execute. Partial replay can run the LLM live against a new prompt with tools frozen | Fast, safe agent tests in CI, plus a CLI with an ASCII waterfall, diff and an HTML viewer |
|
|
127
|
+
| [Stepfork](https://github.com/utsab345/stepfork) | Instrumented tool calls (`@trace_tool`) and LLM requests; adapters for OpenAI chat completions (non-streaming) and LangGraph | Frozen replay of your entrypoint; hybrid replay re-runs selected LLM calls; fork-at-step with output overrides | Turning a failed run into a pytest regression test, with behavioral diffs and failure minimization |
|
|
128
|
+
| AgentRewind ([JWang-Langer](https://github.com/JWang-Langer/agentrewind)) | Request fingerprints for OpenAI-compatible upstreams; side effects go through an idempotent channel | Deterministic replay; `arw rewind` re-runs live from step N with changed settings | A CLI that pinpoints the first diverging step and explains the request-level differences; no visual dashboard |
|
|
129
|
+
| AgentRewind ([Akshatkasera](https://github.com/Akshatkasera/codex-community-hackathon-del-AgentRewind), a hackathon project) | Imports runs from common agent frameworks | Patch a broken step and replay the downstream branch | A FastAPI + React app that diagnoses failed multi-agent runs and turns fixes into regression tests |
|
|
130
|
+
|
|
131
|
+
What Stepline does differently, and what it doesn't do:
|
|
132
|
+
|
|
133
|
+
- **Recording is an in-process wrapper:** `stepline.wrap(client)` around the Anthropic Python
|
|
134
|
+
SDK. There's no proxy or certificate to install and no decorators on your tools. Streaming,
|
|
135
|
+
async and tool-result linking work automatically. The flip side is that only that SDK is
|
|
136
|
+
supported. Rewind's proxy covers more providers and any language.
|
|
137
|
+
- **Replay is a live, edited re-send of one call,** not a deterministic re-execution of your
|
|
138
|
+
program. You change the prompt, messages, tools, parameters or model, see the estimated cost,
|
|
139
|
+
and get a new branch you can diff against the original. Stepline doesn't capture or re-run
|
|
140
|
+
your tools, so a fork ends at that step. AgentTape, Stepfork and AgentRewind (JWang-Langer)
|
|
141
|
+
record tool calls or side effects and serve them back on replay. Both AgentRewind projects
|
|
142
|
+
can continue a run past the point you changed.
|
|
143
|
+
- **It's an interactive debugger, not a test harness.** The main interface is a local web UI
|
|
144
|
+
with a timeline of every step (tokens, cost, latency) and a step-by-step diff view. There's
|
|
145
|
+
no pytest export, cassette replay in CI, divergence-cause inference or mutation testing. If you
|
|
146
|
+
need those, the tools above are built for them.
|
|
147
|
+
- **It's local and needs no account,** like most of the open-source tools above.
|
|
148
|
+
|
|
149
|
+
### Other tools in this space
|
|
150
|
+
|
|
151
|
+
- [Retrace](https://retraceai.tech)
|
|
152
|
+
|
|
153
|
+
## How it works
|
|
154
|
+
|
|
155
|
+
```mermaid
|
|
156
|
+
flowchart LR
|
|
157
|
+
subgraph app["Your agent process"]
|
|
158
|
+
code["agent code"] -->|"client.messages.create / .stream"| wrapped["stepline.wrap(client)"]
|
|
159
|
+
wrapped -->|unchanged call| sdk["Anthropic SDK"]
|
|
160
|
+
wrapped -. "snapshot + timing<br/>(non-blocking queue)" .-> writer["background writer thread"]
|
|
161
|
+
end
|
|
162
|
+
sdk <--> anthropic[("Anthropic API")]
|
|
163
|
+
writer --> db[("./.stepline/traces.db<br/>SQLite: runs → steps")]
|
|
164
|
+
db --> cli["stepline ls / show"]
|
|
165
|
+
db --> server["stepline ui<br/>FastAPI on 127.0.0.1"]
|
|
166
|
+
server --> browser["Web UI<br/>(bundled React app)"]
|
|
167
|
+
browser -- "replay (explicit, one call)" --> server
|
|
168
|
+
server -- "edited request" --> anthropic
|
|
169
|
+
server -- "new forked run" --> db
|
|
170
|
+
```
|
|
171
|
+
|
|
172
|
+
- **Recorder.** `wrap()` patches the SDK's `messages` resources in place. A call snapshots its
|
|
173
|
+
request on your thread and passes straight through to the SDK. Streams are observed chunk by
|
|
174
|
+
chunk without being buffered. Normalization, redaction and SQL writes happen on a daemon
|
|
175
|
+
thread. SDK support sits behind a small adapter interface, so other SDKs can be added.
|
|
176
|
+
- **Storage.** A run has many steps. Each step points at the step before it
|
|
177
|
+
(`parent_step_id`), and a replay creates a new run whose step branches off the original's
|
|
178
|
+
parent. Together the runs form a tree.
|
|
179
|
+
- **Server and UI.** A FastAPI app that binds to loopback, checks Host and Origin on every
|
|
180
|
+
request, and serves a strict Content-Security-Policy. The UI is built into the Python
|
|
181
|
+
package, so nothing is fetched from a CDN.
|
|
182
|
+
|
|
183
|
+
## The example agent
|
|
184
|
+
|
|
185
|
+
[`examples/weather_agent.py`](examples/weather_agent.py) is a ~150-line tool-using agent with
|
|
186
|
+
three toy tools that return canned data. It runs streamed turns, makes parallel tool calls,
|
|
187
|
+
and handles a tool failure.
|
|
188
|
+
|
|
189
|
+
```bash
|
|
190
|
+
python examples/weather_agent.py --mock # offline mock API, no key needed
|
|
191
|
+
python examples/weather_agent.py # real API, needs ANTHROPIC_API_KEY (costs a few cents)
|
|
192
|
+
stepline ui
|
|
193
|
+
```
|
|
194
|
+
|
|
195
|
+
`stepline demo` runs the same file in mock mode.
|
|
196
|
+
|
|
197
|
+
## Configuration
|
|
198
|
+
|
|
199
|
+
Zero configuration is required. The options are in [docs/configuration.md](docs/configuration.md):
|
|
200
|
+
|
|
201
|
+
- where traces are stored (`STEPLINE_DB`, `STEPLINE_DIR`), and how to turn recording off
|
|
202
|
+
(`STEPLINE_DISABLED=1`);
|
|
203
|
+
- named runs: `with stepline.run("triage"): ...`;
|
|
204
|
+
- custom prices in `.stepline/prices.toml`;
|
|
205
|
+
- replay and API keys: the key is read from the environment when you replay and is never stored;
|
|
206
|
+
- the UI server's security model.
|
|
207
|
+
|
|
208
|
+
## Install from source
|
|
209
|
+
|
|
210
|
+
```bash
|
|
211
|
+
git clone https://github.com/dapdap777/stepline && cd stepline
|
|
212
|
+
uv sync # Python deps (Python 3.10+)
|
|
213
|
+
(cd frontend && npm ci && npm run build) # builds the UI into the package (Node 20+)
|
|
214
|
+
uv run stepline demo && uv run stepline ui
|
|
215
|
+
```
|
|
216
|
+
|
|
217
|
+
Without the frontend build, `stepline ui` serves a page explaining how to build it. The CLI
|
|
218
|
+
works either way.
|
|
219
|
+
|
|
220
|
+
## Roadmap
|
|
221
|
+
|
|
222
|
+
1. **OpenAI Python SDK adapter.** The recorder already goes through a `ProviderAdapter`
|
|
223
|
+
interface; Chat Completions and Responses come next.
|
|
224
|
+
2. **Replay that continues:** opt-in registration of your tool functions, so a fork can keep
|
|
225
|
+
running past the replayed step.
|
|
226
|
+
3. **Export and import a run** as a JSON file, to share a trace in a bug report.
|
|
227
|
+
4. **Search inside step content** (messages, tool inputs and results).
|
|
228
|
+
5. **Prompt-cache view:** cache hit rate per step and per run.
|
|
229
|
+
6. **Retention:** `stepline gc --older-than 30d`.
|
|
230
|
+
|
|
231
|
+
Smaller, self-contained items are listed as good first issues in [CONTRIBUTING.md](CONTRIBUTING.md).
|
|
232
|
+
|
|
233
|
+
## Contributing
|
|
234
|
+
|
|
235
|
+
Contributions are welcome: bug reports, adapters for other SDKs, UI improvements. Start with
|
|
236
|
+
[CONTRIBUTING.md](CONTRIBUTING.md), which covers development setup, the test commands
|
|
237
|
+
(`make check`, `npm run e2e`) and good first issues. Please report security issues privately,
|
|
238
|
+
as described in [SECURITY.md](SECURITY.md).
|
|
239
|
+
|
|
240
|
+
## License
|
|
241
|
+
|
|
242
|
+
[MIT](LICENSE)
|