timefork 0.1.0__tar.gz → 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {timefork-0.1.0 → timefork-0.2.0}/.gitignore +2 -0
- {timefork-0.1.0 → timefork-0.2.0}/CHANGELOG.md +72 -4
- timefork-0.1.0/README.md → timefork-0.2.0/PKG-INFO +105 -43
- timefork-0.1.0/PKG-INFO → timefork-0.2.0/README.md +65 -77
- {timefork-0.1.0 → timefork-0.2.0}/pyproject.toml +10 -3
- {timefork-0.1.0 → timefork-0.2.0}/src/timefork/actor.py +33 -3
- {timefork-0.1.0 → timefork-0.2.0}/src/timefork/api.py +42 -3
- {timefork-0.1.0 → timefork-0.2.0}/src/timefork/bisect.py +71 -11
- {timefork-0.1.0 → timefork-0.2.0}/src/timefork/ci.py +21 -7
- {timefork-0.1.0 → timefork-0.2.0}/src/timefork/cli.py +85 -10
- {timefork-0.1.0 → timefork-0.2.0}/src/timefork/lineage.py +67 -7
- {timefork-0.1.0 → timefork-0.2.0}/src/timefork/providers/anthropic.py +2 -0
- {timefork-0.1.0 → timefork-0.2.0}/src/timefork/providers/base.py +18 -1
- {timefork-0.1.0 → timefork-0.2.0}/src/timefork/providers/bedrock.py +1 -0
- {timefork-0.1.0 → timefork-0.2.0}/src/timefork/providers/google.py +34 -1
- {timefork-0.1.0 → timefork-0.2.0}/src/timefork/providers/openai.py +20 -1
- {timefork-0.1.0 → timefork-0.2.0}/src/timefork/proxy/app.py +35 -3
- {timefork-0.1.0 → timefork-0.2.0}/src/timefork/proxy/session.py +5 -0
- {timefork-0.1.0 → timefork-0.2.0}/src/timefork/redact.py +18 -4
- {timefork-0.1.0 → timefork-0.2.0}/src/timefork/rehash.py +34 -3
- {timefork-0.1.0 → timefork-0.2.0}/src/timefork/runtime.py +3 -1
- timefork-0.2.0/src/timefork/suspects.py +112 -0
- {timefork-0.1.0 → timefork-0.2.0}/src/timefork/tape/model.py +4 -0
- {timefork-0.1.0 → timefork-0.2.0}/src/timefork/tape/store.py +12 -4
- {timefork-0.1.0 → timefork-0.2.0}/src/timefork/tape/tape.py +18 -3
- {timefork-0.1.0 → timefork-0.2.0}/src/timefork/ui/server.py +31 -13
- {timefork-0.1.0 → timefork-0.2.0}/src/timefork/ui/static/index.html +77 -5
- timefork-0.1.0/.github/ISSUE_TEMPLATE/bug.yml +0 -29
- timefork-0.1.0/.github/ISSUE_TEMPLATE/feature.yml +0 -16
- timefork-0.1.0/.github/pull_request_template.md +0 -17
- timefork-0.1.0/.python-version +0 -1
- timefork-0.1.0/ARCHITECTURE.md +0 -430
- timefork-0.1.0/CITATION.cff +0 -16
- timefork-0.1.0/CODE_OF_CONDUCT.md +0 -29
- timefork-0.1.0/CONTRIBUTING.md +0 -70
- timefork-0.1.0/LAYMAN.md +0 -229
- timefork-0.1.0/PAPER.md +0 -999
- timefork-0.1.0/PATENT.md +0 -331
- timefork-0.1.0/SECURITY.md +0 -62
- timefork-0.1.0/TODO.md +0 -138
- timefork-0.1.0/bench/agent_under_test.py +0 -40
- timefork-0.1.0/bench/faults-flaky-p0.5-r1.json +0 -12
- timefork-0.1.0/bench/faults-flaky-p0.5-r4.json +0 -12
- timefork-0.1.0/bench/faults-flaky-p0.75-r1.json +0 -12
- timefork-0.1.0/bench/faults-flaky-p0.75-r3.json +0 -12
- timefork-0.1.0/bench/faults-flaky-p0.75-r4.json +0 -12
- timefork-0.1.0/bench/faults-flaky-p0.75-r5.json +0 -12
- timefork-0.1.0/bench/faults-r1.json +0 -32
- timefork-0.1.0/bench/faults-r3.json +0 -32
- timefork-0.1.0/bench/faults.json +0 -31
- timefork-0.1.0/bench/faults.py +0 -272
- timefork-0.1.0/bench/lineage.py +0 -212
- timefork-0.1.0/bench/measure.py +0 -239
- timefork-0.1.0/bench/report.py +0 -177
- timefork-0.1.0/bench/results.json +0 -131
- timefork-0.1.0/docs/REAL.md +0 -99
- timefork-0.1.0/docs/SCENARIOS.md +0 -188
- timefork-0.1.0/docs/images/investigation-dark.png +0 -0
- timefork-0.1.0/docs/images/investigation-light.png +0 -0
- timefork-0.1.0/docs/images/scenario-bisect-deterministic-dark.jpg +0 -0
- timefork-0.1.0/docs/images/scenario-bisect-deterministic-light.jpg +0 -0
- timefork-0.1.0/docs/images/scenario-bisect-environmental-dark.jpg +0 -0
- timefork-0.1.0/docs/images/scenario-bisect-environmental-light.jpg +0 -0
- timefork-0.1.0/docs/images/scenario-bisect-transient-dark.jpg +0 -0
- timefork-0.1.0/docs/images/scenario-bisect-transient-light.jpg +0 -0
- timefork-0.1.0/docs/images/scenario-clean-replay-dark.jpg +0 -0
- timefork-0.1.0/docs/images/scenario-clean-replay-light.jpg +0 -0
- timefork-0.1.0/docs/images/scenario-delegation-tree-dark.jpg +0 -0
- timefork-0.1.0/docs/images/scenario-delegation-tree-light.jpg +0 -0
- timefork-0.1.0/docs/images/scenario-fan-in-dark.jpg +0 -0
- timefork-0.1.0/docs/images/scenario-fan-in-light.jpg +0 -0
- timefork-0.1.0/docs/images/scenario-long-run-dark.jpg +0 -0
- timefork-0.1.0/docs/images/scenario-long-run-light.jpg +0 -0
- timefork-0.1.0/docs/images/scenario-multi-agent-dark.jpg +0 -0
- timefork-0.1.0/docs/images/scenario-multi-agent-light.jpg +0 -0
- timefork-0.1.0/docs/images/scenario-prompt-edit-dark.jpg +0 -0
- timefork-0.1.0/docs/images/scenario-prompt-edit-light.jpg +0 -0
- timefork-0.1.0/docs/images/scenario-shared-tape-dark.jpg +0 -0
- timefork-0.1.0/docs/images/scenario-shared-tape-light.jpg +0 -0
- timefork-0.1.0/docs/images/scenario-tool-calls-dark.jpg +0 -0
- timefork-0.1.0/docs/images/scenario-tool-calls-light.jpg +0 -0
- timefork-0.1.0/docs/images/scenario-what-if-dark.jpg +0 -0
- timefork-0.1.0/docs/images/scenario-what-if-light.jpg +0 -0
- timefork-0.1.0/docs/images/subagents-dark.jpg +0 -0
- timefork-0.1.0/docs/images/subagents-light.jpg +0 -0
- timefork-0.1.0/docs/images/tour.gif +0 -0
- timefork-0.1.0/docs/real.py +0 -339
- timefork-0.1.0/docs/scenarios.py +0 -669
- timefork-0.1.0/docs/subagents.py +0 -67
- timefork-0.1.0/docs/tour.py +0 -44
- timefork-0.1.0/examples/__init__.py +0 -0
- timefork-0.1.0/examples/fake_fs_server.py +0 -33
- timefork-0.1.0/examples/pipeline/README.md +0 -83
- timefork-0.1.0/examples/pipeline/__init__.py +0 -27
- timefork-0.1.0/examples/pipeline/agents.py +0 -215
- timefork-0.1.0/examples/pipeline/model.py +0 -257
- timefork-0.1.0/examples/pipeline/run.py +0 -32
- timefork-0.1.0/examples/pipeline/serve_model.py +0 -29
- timefork-0.1.0/examples/pipeline/tree.py +0 -28
- timefork-0.1.0/examples/real/__init__.py +0 -14
- timefork-0.1.0/examples/real/agent.py +0 -108
- timefork-0.1.0/examples/real/assets/ledger.pdf +0 -32
- timefork-0.1.0/examples/real/assets/red-square.png +0 -0
- timefork-0.1.0/examples/real/chat.py +0 -58
- timefork-0.1.0/examples/real/document.py +0 -35
- timefork-0.1.0/examples/real/mcp_server.py +0 -62
- timefork-0.1.0/examples/real/sdk.py +0 -68
- timefork-0.1.0/examples/real/team.py +0 -26
- timefork-0.1.0/examples/real/tree.py +0 -31
- timefork-0.1.0/examples/real/vision.py +0 -35
- timefork-0.1.0/examples/seed_demo.py +0 -87
- timefork-0.1.0/tests/__init__.py +0 -0
- timefork-0.1.0/tests/conftest.py +0 -92
- timefork-0.1.0/tests/fake_mcp_server.py +0 -41
- timefork-0.1.0/tests/test_agent_shapes.py +0 -286
- timefork-0.1.0/tests/test_api.py +0 -149
- timefork-0.1.0/tests/test_bedrock.py +0 -132
- timefork-0.1.0/tests/test_bisect.py +0 -171
- timefork-0.1.0/tests/test_bisect_e2e.py +0 -142
- timefork-0.1.0/tests/test_calibrate.py +0 -88
- timefork-0.1.0/tests/test_canon.py +0 -162
- timefork-0.1.0/tests/test_ci.py +0 -204
- timefork-0.1.0/tests/test_doctor.py +0 -41
- timefork-0.1.0/tests/test_edit.py +0 -258
- timefork-0.1.0/tests/test_frameworks.py +0 -177
- timefork-0.1.0/tests/test_judge.py +0 -60
- timefork-0.1.0/tests/test_lineage.py +0 -249
- timefork-0.1.0/tests/test_mcp.py +0 -159
- timefork-0.1.0/tests/test_pipeline.py +0 -180
- timefork-0.1.0/tests/test_redact_policy.py +0 -80
- timefork-0.1.0/tests/test_rehash.py +0 -150
- timefork-0.1.0/tests/test_replay.py +0 -464
- timefork-0.1.0/tests/test_search.py +0 -85
- timefork-0.1.0/tests/test_share.py +0 -172
- timefork-0.1.0/tests/test_stops.py +0 -116
- timefork-0.1.0/tests/test_sweep.py +0 -168
- timefork-0.1.0/tests/test_tape.py +0 -240
- timefork-0.1.0/tests/test_tokens.py +0 -157
- timefork-0.1.0/tests/test_tree.py +0 -122
- timefork-0.1.0/tests/test_ui_actions.py +0 -152
- {timefork-0.1.0 → timefork-0.2.0}/LICENSE +0 -0
- {timefork-0.1.0 → timefork-0.2.0}/NOTICE +0 -0
- {timefork-0.1.0 → timefork-0.2.0}/src/timefork/__init__.py +0 -0
- {timefork-0.1.0 → timefork-0.2.0}/src/timefork/calibrate.py +0 -0
- {timefork-0.1.0 → timefork-0.2.0}/src/timefork/doctor.py +0 -0
- {timefork-0.1.0 → timefork-0.2.0}/src/timefork/judge.py +0 -0
- {timefork-0.1.0 → timefork-0.2.0}/src/timefork/mcp/__init__.py +0 -0
- {timefork-0.1.0 → timefork-0.2.0}/src/timefork/mcp/jsonrpc.py +0 -0
- {timefork-0.1.0 → timefork-0.2.0}/src/timefork/mcp/shim.py +0 -0
- {timefork-0.1.0 → timefork-0.2.0}/src/timefork/probe.py +0 -0
- {timefork-0.1.0 → timefork-0.2.0}/src/timefork/providers/__init__.py +0 -0
- {timefork-0.1.0 → timefork-0.2.0}/src/timefork/providers/registry.py +0 -0
- {timefork-0.1.0 → timefork-0.2.0}/src/timefork/proxy/__init__.py +0 -0
- {timefork-0.1.0 → timefork-0.2.0}/src/timefork/proxy/diff.py +0 -0
- {timefork-0.1.0 → timefork-0.2.0}/src/timefork/proxy/signing.py +0 -0
- {timefork-0.1.0 → timefork-0.2.0}/src/timefork/replies.py +0 -0
- {timefork-0.1.0 → timefork-0.2.0}/src/timefork/search.py +0 -0
- {timefork-0.1.0 → timefork-0.2.0}/src/timefork/share.py +0 -0
- {timefork-0.1.0 → timefork-0.2.0}/src/timefork/sweep.py +0 -0
- {timefork-0.1.0 → timefork-0.2.0}/src/timefork/tape/__init__.py +0 -0
- {timefork-0.1.0 → timefork-0.2.0}/src/timefork/tape/canon.py +0 -0
- {timefork-0.1.0 → timefork-0.2.0}/src/timefork/tape/cas.py +0 -0
- {timefork-0.1.0 → timefork-0.2.0}/src/timefork/tape/chunk.py +0 -0
- {timefork-0.1.0 → timefork-0.2.0}/src/timefork/ui/__init__.py +0 -0
- {timefork-0.1.0 → timefork-0.2.0}/src/timefork/wire/__init__.py +0 -0
- {timefork-0.1.0 → timefork-0.2.0}/src/timefork/wire/sse.py +0 -0
|
@@ -2,6 +2,74 @@
|
|
|
2
2
|
|
|
3
3
|
Plain words about what changed and why. Newest first.
|
|
4
4
|
|
|
5
|
+
## 0.2.0 - 2026-09-27
|
|
6
|
+
|
|
7
|
+
- Twelve framework runs recorded and replayed on real providers with no code
|
|
8
|
+
that knows about the proxy: Strands, Pydantic AI, LangGraph, Google ADK,
|
|
9
|
+
CrewAI, AutoGen and the OpenAI Agents SDK on Claude; LangGraph, CrewAI,
|
|
10
|
+
AutoGen and the Agents SDK on OpenAI; Google ADK on Gemini through Vertex
|
|
11
|
+
AI, the Gemini adapter's first real traffic. `docs/real_frameworks.py`
|
|
12
|
+
writes `docs/FRAMEWORKS.md`; the README's table claims only what is in it.
|
|
13
|
+
- The proxy environment points the google-genai client's Vertex base URL at
|
|
14
|
+
the proxy, so Vertex AI express mode records like the Gemini API.
|
|
15
|
+
- `timefork replay --pace` hands a streamed reply back at the rhythm it was
|
|
16
|
+
recorded, chunk by chunk, so an agent's streaming code and timeouts are
|
|
17
|
+
tested too; the arrival time of every chunk is now kept on the step.
|
|
18
|
+
- `timefork rehash` reports the actors it renamed.
|
|
19
|
+
- A turn whose first message was re-sent as one string instead of a list
|
|
20
|
+
of text blocks, which the Claude Agent SDK does, is read as the same
|
|
21
|
+
conversation; the SDK's session-naming call is named `claude code title`.
|
|
22
|
+
- The Claude Agent SDK, driving Claude Code with one subagent, recorded
|
|
23
|
+
through the proxy by `docs/subagents.py --sdk` and pictured.
|
|
24
|
+
- `timefork suspects`: across many recordings of one workflow, the step
|
|
25
|
+
whose words predict failure, scored by how much more often the failed
|
|
26
|
+
runs said them than the passing ones.
|
|
27
|
+
- OpenAI's `content_filter` finish and `refusal` field, and Gemini's safety
|
|
28
|
+
finish reasons and blocked prompts, read as refusals: on the reel line, on
|
|
29
|
+
the frame, and in the Answer pane.
|
|
30
|
+
- `take.transport()` hands an async client's requests to the recorder
|
|
31
|
+
in-process, with no socket and no environment variable.
|
|
32
|
+
- The Wiring view folds a parent's helpers into one lane once there are
|
|
33
|
+
more than twelve lanes, every wire still drawn; a click opens them. The
|
|
34
|
+
gallery's new `swarm` scenario is sixteen helpers under one supervisor.
|
|
35
|
+
- `timefork bisect --ancestors` probes only the steps the graph says reached
|
|
36
|
+
the final one, plus one probe outside them that catches a graph that
|
|
37
|
+
missed an edge; the same through `Timefork.bisect(ancestors=True)`.
|
|
38
|
+
- Streamed tool calls on the OpenAI shape have met traffic: a test streams a
|
|
39
|
+
call in fragments the way the wire does and checks the page's description,
|
|
40
|
+
the replay and an edited call in the same shape.
|
|
41
|
+
- A conversation continued under another agent, which is how the OpenAI
|
|
42
|
+
Agents SDK hands off, is drawn as a handoff into a nested lane, named by
|
|
43
|
+
the transfer tool. The OpenAI shape's leading system message is read as
|
|
44
|
+
the prompt, not as part of the conversation.
|
|
45
|
+
- Two gallery scenarios run real frameworks against the scripted model with
|
|
46
|
+
no key: a LangGraph of three ChatOpenAI nodes, and the OpenAI Agents SDK
|
|
47
|
+
with a handoff and a function tool. Both recorded through the proxy with
|
|
48
|
+
no code that knows about it, both replayed free, both in the Wiring view.
|
|
49
|
+
- The scripted model speaks the OpenAI chat-completions shape, tool calls
|
|
50
|
+
and one-chunk streams included.
|
|
51
|
+
- `timefork redact --hash` replaces a value with a stable token derived from
|
|
52
|
+
it, so two steps that carried one account still read as one account.
|
|
53
|
+
- `timefork ci --judge "requirement"` scores each replayed run's final answer
|
|
54
|
+
with the same model judge `bisect` uses.
|
|
55
|
+
- Claude Code's main, subagent and title calls are named `claude code`,
|
|
56
|
+
`claude code subagent` and `claude code title` from markers only they
|
|
57
|
+
carry; `rehash` gives old tapes the same names.
|
|
58
|
+
- Commands started from the page run in their own process group, so
|
|
59
|
+
stopping one stops the agent it ran; no `pgrep`, so nothing Windows lacks.
|
|
60
|
+
- `rehash` says what a missing request path costs on Google and Bedrock
|
|
61
|
+
tapes from before 2026-09-12.
|
|
62
|
+
|
|
63
|
+
## 0.1.1 - 2026-09-26
|
|
64
|
+
|
|
65
|
+
- The source distribution holds the package, the README, the licence, the
|
|
66
|
+
notice and this changelog, and nothing else. The 0.1.0 sdist carried the
|
|
67
|
+
whole repository.
|
|
68
|
+
- The README opens with one line, git bisect for agent runs, and the install
|
|
69
|
+
section leads with the `uvx` path and the times it measured.
|
|
70
|
+
- Pictures are plain markdown images and the architecture diagrams are
|
|
71
|
+
rendered to PNG, so the GitHub mobile app and PyPI show them.
|
|
72
|
+
|
|
5
73
|
## 0.1.0 - 2026-09-26
|
|
6
74
|
|
|
7
75
|
The first version. Everything below was built and measured between the 7th and
|
|
@@ -69,9 +137,9 @@ pipeline, and real Claude Code sessions.
|
|
|
69
137
|
Claude Code. `python docs/real.py` runs every scenario and writes
|
|
70
138
|
`docs/REAL.md` with the numbers read back off the tape. A refusal, a
|
|
71
139
|
thinking-only reply, a parallel delegation and a wrapped error status
|
|
72
|
-
were all found this way
|
|
73
|
-
- `bench/report.py` renders the
|
|
74
|
-
|
|
140
|
+
were all found this way.
|
|
141
|
+
- `bench/report.py` renders the benches' tables from their own output and
|
|
142
|
+
writes them into any document between markers, so a number there is
|
|
75
143
|
something a script wrote.
|
|
76
144
|
- Python 3.11 is supported; the whole suite passes on it.
|
|
77
145
|
|
|
@@ -99,4 +167,4 @@ pipeline, and real Claude Code sessions.
|
|
|
99
167
|
- `timefork grep` searches every recording's text.
|
|
100
168
|
- A scenario gallery, `docs/SCENARIOS.md`, rebuilt by `docs/scenarios.py`, from
|
|
101
169
|
which the README's pictures come.
|
|
102
|
-
-
|
|
170
|
+
- Benchmarks under `bench/`, every number reproducible by a script.
|
|
@@ -1,30 +1,67 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: timefork
|
|
3
|
+
Version: 0.2.0
|
|
4
|
+
Summary: git bisect for agent runs. Record once through a proxy, replay for free, change one thing, find the step that broke it.
|
|
5
|
+
Author: Arun Shankar
|
|
6
|
+
License: Apache-2.0
|
|
7
|
+
License-File: LICENSE
|
|
8
|
+
License-File: NOTICE
|
|
9
|
+
Keywords: agents,debugging,llm,mcp,observability,replay
|
|
10
|
+
Classifier: Development Status :: 3 - Alpha
|
|
11
|
+
Classifier: Intended Audience :: Developers
|
|
12
|
+
Classifier: License :: OSI Approved :: Apache Software License
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
15
|
+
Classifier: Topic :: Software Development :: Debuggers
|
|
16
|
+
Requires-Python: >=3.11
|
|
17
|
+
Requires-Dist: click>=8.1
|
|
18
|
+
Requires-Dist: httpx>=0.27
|
|
19
|
+
Requires-Dist: rich>=13.9
|
|
20
|
+
Requires-Dist: starlette>=0.41
|
|
21
|
+
Requires-Dist: uvicorn>=0.32
|
|
22
|
+
Provides-Extra: aws
|
|
23
|
+
Requires-Dist: botocore>=1.34; extra == 'aws'
|
|
24
|
+
Provides-Extra: dev
|
|
25
|
+
Requires-Dist: mypy>=1.13; extra == 'dev'
|
|
26
|
+
Requires-Dist: pytest-asyncio>=0.24; extra == 'dev'
|
|
27
|
+
Requires-Dist: pytest>=8.3; extra == 'dev'
|
|
28
|
+
Requires-Dist: ruff>=0.7; extra == 'dev'
|
|
29
|
+
Provides-Extra: frameworks
|
|
30
|
+
Requires-Dist: google-adk>=1.0; extra == 'frameworks'
|
|
31
|
+
Requires-Dist: langchain-anthropic>=0.3; extra == 'frameworks'
|
|
32
|
+
Requires-Dist: langchain-openai>=0.2; extra == 'frameworks'
|
|
33
|
+
Requires-Dist: langgraph>=0.2; extra == 'frameworks'
|
|
34
|
+
Requires-Dist: litellm>=1.40; extra == 'frameworks'
|
|
35
|
+
Requires-Dist: openai-agents>=0.1; extra == 'frameworks'
|
|
36
|
+
Requires-Dist: openai>=1.40; extra == 'frameworks'
|
|
37
|
+
Requires-Dist: pydantic-ai>=1.0; extra == 'frameworks'
|
|
38
|
+
Requires-Dist: strands-agents>=1.0; extra == 'frameworks'
|
|
39
|
+
Description-Content-Type: text/markdown
|
|
40
|
+
|
|
1
41
|
<h1 align="center">Timefork</h1>
|
|
2
42
|
|
|
3
43
|
<p align="center">
|
|
4
|
-
<b>
|
|
5
|
-
Record a run
|
|
44
|
+
<b>git bisect for agent runs.</b><br>
|
|
45
|
+
Record a run once, through a proxy. Replay it for free. Change one thing. Find the step that broke it.
|
|
6
46
|
</p>
|
|
7
47
|
|
|
8
48
|
<p align="center">
|
|
9
49
|
<a href="#install"><img alt="python" src="https://img.shields.io/badge/python-3.11%2B-1E2838?style=flat-square&labelColor=0F1520"></a>
|
|
10
50
|
<a href="LICENSE"><img alt="license" src="https://img.shields.io/badge/license-Apache--2.0-2DD4A7?style=flat-square&labelColor=0F1520"></a>
|
|
11
51
|
<a href="#it-works-with-your-framework-because-it-has-never-heard-of-it"><img alt="frameworks" src="https://img.shields.io/badge/frameworks-all%20of%20them-8AB4FF?style=flat-square&labelColor=0F1520"></a>
|
|
12
|
-
<a href="#status"><img alt="tests" src="https://img.shields.io/badge/tests-
|
|
52
|
+
<a href="#status"><img alt="tests" src="https://img.shields.io/badge/tests-real%20HTTP%20over%20TCP-2DD4A7?style=flat-square&labelColor=0F1520"></a>
|
|
13
53
|
<a href="docs/REAL.md"><img alt="real api" src="https://img.shields.io/badge/proven%20on-the%20real%20API-FFA94D?style=flat-square&labelColor=0F1520"></a>
|
|
14
54
|
</p>
|
|
15
55
|
|
|
16
|
-
|
|
17
|
-
<picture>
|
|
18
|
-
<source media="(prefers-color-scheme: dark)" srcset="docs/images/investigation-dark.png">
|
|
19
|
-
<img alt="A bisected agent run: the step that broke it, the probes that found it, and the answer that would have worked" src="docs/images/investigation-light.png" width="100%">
|
|
20
|
-
</picture>
|
|
21
|
-
</p>
|
|
56
|
+

|
|
22
57
|
|
|
23
58
|
<p align="center"><sub>A finished investigation. Five steps, three agents, one worker answered badly once. The search was told nothing and named the step in four probes.</sub></p>
|
|
24
59
|
|
|
25
60
|
<details>
|
|
26
61
|
<summary><b>Ten seconds of the timeline</b> - the investigation, the wiring, a what-if, and a real Claude Code session with a subagent</summary>
|
|
27
|
-
|
|
62
|
+
|
|
63
|
+

|
|
64
|
+
|
|
28
65
|
</details>
|
|
29
66
|
|
|
30
67
|
---
|
|
@@ -62,6 +99,19 @@ Every observability tool on the market will show you the trace.
|
|
|
62
99
|
|
|
63
100
|
## Install
|
|
64
101
|
|
|
102
|
+
Nothing to install first if you have [uv](https://docs.astral.sh/uv/):
|
|
103
|
+
|
|
104
|
+
```bash
|
|
105
|
+
uvx timefork record -- claude -p "what does this repo do"
|
|
106
|
+
uvx timefork serve
|
|
107
|
+
uvx timefork replay last -- claude -p "what does this repo do" # free
|
|
108
|
+
```
|
|
109
|
+
|
|
110
|
+
Measured cold on a laptop with a one-line prompt: the first command was
|
|
111
|
+
running 1.2 s after being typed, the Claude Code run recorded in 9 s, the
|
|
112
|
+
page was up 1.5 s later, and the replay finished in 2.5 s without touching
|
|
113
|
+
the provider. For keeps:
|
|
114
|
+
|
|
65
115
|
```bash
|
|
66
116
|
pip install timefork # or: uv tool install timefork
|
|
67
117
|
cd your-project && timefork init
|
|
@@ -135,17 +185,12 @@ intermittent failures, `-r 4` combines draws by disjunction rather than
|
|
|
135
185
|
majority, which takes exact localisation from 0.12 to 0.84 where resampling
|
|
136
186
|
rescues the run half the time. All three verdicts have been confirmed on real
|
|
137
187
|
Claude Code sessions, and a real refusal on the real API was called
|
|
138
|
-
*deterministic* correctly.
|
|
139
|
-
|
|
188
|
+
*deterministic* correctly. `bench/faults.py` measures it against ground
|
|
189
|
+
truth.
|
|
140
190
|
|
|
141
191
|
### 3. Change what the model said, and replay from there
|
|
142
192
|
|
|
143
|
-
|
|
144
|
-
<picture>
|
|
145
|
-
<source media="(prefers-color-scheme: dark)" srcset="docs/images/scenario-what-if-dark.jpg">
|
|
146
|
-
<img alt="A worker's answer changed from 66 to 999 on the timeline; three steps replayed free, the critic re-ran live" src="docs/images/scenario-what-if-light.jpg" width="100%">
|
|
147
|
-
</picture>
|
|
148
|
-
</p>
|
|
193
|
+

|
|
149
194
|
|
|
150
195
|
Pick a step, edit its answer - or the tool it called - and replay. The edit is
|
|
151
196
|
served to the agent in the vendor's own wire shape; every step that did not
|
|
@@ -166,21 +211,35 @@ its model provider. There is nothing to import, subclass, wrap or configure.
|
|
|
166
211
|
|
|
167
212
|
| | |
|
|
168
213
|
|:--|:--|
|
|
169
|
-
| Google ADK | ✓ |
|
|
170
|
-
| LangChain · LangGraph | ✓ |
|
|
171
|
-
|
|
|
172
|
-
|
|
|
173
|
-
|
|
|
174
|
-
|
|
|
214
|
+
| Google ADK | ✓ [on Claude and on Gemini](docs/FRAMEWORKS.md) |
|
|
215
|
+
| LangChain · LangGraph | ✓ [on Claude and on OpenAI](docs/FRAMEWORKS.md) · [pictured](docs/SCENARIOS.md) |
|
|
216
|
+
| OpenAI Agents SDK, handoffs and tools | ✓ [on OpenAI and on Claude](docs/FRAMEWORKS.md) · [pictured](docs/SCENARIOS.md) |
|
|
217
|
+
| CrewAI | ✓ [on Claude and on OpenAI](docs/FRAMEWORKS.md) |
|
|
218
|
+
| AutoGen | ✓ [on Claude and on OpenAI](docs/FRAMEWORKS.md) |
|
|
219
|
+
| Pydantic AI | ✓ [on Claude](docs/FRAMEWORKS.md) |
|
|
220
|
+
| Strands Agents (AWS) | ✓ [on Claude](docs/FRAMEWORKS.md) |
|
|
221
|
+
| Claude Code | ✓ real sessions, [pictured](docs/images/subagents-light.jpg) |
|
|
222
|
+
| Claude Agent SDK, subagent and all | ✓ real session, [pictured](docs/images/agent-sdk-light.jpg) |
|
|
223
|
+
| forty lines of `httpx` | ✓ most of the test suite |
|
|
224
|
+
| Cursor · Cline | the same wire shape, not run here: they are desktop programs |
|
|
225
|
+
| the thing you wrote this morning | ✓ if it speaks HTTP to a provider |
|
|
226
|
+
|
|
227
|
+
Every mark above is a program in this repository that was run through the
|
|
228
|
+
proxy against a real model, recorded, and then replayed with every step from
|
|
229
|
+
the tape and the provider never contacted. The numbers are in
|
|
230
|
+
[docs/FRAMEWORKS.md](docs/FRAMEWORKS.md), rebuilt by `python docs/real_frameworks.py`.
|
|
175
231
|
|
|
176
232
|
If it speaks HTTP to Anthropic, OpenAI, Gemini, Vertex or Bedrock - or to any of
|
|
177
233
|
the many APIs that copied the OpenAI shape, including Azure, Groq, Together,
|
|
178
|
-
vLLM and Ollama - it is recorded.
|
|
234
|
+
vLLM and Ollama - it is recorded the same way. Anthropic, OpenAI and Gemini on
|
|
235
|
+
Vertex AI have each met real traffic here; Bedrock and the OpenAI-shaped clones
|
|
236
|
+
are handled by the same adapters and have been run only against fakes.
|
|
179
237
|
|
|
180
238
|
Bedrock is the one that needs something extra. SigV4 covers the host header, so
|
|
181
239
|
the signature your agent produced is for the proxy and AWS rejects it; the proxy
|
|
182
240
|
strips it and signs again, which means **the proxy needs its own AWS
|
|
183
|
-
credentials**. Install with `pip install 'timefork[aws]'`.
|
|
241
|
+
credentials**. Install with `pip install 'timefork[aws]'`. The re-signing is
|
|
242
|
+
tested against a fake; a real Bedrock run has not been made here.
|
|
184
243
|
|
|
185
244
|
**Wrap a command:**
|
|
186
245
|
|
|
@@ -212,7 +271,7 @@ have produced: a mid-stream safety refusal on a ledger-adding tool call, which
|
|
|
212
271
|
`bisect` correctly called deterministic; a worker that thought for sixteen
|
|
213
272
|
tokens and said nothing; a model delegating to two helpers in one turn; and a
|
|
214
273
|
proxy that wrapped an upstream 400 in a 200 event stream. All four are handled
|
|
215
|
-
and written up in [
|
|
274
|
+
and written up in [docs/REAL.md](docs/REAL.md).
|
|
216
275
|
|
|
217
276
|
<br>
|
|
218
277
|
|
|
@@ -248,12 +307,7 @@ indented under the one that started it, a curve for every handoff, and after a
|
|
|
248
307
|
bisect, the path the blamed answer took in rose. This is a real Claude Code
|
|
249
308
|
session that was asked to spawn a subagent; nothing was declared:
|
|
250
309
|
|
|
251
|
-
|
|
252
|
-
<picture>
|
|
253
|
-
<source media="(prefers-color-scheme: dark)" srcset="docs/images/subagents-dark.jpg">
|
|
254
|
-
<img alt="A real Claude Code session with a subagent, drawn as lanes: the main agent, the subagent nested under it, the brief going down and the report coming back" src="docs/images/subagents-light.jpg" width="100%">
|
|
255
|
-
</picture>
|
|
256
|
-
</p>
|
|
310
|
+

|
|
257
311
|
|
|
258
312
|
Measured against runs with a known graph (`bench/lineage.py`): no errors on
|
|
259
313
|
verbatim or paraphrased quotes, 0.99 precision on one-word answers, and an
|
|
@@ -304,7 +358,9 @@ A recording becomes a test with one line: `timefork expect checkout-flow
|
|
|
304
358
|
"order 4471 confirmed"`. Then a drift whose final answer still says that is a
|
|
305
359
|
*drift* - the prompts moved, the outcome did not - and one whose answer no
|
|
306
360
|
longer says it is a *fail*, which is the thing a regression suite is for. Run
|
|
307
|
-
`ci --on-diverge live` to let a drifted run reach its answer and be scored
|
|
361
|
+
`ci --on-diverge live` to let a drifted run reach its answer and be scored, and
|
|
362
|
+
`ci --judge "the order was confirmed"` to have a model score it instead of a
|
|
363
|
+
substring.
|
|
308
364
|
|
|
309
365
|
<br>
|
|
310
366
|
|
|
@@ -336,6 +392,11 @@ result = deck.bisect("nightly", play=lambda env: my_agent(env=env) == 0)
|
|
|
336
392
|
Every take is a run on the tape, exactly as the command line would have made
|
|
337
393
|
it: the timeline shows it, `share` exports it, `bisect` can search it.
|
|
338
394
|
|
|
395
|
+
|
|
396
|
+
An async client that cannot be pointed at a base URL by its environment can
|
|
397
|
+
be handed the recorder directly: `httpx.AsyncClient(transport=take.transport())`
|
|
398
|
+
reaches it in-process, with no socket. Sync clients take `base_url=take.base_url`.
|
|
399
|
+
|
|
339
400
|
<br>
|
|
340
401
|
|
|
341
402
|
## Send someone the recording
|
|
@@ -436,19 +497,20 @@ which is the right default in CI.
|
|
|
436
497
|
| `timefork init` | create a tape here |
|
|
437
498
|
| `timefork doctor` | check this machine before the first recording |
|
|
438
499
|
| `timefork record -- CMD` | run CMD with recording on |
|
|
439
|
-
| `timefork replay REF -- CMD` | replay a run as a new branch; `--edit`, `--call`, `--from`, `--on-diverge block` |
|
|
440
|
-
| `timefork bisect REF --check CMD` | find the step that caused a failure; `--judge "…"` lets a model decide |
|
|
500
|
+
| `timefork replay REF -- CMD` | replay a run as a new branch; `--edit`, `--call`, `--from`, `--on-diverge block`, `--pace` |
|
|
501
|
+
| `timefork bisect REF --check CMD` | find the step that caused a failure; `--judge "…"` lets a model decide; `--ancestors` probes only the steps that reached the last one |
|
|
441
502
|
| `timefork serve` | open the timeline |
|
|
442
503
|
| `timefork graph REF` | draw who handed what to whom |
|
|
443
504
|
| `timefork mcp -- SERVER` | record an MCP server's tool calls |
|
|
444
505
|
| `timefork share REF` | export a run as one shareable file |
|
|
445
|
-
| `timefork ci` | replay every recording and report what changed |
|
|
506
|
+
| `timefork ci` | replay every recording and report what changed; `--judge "…"` lets a model score the answers |
|
|
446
507
|
| `timefork expect REF TEXT` | what a recording's final answer must say, for `ci` to score |
|
|
508
|
+
| `timefork suspects` | across many recordings, the step whose words predict failure |
|
|
447
509
|
| `timefork sweep REF --vary F --variants D` | try many versions of a prompt |
|
|
448
510
|
| `timefork calibrate -- CMD` | run twice, find what varies on its own |
|
|
449
511
|
| `timefork ignore PATH` | stop a varying field counting as a change |
|
|
450
512
|
| `timefork rehash` | re-file the tape after editing its ignore list by hand |
|
|
451
|
-
| `timefork redact -p REGEX --path PATH` | never store these values; scrubbed before hashing, so replay still matches |
|
|
513
|
+
| `timefork redact -p REGEX --path PATH` | never store these values; scrubbed before hashing, so replay still matches; `--hash` keeps a stable token per value |
|
|
452
514
|
| `timefork log` · `show REF` · `grep TEXT` · `stat` | list runs, list steps, search, size |
|
|
453
515
|
| `timefork up` · `timefork env` | standalone proxy |
|
|
454
516
|
|
|
@@ -477,7 +539,7 @@ diagrams.
|
|
|
477
539
|
|
|
478
540
|
## Status
|
|
479
541
|
|
|
480
|
-
`v0.
|
|
542
|
+
`v0.2` - alpha, and honest about it. What is in it: [CHANGELOG.md](CHANGELOG.md).
|
|
481
543
|
What is left, and where it is going: [TODO.md](TODO.md).
|
|
482
544
|
|
|
483
545
|
| | |
|
|
@@ -494,9 +556,10 @@ What is left, and where it is going: [TODO.md](TODO.md).
|
|
|
494
556
|
| shareable single-file export, redacted | ✓ |
|
|
495
557
|
| `ci` regression replay over a whole corpus | ✓ |
|
|
496
558
|
| Python API, sync and async, with callbacks | ✓ |
|
|
559
|
+
| twelve framework runs on Claude, OpenAI and Gemini, none of them knowing | ✓ [docs/FRAMEWORKS.md](docs/FRAMEWORKS.md) |
|
|
497
560
|
| cost in dollars rather than tokens | ✗ |
|
|
498
561
|
|
|
499
|
-
|
|
562
|
+
Every test runs a real upstream, a real proxy and real HTTP over TCP. The
|
|
500
563
|
assertion that matters most is that a replay leaves the provider's call count
|
|
501
564
|
untouched.
|
|
502
565
|
|
|
@@ -506,10 +569,9 @@ untouched.
|
|
|
506
569
|
|
|
507
570
|
- [docs/SCENARIOS.md](docs/SCENARIOS.md) - every scenario the tool is built for, as a real tape with a picture, in both themes, rebuilt by a script.
|
|
508
571
|
- [docs/REAL.md](docs/REAL.md) - the same scenarios on the real API, with numbers.
|
|
572
|
+
- [docs/FRAMEWORKS.md](docs/FRAMEWORKS.md) - seven agent frameworks recorded and replayed on the real API, none of them knowing.
|
|
573
|
+
- [docs/LAUNCH.md](docs/LAUNCH.md) and [docs/VIDEO.md](docs/VIDEO.md) - the launch post and the sixty-second video's script, both built from the numbers above.
|
|
509
574
|
- [ARCHITECTURE.md](ARCHITECTURE.md) - how it works, in plain English and diagrams.
|
|
510
|
-
- [PAPER.md](PAPER.md) - the technical write-up: design, the intervention search, measured numbers, and what has not been evaluated.
|
|
511
|
-
- [LAYMAN.md](LAYMAN.md) - the idea without jargon, and why anyone would pay for it.
|
|
512
|
-
- [PATENT.md](PATENT.md) - the invention disclosure, with the weak points named.
|
|
513
575
|
- [SECURITY.md](SECURITY.md) - what the tape holds, what `share` scrubs, what the page can run, and `timefork redact` for values a tape must never hold.
|
|
514
576
|
|
|
515
577
|
## License
|
|
@@ -1,64 +1,27 @@
|
|
|
1
|
-
Metadata-Version: 2.5
|
|
2
|
-
Name: timefork
|
|
3
|
-
Version: 0.1.0
|
|
4
|
-
Summary: Time-travel debugging for LLM agents. Record a run, rewind to any step, change one thing, replay.
|
|
5
|
-
Author: Arun Shankar
|
|
6
|
-
License: Apache-2.0
|
|
7
|
-
License-File: LICENSE
|
|
8
|
-
License-File: NOTICE
|
|
9
|
-
Keywords: agents,debugging,llm,mcp,observability,replay
|
|
10
|
-
Classifier: Development Status :: 3 - Alpha
|
|
11
|
-
Classifier: Intended Audience :: Developers
|
|
12
|
-
Classifier: License :: OSI Approved :: Apache Software License
|
|
13
|
-
Classifier: Programming Language :: Python :: 3.11
|
|
14
|
-
Classifier: Programming Language :: Python :: 3.12
|
|
15
|
-
Classifier: Topic :: Software Development :: Debuggers
|
|
16
|
-
Requires-Python: >=3.11
|
|
17
|
-
Requires-Dist: click>=8.1
|
|
18
|
-
Requires-Dist: httpx>=0.27
|
|
19
|
-
Requires-Dist: rich>=13.9
|
|
20
|
-
Requires-Dist: starlette>=0.41
|
|
21
|
-
Requires-Dist: uvicorn>=0.32
|
|
22
|
-
Provides-Extra: aws
|
|
23
|
-
Requires-Dist: botocore>=1.34; extra == 'aws'
|
|
24
|
-
Provides-Extra: dev
|
|
25
|
-
Requires-Dist: mypy>=1.13; extra == 'dev'
|
|
26
|
-
Requires-Dist: pytest-asyncio>=0.24; extra == 'dev'
|
|
27
|
-
Requires-Dist: pytest>=8.3; extra == 'dev'
|
|
28
|
-
Requires-Dist: ruff>=0.7; extra == 'dev'
|
|
29
|
-
Provides-Extra: frameworks
|
|
30
|
-
Requires-Dist: langchain-openai>=0.2; extra == 'frameworks'
|
|
31
|
-
Requires-Dist: litellm>=1.40; extra == 'frameworks'
|
|
32
|
-
Requires-Dist: openai>=1.40; extra == 'frameworks'
|
|
33
|
-
Description-Content-Type: text/markdown
|
|
34
|
-
|
|
35
1
|
<h1 align="center">Timefork</h1>
|
|
36
2
|
|
|
37
3
|
<p align="center">
|
|
38
|
-
<b>
|
|
39
|
-
Record a run
|
|
4
|
+
<b>git bisect for agent runs.</b><br>
|
|
5
|
+
Record a run once, through a proxy. Replay it for free. Change one thing. Find the step that broke it.
|
|
40
6
|
</p>
|
|
41
7
|
|
|
42
8
|
<p align="center">
|
|
43
9
|
<a href="#install"><img alt="python" src="https://img.shields.io/badge/python-3.11%2B-1E2838?style=flat-square&labelColor=0F1520"></a>
|
|
44
10
|
<a href="LICENSE"><img alt="license" src="https://img.shields.io/badge/license-Apache--2.0-2DD4A7?style=flat-square&labelColor=0F1520"></a>
|
|
45
11
|
<a href="#it-works-with-your-framework-because-it-has-never-heard-of-it"><img alt="frameworks" src="https://img.shields.io/badge/frameworks-all%20of%20them-8AB4FF?style=flat-square&labelColor=0F1520"></a>
|
|
46
|
-
<a href="#status"><img alt="tests" src="https://img.shields.io/badge/tests-
|
|
12
|
+
<a href="#status"><img alt="tests" src="https://img.shields.io/badge/tests-real%20HTTP%20over%20TCP-2DD4A7?style=flat-square&labelColor=0F1520"></a>
|
|
47
13
|
<a href="docs/REAL.md"><img alt="real api" src="https://img.shields.io/badge/proven%20on-the%20real%20API-FFA94D?style=flat-square&labelColor=0F1520"></a>
|
|
48
14
|
</p>
|
|
49
15
|
|
|
50
|
-
|
|
51
|
-
<picture>
|
|
52
|
-
<source media="(prefers-color-scheme: dark)" srcset="docs/images/investigation-dark.png">
|
|
53
|
-
<img alt="A bisected agent run: the step that broke it, the probes that found it, and the answer that would have worked" src="docs/images/investigation-light.png" width="100%">
|
|
54
|
-
</picture>
|
|
55
|
-
</p>
|
|
16
|
+

|
|
56
17
|
|
|
57
18
|
<p align="center"><sub>A finished investigation. Five steps, three agents, one worker answered badly once. The search was told nothing and named the step in four probes.</sub></p>
|
|
58
19
|
|
|
59
20
|
<details>
|
|
60
21
|
<summary><b>Ten seconds of the timeline</b> - the investigation, the wiring, a what-if, and a real Claude Code session with a subagent</summary>
|
|
61
|
-
|
|
22
|
+
|
|
23
|
+

|
|
24
|
+
|
|
62
25
|
</details>
|
|
63
26
|
|
|
64
27
|
---
|
|
@@ -96,6 +59,19 @@ Every observability tool on the market will show you the trace.
|
|
|
96
59
|
|
|
97
60
|
## Install
|
|
98
61
|
|
|
62
|
+
Nothing to install first if you have [uv](https://docs.astral.sh/uv/):
|
|
63
|
+
|
|
64
|
+
```bash
|
|
65
|
+
uvx timefork record -- claude -p "what does this repo do"
|
|
66
|
+
uvx timefork serve
|
|
67
|
+
uvx timefork replay last -- claude -p "what does this repo do" # free
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
Measured cold on a laptop with a one-line prompt: the first command was
|
|
71
|
+
running 1.2 s after being typed, the Claude Code run recorded in 9 s, the
|
|
72
|
+
page was up 1.5 s later, and the replay finished in 2.5 s without touching
|
|
73
|
+
the provider. For keeps:
|
|
74
|
+
|
|
99
75
|
```bash
|
|
100
76
|
pip install timefork # or: uv tool install timefork
|
|
101
77
|
cd your-project && timefork init
|
|
@@ -169,17 +145,12 @@ intermittent failures, `-r 4` combines draws by disjunction rather than
|
|
|
169
145
|
majority, which takes exact localisation from 0.12 to 0.84 where resampling
|
|
170
146
|
rescues the run half the time. All three verdicts have been confirmed on real
|
|
171
147
|
Claude Code sessions, and a real refusal on the real API was called
|
|
172
|
-
*deterministic* correctly.
|
|
173
|
-
|
|
148
|
+
*deterministic* correctly. `bench/faults.py` measures it against ground
|
|
149
|
+
truth.
|
|
174
150
|
|
|
175
151
|
### 3. Change what the model said, and replay from there
|
|
176
152
|
|
|
177
|
-
|
|
178
|
-
<picture>
|
|
179
|
-
<source media="(prefers-color-scheme: dark)" srcset="docs/images/scenario-what-if-dark.jpg">
|
|
180
|
-
<img alt="A worker's answer changed from 66 to 999 on the timeline; three steps replayed free, the critic re-ran live" src="docs/images/scenario-what-if-light.jpg" width="100%">
|
|
181
|
-
</picture>
|
|
182
|
-
</p>
|
|
153
|
+

|
|
183
154
|
|
|
184
155
|
Pick a step, edit its answer - or the tool it called - and replay. The edit is
|
|
185
156
|
served to the agent in the vendor's own wire shape; every step that did not
|
|
@@ -200,21 +171,35 @@ its model provider. There is nothing to import, subclass, wrap or configure.
|
|
|
200
171
|
|
|
201
172
|
| | |
|
|
202
173
|
|:--|:--|
|
|
203
|
-
| Google ADK | ✓ |
|
|
204
|
-
| LangChain · LangGraph | ✓ |
|
|
205
|
-
|
|
|
206
|
-
|
|
|
207
|
-
|
|
|
208
|
-
|
|
|
174
|
+
| Google ADK | ✓ [on Claude and on Gemini](docs/FRAMEWORKS.md) |
|
|
175
|
+
| LangChain · LangGraph | ✓ [on Claude and on OpenAI](docs/FRAMEWORKS.md) · [pictured](docs/SCENARIOS.md) |
|
|
176
|
+
| OpenAI Agents SDK, handoffs and tools | ✓ [on OpenAI and on Claude](docs/FRAMEWORKS.md) · [pictured](docs/SCENARIOS.md) |
|
|
177
|
+
| CrewAI | ✓ [on Claude and on OpenAI](docs/FRAMEWORKS.md) |
|
|
178
|
+
| AutoGen | ✓ [on Claude and on OpenAI](docs/FRAMEWORKS.md) |
|
|
179
|
+
| Pydantic AI | ✓ [on Claude](docs/FRAMEWORKS.md) |
|
|
180
|
+
| Strands Agents (AWS) | ✓ [on Claude](docs/FRAMEWORKS.md) |
|
|
181
|
+
| Claude Code | ✓ real sessions, [pictured](docs/images/subagents-light.jpg) |
|
|
182
|
+
| Claude Agent SDK, subagent and all | ✓ real session, [pictured](docs/images/agent-sdk-light.jpg) |
|
|
183
|
+
| forty lines of `httpx` | ✓ most of the test suite |
|
|
184
|
+
| Cursor · Cline | the same wire shape, not run here: they are desktop programs |
|
|
185
|
+
| the thing you wrote this morning | ✓ if it speaks HTTP to a provider |
|
|
186
|
+
|
|
187
|
+
Every mark above is a program in this repository that was run through the
|
|
188
|
+
proxy against a real model, recorded, and then replayed with every step from
|
|
189
|
+
the tape and the provider never contacted. The numbers are in
|
|
190
|
+
[docs/FRAMEWORKS.md](docs/FRAMEWORKS.md), rebuilt by `python docs/real_frameworks.py`.
|
|
209
191
|
|
|
210
192
|
If it speaks HTTP to Anthropic, OpenAI, Gemini, Vertex or Bedrock - or to any of
|
|
211
193
|
the many APIs that copied the OpenAI shape, including Azure, Groq, Together,
|
|
212
|
-
vLLM and Ollama - it is recorded.
|
|
194
|
+
vLLM and Ollama - it is recorded the same way. Anthropic, OpenAI and Gemini on
|
|
195
|
+
Vertex AI have each met real traffic here; Bedrock and the OpenAI-shaped clones
|
|
196
|
+
are handled by the same adapters and have been run only against fakes.
|
|
213
197
|
|
|
214
198
|
Bedrock is the one that needs something extra. SigV4 covers the host header, so
|
|
215
199
|
the signature your agent produced is for the proxy and AWS rejects it; the proxy
|
|
216
200
|
strips it and signs again, which means **the proxy needs its own AWS
|
|
217
|
-
credentials**. Install with `pip install 'timefork[aws]'`.
|
|
201
|
+
credentials**. Install with `pip install 'timefork[aws]'`. The re-signing is
|
|
202
|
+
tested against a fake; a real Bedrock run has not been made here.
|
|
218
203
|
|
|
219
204
|
**Wrap a command:**
|
|
220
205
|
|
|
@@ -246,7 +231,7 @@ have produced: a mid-stream safety refusal on a ledger-adding tool call, which
|
|
|
246
231
|
`bisect` correctly called deterministic; a worker that thought for sixteen
|
|
247
232
|
tokens and said nothing; a model delegating to two helpers in one turn; and a
|
|
248
233
|
proxy that wrapped an upstream 400 in a 200 event stream. All four are handled
|
|
249
|
-
and written up in [
|
|
234
|
+
and written up in [docs/REAL.md](docs/REAL.md).
|
|
250
235
|
|
|
251
236
|
<br>
|
|
252
237
|
|
|
@@ -282,12 +267,7 @@ indented under the one that started it, a curve for every handoff, and after a
|
|
|
282
267
|
bisect, the path the blamed answer took in rose. This is a real Claude Code
|
|
283
268
|
session that was asked to spawn a subagent; nothing was declared:
|
|
284
269
|
|
|
285
|
-
|
|
286
|
-
<picture>
|
|
287
|
-
<source media="(prefers-color-scheme: dark)" srcset="docs/images/subagents-dark.jpg">
|
|
288
|
-
<img alt="A real Claude Code session with a subagent, drawn as lanes: the main agent, the subagent nested under it, the brief going down and the report coming back" src="docs/images/subagents-light.jpg" width="100%">
|
|
289
|
-
</picture>
|
|
290
|
-
</p>
|
|
270
|
+

|
|
291
271
|
|
|
292
272
|
Measured against runs with a known graph (`bench/lineage.py`): no errors on
|
|
293
273
|
verbatim or paraphrased quotes, 0.99 precision on one-word answers, and an
|
|
@@ -338,7 +318,9 @@ A recording becomes a test with one line: `timefork expect checkout-flow
|
|
|
338
318
|
"order 4471 confirmed"`. Then a drift whose final answer still says that is a
|
|
339
319
|
*drift* - the prompts moved, the outcome did not - and one whose answer no
|
|
340
320
|
longer says it is a *fail*, which is the thing a regression suite is for. Run
|
|
341
|
-
`ci --on-diverge live` to let a drifted run reach its answer and be scored
|
|
321
|
+
`ci --on-diverge live` to let a drifted run reach its answer and be scored, and
|
|
322
|
+
`ci --judge "the order was confirmed"` to have a model score it instead of a
|
|
323
|
+
substring.
|
|
342
324
|
|
|
343
325
|
<br>
|
|
344
326
|
|
|
@@ -370,6 +352,11 @@ result = deck.bisect("nightly", play=lambda env: my_agent(env=env) == 0)
|
|
|
370
352
|
Every take is a run on the tape, exactly as the command line would have made
|
|
371
353
|
it: the timeline shows it, `share` exports it, `bisect` can search it.
|
|
372
354
|
|
|
355
|
+
|
|
356
|
+
An async client that cannot be pointed at a base URL by its environment can
|
|
357
|
+
be handed the recorder directly: `httpx.AsyncClient(transport=take.transport())`
|
|
358
|
+
reaches it in-process, with no socket. Sync clients take `base_url=take.base_url`.
|
|
359
|
+
|
|
373
360
|
<br>
|
|
374
361
|
|
|
375
362
|
## Send someone the recording
|
|
@@ -470,19 +457,20 @@ which is the right default in CI.
|
|
|
470
457
|
| `timefork init` | create a tape here |
|
|
471
458
|
| `timefork doctor` | check this machine before the first recording |
|
|
472
459
|
| `timefork record -- CMD` | run CMD with recording on |
|
|
473
|
-
| `timefork replay REF -- CMD` | replay a run as a new branch; `--edit`, `--call`, `--from`, `--on-diverge block` |
|
|
474
|
-
| `timefork bisect REF --check CMD` | find the step that caused a failure; `--judge "…"` lets a model decide |
|
|
460
|
+
| `timefork replay REF -- CMD` | replay a run as a new branch; `--edit`, `--call`, `--from`, `--on-diverge block`, `--pace` |
|
|
461
|
+
| `timefork bisect REF --check CMD` | find the step that caused a failure; `--judge "…"` lets a model decide; `--ancestors` probes only the steps that reached the last one |
|
|
475
462
|
| `timefork serve` | open the timeline |
|
|
476
463
|
| `timefork graph REF` | draw who handed what to whom |
|
|
477
464
|
| `timefork mcp -- SERVER` | record an MCP server's tool calls |
|
|
478
465
|
| `timefork share REF` | export a run as one shareable file |
|
|
479
|
-
| `timefork ci` | replay every recording and report what changed |
|
|
466
|
+
| `timefork ci` | replay every recording and report what changed; `--judge "…"` lets a model score the answers |
|
|
480
467
|
| `timefork expect REF TEXT` | what a recording's final answer must say, for `ci` to score |
|
|
468
|
+
| `timefork suspects` | across many recordings, the step whose words predict failure |
|
|
481
469
|
| `timefork sweep REF --vary F --variants D` | try many versions of a prompt |
|
|
482
470
|
| `timefork calibrate -- CMD` | run twice, find what varies on its own |
|
|
483
471
|
| `timefork ignore PATH` | stop a varying field counting as a change |
|
|
484
472
|
| `timefork rehash` | re-file the tape after editing its ignore list by hand |
|
|
485
|
-
| `timefork redact -p REGEX --path PATH` | never store these values; scrubbed before hashing, so replay still matches |
|
|
473
|
+
| `timefork redact -p REGEX --path PATH` | never store these values; scrubbed before hashing, so replay still matches; `--hash` keeps a stable token per value |
|
|
486
474
|
| `timefork log` · `show REF` · `grep TEXT` · `stat` | list runs, list steps, search, size |
|
|
487
475
|
| `timefork up` · `timefork env` | standalone proxy |
|
|
488
476
|
|
|
@@ -511,7 +499,7 @@ diagrams.
|
|
|
511
499
|
|
|
512
500
|
## Status
|
|
513
501
|
|
|
514
|
-
`v0.
|
|
502
|
+
`v0.2` - alpha, and honest about it. What is in it: [CHANGELOG.md](CHANGELOG.md).
|
|
515
503
|
What is left, and where it is going: [TODO.md](TODO.md).
|
|
516
504
|
|
|
517
505
|
| | |
|
|
@@ -528,9 +516,10 @@ What is left, and where it is going: [TODO.md](TODO.md).
|
|
|
528
516
|
| shareable single-file export, redacted | ✓ |
|
|
529
517
|
| `ci` regression replay over a whole corpus | ✓ |
|
|
530
518
|
| Python API, sync and async, with callbacks | ✓ |
|
|
519
|
+
| twelve framework runs on Claude, OpenAI and Gemini, none of them knowing | ✓ [docs/FRAMEWORKS.md](docs/FRAMEWORKS.md) |
|
|
531
520
|
| cost in dollars rather than tokens | ✗ |
|
|
532
521
|
|
|
533
|
-
|
|
522
|
+
Every test runs a real upstream, a real proxy and real HTTP over TCP. The
|
|
534
523
|
assertion that matters most is that a replay leaves the provider's call count
|
|
535
524
|
untouched.
|
|
536
525
|
|
|
@@ -540,10 +529,9 @@ untouched.
|
|
|
540
529
|
|
|
541
530
|
- [docs/SCENARIOS.md](docs/SCENARIOS.md) - every scenario the tool is built for, as a real tape with a picture, in both themes, rebuilt by a script.
|
|
542
531
|
- [docs/REAL.md](docs/REAL.md) - the same scenarios on the real API, with numbers.
|
|
532
|
+
- [docs/FRAMEWORKS.md](docs/FRAMEWORKS.md) - seven agent frameworks recorded and replayed on the real API, none of them knowing.
|
|
533
|
+
- [docs/LAUNCH.md](docs/LAUNCH.md) and [docs/VIDEO.md](docs/VIDEO.md) - the launch post and the sixty-second video's script, both built from the numbers above.
|
|
543
534
|
- [ARCHITECTURE.md](ARCHITECTURE.md) - how it works, in plain English and diagrams.
|
|
544
|
-
- [PAPER.md](PAPER.md) - the technical write-up: design, the intervention search, measured numbers, and what has not been evaluated.
|
|
545
|
-
- [LAYMAN.md](LAYMAN.md) - the idea without jargon, and why anyone would pay for it.
|
|
546
|
-
- [PATENT.md](PATENT.md) - the invention disclosure, with the weak points named.
|
|
547
535
|
- [SECURITY.md](SECURITY.md) - what the tape holds, what `share` scrubs, what the page can run, and `timefork redact` for values a tape must never hold.
|
|
548
536
|
|
|
549
537
|
## License
|