fixtura 1.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fixtura-1.0.0/LICENSE +21 -0
- fixtura-1.0.0/MANIFEST.in +3 -0
- fixtura-1.0.0/PKG-INFO +68 -0
- fixtura-1.0.0/README.md +50 -0
- fixtura-1.0.0/cli/__init__.py +1 -0
- fixtura-1.0.0/cli/eval.py +42 -0
- fixtura-1.0.0/cli/html_view.py +7 -0
- fixtura-1.0.0/cli/inspect.py +18 -0
- fixtura-1.0.0/cli/main.py +52 -0
- fixtura-1.0.0/cli/record.py +103 -0
- fixtura-1.0.0/cli/replay.py +8 -0
- fixtura-1.0.0/cli/view.py +7 -0
- fixtura-1.0.0/fixtura.egg-info/PKG-INFO +68 -0
- fixtura-1.0.0/fixtura.egg-info/SOURCES.txt +37 -0
- fixtura-1.0.0/fixtura.egg-info/dependency_links.txt +1 -0
- fixtura-1.0.0/fixtura.egg-info/entry_points.txt +2 -0
- fixtura-1.0.0/fixtura.egg-info/requires.txt +8 -0
- fixtura-1.0.0/fixtura.egg-info/top_level.txt +5 -0
- fixtura-1.0.0/pyproject.toml +39 -0
- fixtura-1.0.0/recorder/__init__.py +1 -0
- fixtura-1.0.0/recorder/recorder.py +36 -0
- fixtura-1.0.0/recorder/sanitizer.py +72 -0
- fixtura-1.0.0/recorder/trace_reader.py +75 -0
- fixtura-1.0.0/replay/__init__.py +1 -0
- fixtura-1.0.0/replay/passive_replay.py +45 -0
- fixtura-1.0.0/replay/step_inspector.py +33 -0
- fixtura-1.0.0/security/__init__.py +1 -0
- fixtura-1.0.0/security/permission_engine.py +91 -0
- fixtura-1.0.0/security/rate_limiter.py +60 -0
- fixtura-1.0.0/setup.cfg +4 -0
- fixtura-1.0.0/tools/__init__.py +1 -0
- fixtura-1.0.0/tools/base.py +20 -0
- fixtura-1.0.0/tools/base_tool.py +47 -0
- fixtura-1.0.0/tools/filesystem_tool.py +39 -0
- fixtura-1.0.0/tools/html_viewer.py +129 -0
- fixtura-1.0.0/tools/http_tool.py +91 -0
- fixtura-1.0.0/tools/openeval_adapter.py +111 -0
- fixtura-1.0.0/tools/sqlite_tool.py +36 -0
- fixtura-1.0.0/tools/viewer.py +105 -0
fixtura-1.0.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Yashrajsinh Rathod
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
fixtura-1.0.0/PKG-INFO
ADDED
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: fixtura
|
|
3
|
+
Version: 1.0.0
|
|
4
|
+
Summary: Deterministic execution recording and replay for AI agents
|
|
5
|
+
Author: Yashrajsinh Rathod
|
|
6
|
+
License: MIT
|
|
7
|
+
Requires-Python: >=3.11
|
|
8
|
+
Description-Content-Type: text/markdown
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
Requires-Dist: pydantic<3.0,>=2.0
|
|
11
|
+
Requires-Dist: requests<3.0,>=2.0
|
|
12
|
+
Requires-Dist: zstandard<1.0,>=0.20
|
|
13
|
+
Provides-Extra: dev
|
|
14
|
+
Requires-Dist: pytest>=7.0; extra == "dev"
|
|
15
|
+
Requires-Dist: mypy>=1.0; extra == "dev"
|
|
16
|
+
Requires-Dist: types-requests>=2.0; extra == "dev"
|
|
17
|
+
Dynamic: license-file
|
|
18
|
+
|
|
19
|
+
# Fixtura
|
|
20
|
+
|
|
21
|
+
**Deterministic execution recording and replay for AI agents — turn real agent runs into regression tests.**
|
|
22
|
+
|
|
23
|
+
## What this is
|
|
24
|
+
|
|
25
|
+
Fixtura watches an AI agent's tool calls (filesystem, database, API, etc.), records everything that happens through a permission-checked execution layer, and lets you replay that recording later — deterministically, without touching live systems — to debug failures or gate CI on regressions.
|
|
26
|
+
|
|
27
|
+
It's built to work with **OpenEval**, a deterministic (non-LLM-judge) agent evaluation engine already built and published separately. Fixtura produces the recordings; OpenEval scores them.
|
|
28
|
+
|
|
29
|
+
## Why this exists
|
|
30
|
+
|
|
31
|
+
Most "agent observability" tools (Langfuse, Phoenix, Braintrust, Laminar) are built for production monitoring at scale. Fixtura's angle is narrower and more testable: **recorded traces as literal test fixtures**, replayable offline, usable to gate pull requests the same way unit test snapshots do. See ARCHITECTURE.md for why this is scoped the way it is, and what was deliberately cut.
|
|
32
|
+
|
|
33
|
+
## Status
|
|
34
|
+
|
|
35
|
+
✅ v1.0.0 is released. See ROADMAP.md for future features like counterfactual replay (Live Branching).
|
|
36
|
+
|
|
37
|
+
## OpenEval Adapter (optional, manual install required)
|
|
38
|
+
|
|
39
|
+
The OpenEval evaluation harness (Acceptance Test 4) relies on OpenEval, which is currently unpublished on PyPI. Note that the openeval namespace on PyPI is occupied by an unrelated placeholder package.
|
|
40
|
+
|
|
41
|
+
To use the OpenEval Adapter, you must manually install OpenEval via git:
|
|
42
|
+
`ash
|
|
43
|
+
pip install git+https://github.com/yash161004/OpenEval.git@4cb6cfe362c770a7674f5b0111ff54646883709b
|
|
44
|
+
`
|
|
45
|
+
Without this, pip install fixtura will give you the core recording, permissions, and replay capabilities, but importing ixtura.tools.openeval_adapter will raise a clear RuntimeError prompting this manual installation.
|
|
46
|
+
|
|
47
|
+
## Documents in this repo
|
|
48
|
+
|
|
49
|
+
| Doc | Purpose |
|
|
50
|
+
|---|---|
|
|
51
|
+
| ARCHITECTURE.md | System design, components, data flow |
|
|
52
|
+
| THREAT_MODEL.md | Trust boundaries, what could go wrong, mitigations |
|
|
53
|
+
| ROADMAP.md | Frozen v1 scope table + acceptance tests + v1.1/v2 future work |
|
|
54
|
+
| docs/TRACE_FORMAT_SPEC.md | Exact schema every recording must produce |
|
|
55
|
+
| docs/COLLABORATION.md | How the owner, Claude, ChatGPT, and Antigravity work together on this project |
|
|
56
|
+
|
|
57
|
+
## Core components
|
|
58
|
+
|
|
59
|
+
1. Tool Execution Layer
|
|
60
|
+
2. Permission Engine
|
|
61
|
+
3. Execution Recorder + Sanitizer
|
|
62
|
+
4. Passive Replay + Step Inspection
|
|
63
|
+
5. Trace Viewer UI (minimal)
|
|
64
|
+
6. OpenEval adapter (evaluation harness reuse)
|
|
65
|
+
|
|
66
|
+
## License
|
|
67
|
+
|
|
68
|
+
[MIT License](LICENSE)
|
fixtura-1.0.0/README.md
ADDED
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
# Fixtura
|
|
2
|
+
|
|
3
|
+
**Deterministic execution recording and replay for AI agents — turn real agent runs into regression tests.**
|
|
4
|
+
|
|
5
|
+
## What this is
|
|
6
|
+
|
|
7
|
+
Fixtura watches an AI agent's tool calls (filesystem, database, API, etc.), records everything that happens through a permission-checked execution layer, and lets you replay that recording later — deterministically, without touching live systems — to debug failures or gate CI on regressions.
|
|
8
|
+
|
|
9
|
+
It's built to work with **OpenEval**, a deterministic (non-LLM-judge) agent evaluation engine already built and published separately. Fixtura produces the recordings; OpenEval scores them.
|
|
10
|
+
|
|
11
|
+
## Why this exists
|
|
12
|
+
|
|
13
|
+
Most "agent observability" tools (Langfuse, Phoenix, Braintrust, Laminar) are built for production monitoring at scale. Fixtura's angle is narrower and more testable: **recorded traces as literal test fixtures**, replayable offline, usable to gate pull requests the same way unit test snapshots do. See ARCHITECTURE.md for why this is scoped the way it is, and what was deliberately cut.
|
|
14
|
+
|
|
15
|
+
## Status
|
|
16
|
+
|
|
17
|
+
✅ v1.0.0 is released. See ROADMAP.md for future features like counterfactual replay (Live Branching).
|
|
18
|
+
|
|
19
|
+
## OpenEval Adapter (optional, manual install required)
|
|
20
|
+
|
|
21
|
+
The OpenEval evaluation harness (Acceptance Test 4) relies on OpenEval, which is currently unpublished on PyPI. Note that the openeval namespace on PyPI is occupied by an unrelated placeholder package.
|
|
22
|
+
|
|
23
|
+
To use the OpenEval Adapter, you must manually install OpenEval via git:
|
|
24
|
+
`ash
|
|
25
|
+
pip install git+https://github.com/yash161004/OpenEval.git@4cb6cfe362c770a7674f5b0111ff54646883709b
|
|
26
|
+
`
|
|
27
|
+
Without this, pip install fixtura will give you the core recording, permissions, and replay capabilities, but importing ixtura.tools.openeval_adapter will raise a clear RuntimeError prompting this manual installation.
|
|
28
|
+
|
|
29
|
+
## Documents in this repo
|
|
30
|
+
|
|
31
|
+
| Doc | Purpose |
|
|
32
|
+
|---|---|
|
|
33
|
+
| ARCHITECTURE.md | System design, components, data flow |
|
|
34
|
+
| THREAT_MODEL.md | Trust boundaries, what could go wrong, mitigations |
|
|
35
|
+
| ROADMAP.md | Frozen v1 scope table + acceptance tests + v1.1/v2 future work |
|
|
36
|
+
| docs/TRACE_FORMAT_SPEC.md | Exact schema every recording must produce |
|
|
37
|
+
| docs/COLLABORATION.md | How the owner, Claude, ChatGPT, and Antigravity work together on this project |
|
|
38
|
+
|
|
39
|
+
## Core components
|
|
40
|
+
|
|
41
|
+
1. Tool Execution Layer
|
|
42
|
+
2. Permission Engine
|
|
43
|
+
3. Execution Recorder + Sanitizer
|
|
44
|
+
4. Passive Replay + Step Inspection
|
|
45
|
+
5. Trace Viewer UI (minimal)
|
|
46
|
+
6. OpenEval adapter (evaluation harness reuse)
|
|
47
|
+
|
|
48
|
+
## License
|
|
49
|
+
|
|
50
|
+
[MIT License](LICENSE)
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
# Init file
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
import os
|
|
2
|
+
from pprint import pprint
|
|
3
|
+
|
|
4
|
+
from tools.openeval_adapter import trace_to_agent_trace
|
|
5
|
+
from openeval.runner import run_eval
|
|
6
|
+
from openeval.models import EvalTestCase
|
|
7
|
+
from openeval.metrics import ToolSelectionAccuracy, ArgumentCorrectness, StepEfficiency, GoalCompletionRate
|
|
8
|
+
|
|
9
|
+
def run(trace_path: str) -> None:
|
|
10
|
+
print(f"Scoring {trace_path} via OpenEval adapter...")
|
|
11
|
+
metrics = [
|
|
12
|
+
ToolSelectionAccuracy(),
|
|
13
|
+
ArgumentCorrectness(),
|
|
14
|
+
StepEfficiency(),
|
|
15
|
+
GoalCompletionRate()
|
|
16
|
+
]
|
|
17
|
+
|
|
18
|
+
# We match the test case against the fixture agent built in cli.record
|
|
19
|
+
tc = EvalTestCase(
|
|
20
|
+
task_id="canonical-task",
|
|
21
|
+
input="Do stuff",
|
|
22
|
+
expected_tool_calls=[
|
|
23
|
+
{"tool": "filesystem_tool", "args": {"operation": "read", "path": "wrong_path.txt"}},
|
|
24
|
+
{"tool": "sqlite_tool", "args": {"operation": "read", "query": "SELECT * FROM test"}},
|
|
25
|
+
{"tool": "http_tool", "args": {"url": "http://disallowed.com"}}
|
|
26
|
+
],
|
|
27
|
+
expected_final_state={},
|
|
28
|
+
expected_output_contains=[],
|
|
29
|
+
max_steps=5,
|
|
30
|
+
timeout_seconds=10.0
|
|
31
|
+
)
|
|
32
|
+
|
|
33
|
+
trace = trace_to_agent_trace(
|
|
34
|
+
trace_path=trace_path,
|
|
35
|
+
task_id="canonical-task",
|
|
36
|
+
input_text="Do stuff",
|
|
37
|
+
final_output="Done",
|
|
38
|
+
actual_state={}
|
|
39
|
+
)
|
|
40
|
+
|
|
41
|
+
res = run_eval(trace, tc, metrics)
|
|
42
|
+
pprint(res)
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
from replay.step_inspector import StepInspector
|
|
2
|
+
from pprint import pprint
|
|
3
|
+
|
|
4
|
+
def run(trace_path: str) -> None:
|
|
5
|
+
print(f"Running Step Inspector for {trace_path}...")
|
|
6
|
+
inspector = StepInspector(trace_path)
|
|
7
|
+
step = 1
|
|
8
|
+
while inspector.advance():
|
|
9
|
+
current = inspector.current_step()
|
|
10
|
+
print(f"\n--- Paused at Step {step} ---")
|
|
11
|
+
pprint(current)
|
|
12
|
+
try:
|
|
13
|
+
input("Press Enter to continue to next step...")
|
|
14
|
+
except EOFError:
|
|
15
|
+
print("\nInspection ended: no more input available.")
|
|
16
|
+
break
|
|
17
|
+
step += 1
|
|
18
|
+
print("\nReached end of trace.")
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
import argparse
|
|
2
|
+
import sys
|
|
3
|
+
|
|
4
|
+
from cli import record, replay, inspect, view, eval, html_view
|
|
5
|
+
|
|
6
|
+
def main() -> None:
|
|
7
|
+
parser = argparse.ArgumentParser(description="Fixtura CLI")
|
|
8
|
+
subparsers = parser.add_subparsers(dest="command", help="Available commands")
|
|
9
|
+
|
|
10
|
+
# Record
|
|
11
|
+
parser_record = subparsers.add_parser("record", help="Record a trace using the canonical test agent")
|
|
12
|
+
parser_record.add_argument("trace_file", help="Path to output trace file")
|
|
13
|
+
|
|
14
|
+
# Replay
|
|
15
|
+
parser_replay = subparsers.add_parser("replay", help="Passively replay a trace")
|
|
16
|
+
parser_replay.add_argument("trace_file", help="Path to input trace file")
|
|
17
|
+
|
|
18
|
+
# Inspect
|
|
19
|
+
parser_inspect = subparsers.add_parser("inspect", help="Step-inspect a trace interactively")
|
|
20
|
+
parser_inspect.add_argument("trace_file", help="Path to input trace file")
|
|
21
|
+
|
|
22
|
+
# View
|
|
23
|
+
parser_view = subparsers.add_parser("view", help="View a trace summary")
|
|
24
|
+
parser_view.add_argument("trace_file", help="Path to input trace file")
|
|
25
|
+
|
|
26
|
+
# Eval
|
|
27
|
+
parser_eval = subparsers.add_parser("eval", help="Score a trace via OpenEval adapter")
|
|
28
|
+
parser_eval.add_argument("trace_file", help="Path to input trace file")
|
|
29
|
+
|
|
30
|
+
# HTML View
|
|
31
|
+
parser_html_view = subparsers.add_parser("html-view", help="View a trace summary in HTML format")
|
|
32
|
+
parser_html_view.add_argument("trace_file", help="Path to input trace file")
|
|
33
|
+
|
|
34
|
+
args = parser.parse_args()
|
|
35
|
+
|
|
36
|
+
if args.command == "record":
|
|
37
|
+
record.run(args.trace_file)
|
|
38
|
+
elif args.command == "replay":
|
|
39
|
+
replay.run(args.trace_file)
|
|
40
|
+
elif args.command == "inspect":
|
|
41
|
+
inspect.run(args.trace_file)
|
|
42
|
+
elif args.command == "view":
|
|
43
|
+
view.run(args.trace_file)
|
|
44
|
+
elif args.command == "eval":
|
|
45
|
+
eval.run(args.trace_file)
|
|
46
|
+
elif args.command == "html-view":
|
|
47
|
+
html_view.run(args.trace_file)
|
|
48
|
+
else:
|
|
49
|
+
parser.print_help()
|
|
50
|
+
|
|
51
|
+
if __name__ == "__main__":
|
|
52
|
+
main()
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
import os
|
|
2
|
+
import sqlite3
|
|
3
|
+
|
|
4
|
+
from security.permission_engine import CapabilityToken
|
|
5
|
+
from security.rate_limiter import RateLimiter
|
|
6
|
+
from recorder.recorder import ExecutionRecorder
|
|
7
|
+
from tools.filesystem_tool import FilesystemTool
|
|
8
|
+
from tools.sqlite_tool import SqliteTool
|
|
9
|
+
from tools.http_tool import HttpTool
|
|
10
|
+
|
|
11
|
+
def run(trace_path: str) -> None:
|
|
12
|
+
print(f"Recording canonical test trace to {trace_path}...")
|
|
13
|
+
|
|
14
|
+
if os.path.exists(trace_path):
|
|
15
|
+
os.remove(trace_path)
|
|
16
|
+
|
|
17
|
+
recorder = ExecutionRecorder(trace_path)
|
|
18
|
+
|
|
19
|
+
token = CapabilityToken()
|
|
20
|
+
token.filesystem.read = [os.getcwd()]
|
|
21
|
+
token.filesystem.write = [os.getcwd()]
|
|
22
|
+
token.http.allowed_domains = ["example.com"]
|
|
23
|
+
# No DB permissions given
|
|
24
|
+
|
|
25
|
+
rate_limiter = RateLimiter()
|
|
26
|
+
|
|
27
|
+
fs_tool = FilesystemTool(sandbox_root=os.getcwd())
|
|
28
|
+
|
|
29
|
+
db_path = "test.db"
|
|
30
|
+
with sqlite3.connect(db_path) as conn:
|
|
31
|
+
conn.execute("CREATE TABLE IF NOT EXISTS test (id INTEGER)")
|
|
32
|
+
db_tool = SqliteTool(db_path=db_path)
|
|
33
|
+
|
|
34
|
+
http_tool = HttpTool()
|
|
35
|
+
|
|
36
|
+
# 1. LLM Step
|
|
37
|
+
recorder.record_event({
|
|
38
|
+
"event_type": "llm_call",
|
|
39
|
+
"timestamp": 100.0,
|
|
40
|
+
"prompt": "Read the local file",
|
|
41
|
+
"completion": "I'll use filesystem tool",
|
|
42
|
+
"provider": "test",
|
|
43
|
+
"model": "test",
|
|
44
|
+
"input_tokens": 10,
|
|
45
|
+
"output_tokens": 10,
|
|
46
|
+
"finish_reason": "stop",
|
|
47
|
+
"latency_ms": 100
|
|
48
|
+
})
|
|
49
|
+
|
|
50
|
+
# 2. FS Tool (Should be allowed)
|
|
51
|
+
with open("test.txt", "w") as f:
|
|
52
|
+
f.write("test_content secret-sk-12345678901234567890")
|
|
53
|
+
|
|
54
|
+
args = {"operation": "read", "path": os.path.abspath("test.txt")}
|
|
55
|
+
res = fs_tool.execute(token, args, rate_limiter=rate_limiter)
|
|
56
|
+
recorder.record_event({
|
|
57
|
+
"event_type": "tool_call",
|
|
58
|
+
"timestamp": 200.0,
|
|
59
|
+
"tool_name": "filesystem_tool",
|
|
60
|
+
"arguments": args,
|
|
61
|
+
"permission_decision": "allowed" if res.outcome not in ("permission_denied", "validation_error") else "denied",
|
|
62
|
+
"permission_reason": res.reason if res.outcome == "permission_denied" else None,
|
|
63
|
+
"rate_limit_decision": "rate_limited" if res.outcome == "rate_limited" else ("circuit_broken" if res.outcome == "circuit_broken" else "allowed"),
|
|
64
|
+
"rate_limit_reason": res.reason if res.outcome in ("rate_limited", "circuit_broken") else None,
|
|
65
|
+
"response": res.result,
|
|
66
|
+
"latency_ms": 50
|
|
67
|
+
})
|
|
68
|
+
|
|
69
|
+
# 3. DB Tool (Should be denied)
|
|
70
|
+
args = {"operation": "read", "query": "SELECT * FROM test"}
|
|
71
|
+
res = db_tool.execute(token, args, rate_limiter=rate_limiter)
|
|
72
|
+
recorder.record_event({
|
|
73
|
+
"event_type": "tool_call",
|
|
74
|
+
"timestamp": 300.0,
|
|
75
|
+
"tool_name": "sqlite_tool",
|
|
76
|
+
"arguments": args,
|
|
77
|
+
"permission_decision": "allowed" if res.outcome not in ("permission_denied", "validation_error") else "denied",
|
|
78
|
+
"permission_reason": res.reason if res.outcome == "permission_denied" else None,
|
|
79
|
+
"rate_limit_decision": "rate_limited" if res.outcome == "rate_limited" else ("circuit_broken" if res.outcome == "circuit_broken" else "allowed"),
|
|
80
|
+
"rate_limit_reason": res.reason if res.outcome in ("rate_limited", "circuit_broken") else None,
|
|
81
|
+
"response": res.result,
|
|
82
|
+
"latency_ms": 50
|
|
83
|
+
})
|
|
84
|
+
|
|
85
|
+
# 4. HTTP Tool (Should be denied because we pass a disallowed URL to trigger another denial,
|
|
86
|
+
# or we can pass an allowed one. The prompt says "at least one allowed and one denied".
|
|
87
|
+
# I'll pass a denied one to match the OpenEval adapter test expectation of missing tools)
|
|
88
|
+
args = {"url": "http://disallowed.com"}
|
|
89
|
+
res = http_tool.execute(token, args, rate_limiter=rate_limiter)
|
|
90
|
+
recorder.record_event({
|
|
91
|
+
"event_type": "tool_call",
|
|
92
|
+
"timestamp": 400.0,
|
|
93
|
+
"tool_name": "http_tool",
|
|
94
|
+
"arguments": args,
|
|
95
|
+
"permission_decision": "allowed" if res.outcome not in ("permission_denied", "validation_error") else "denied",
|
|
96
|
+
"permission_reason": res.reason if res.outcome == "permission_denied" else None,
|
|
97
|
+
"rate_limit_decision": "rate_limited" if res.outcome == "rate_limited" else ("circuit_broken" if res.outcome == "circuit_broken" else "allowed"),
|
|
98
|
+
"rate_limit_reason": res.reason if res.outcome in ("rate_limited", "circuit_broken") else None,
|
|
99
|
+
"response": res.result,
|
|
100
|
+
"latency_ms": 50
|
|
101
|
+
})
|
|
102
|
+
|
|
103
|
+
print("Trace recording complete!")
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: fixtura
|
|
3
|
+
Version: 1.0.0
|
|
4
|
+
Summary: Deterministic execution recording and replay for AI agents
|
|
5
|
+
Author: Yashrajsinh Rathod
|
|
6
|
+
License: MIT
|
|
7
|
+
Requires-Python: >=3.11
|
|
8
|
+
Description-Content-Type: text/markdown
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
Requires-Dist: pydantic<3.0,>=2.0
|
|
11
|
+
Requires-Dist: requests<3.0,>=2.0
|
|
12
|
+
Requires-Dist: zstandard<1.0,>=0.20
|
|
13
|
+
Provides-Extra: dev
|
|
14
|
+
Requires-Dist: pytest>=7.0; extra == "dev"
|
|
15
|
+
Requires-Dist: mypy>=1.0; extra == "dev"
|
|
16
|
+
Requires-Dist: types-requests>=2.0; extra == "dev"
|
|
17
|
+
Dynamic: license-file
|
|
18
|
+
|
|
19
|
+
# Fixtura
|
|
20
|
+
|
|
21
|
+
**Deterministic execution recording and replay for AI agents — turn real agent runs into regression tests.**
|
|
22
|
+
|
|
23
|
+
## What this is
|
|
24
|
+
|
|
25
|
+
Fixtura watches an AI agent's tool calls (filesystem, database, API, etc.), records everything that happens through a permission-checked execution layer, and lets you replay that recording later — deterministically, without touching live systems — to debug failures or gate CI on regressions.
|
|
26
|
+
|
|
27
|
+
It's built to work with **OpenEval**, a deterministic (non-LLM-judge) agent evaluation engine already built and published separately. Fixtura produces the recordings; OpenEval scores them.
|
|
28
|
+
|
|
29
|
+
## Why this exists
|
|
30
|
+
|
|
31
|
+
Most "agent observability" tools (Langfuse, Phoenix, Braintrust, Laminar) are built for production monitoring at scale. Fixtura's angle is narrower and more testable: **recorded traces as literal test fixtures**, replayable offline, usable to gate pull requests the same way unit test snapshots do. See ARCHITECTURE.md for why this is scoped the way it is, and what was deliberately cut.
|
|
32
|
+
|
|
33
|
+
## Status
|
|
34
|
+
|
|
35
|
+
✅ v1.0.0 is released. See ROADMAP.md for future features like counterfactual replay (Live Branching).
|
|
36
|
+
|
|
37
|
+
## OpenEval Adapter (optional, manual install required)
|
|
38
|
+
|
|
39
|
+
The OpenEval evaluation harness (Acceptance Test 4) relies on OpenEval, which is currently unpublished on PyPI. Note that the openeval namespace on PyPI is occupied by an unrelated placeholder package.
|
|
40
|
+
|
|
41
|
+
To use the OpenEval Adapter, you must manually install OpenEval via git:
|
|
42
|
+
`ash
|
|
43
|
+
pip install git+https://github.com/yash161004/OpenEval.git@4cb6cfe362c770a7674f5b0111ff54646883709b
|
|
44
|
+
`
|
|
45
|
+
Without this, pip install fixtura will give you the core recording, permissions, and replay capabilities, but importing ixtura.tools.openeval_adapter will raise a clear RuntimeError prompting this manual installation.
|
|
46
|
+
|
|
47
|
+
## Documents in this repo
|
|
48
|
+
|
|
49
|
+
| Doc | Purpose |
|
|
50
|
+
|---|---|
|
|
51
|
+
| ARCHITECTURE.md | System design, components, data flow |
|
|
52
|
+
| THREAT_MODEL.md | Trust boundaries, what could go wrong, mitigations |
|
|
53
|
+
| ROADMAP.md | Frozen v1 scope table + acceptance tests + v1.1/v2 future work |
|
|
54
|
+
| docs/TRACE_FORMAT_SPEC.md | Exact schema every recording must produce |
|
|
55
|
+
| docs/COLLABORATION.md | How the owner, Claude, ChatGPT, and Antigravity work together on this project |
|
|
56
|
+
|
|
57
|
+
## Core components
|
|
58
|
+
|
|
59
|
+
1. Tool Execution Layer
|
|
60
|
+
2. Permission Engine
|
|
61
|
+
3. Execution Recorder + Sanitizer
|
|
62
|
+
4. Passive Replay + Step Inspection
|
|
63
|
+
5. Trace Viewer UI (minimal)
|
|
64
|
+
6. OpenEval adapter (evaluation harness reuse)
|
|
65
|
+
|
|
66
|
+
## License
|
|
67
|
+
|
|
68
|
+
[MIT License](LICENSE)
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
LICENSE
|
|
2
|
+
MANIFEST.in
|
|
3
|
+
README.md
|
|
4
|
+
pyproject.toml
|
|
5
|
+
cli/__init__.py
|
|
6
|
+
cli/eval.py
|
|
7
|
+
cli/html_view.py
|
|
8
|
+
cli/inspect.py
|
|
9
|
+
cli/main.py
|
|
10
|
+
cli/record.py
|
|
11
|
+
cli/replay.py
|
|
12
|
+
cli/view.py
|
|
13
|
+
fixtura.egg-info/PKG-INFO
|
|
14
|
+
fixtura.egg-info/SOURCES.txt
|
|
15
|
+
fixtura.egg-info/dependency_links.txt
|
|
16
|
+
fixtura.egg-info/entry_points.txt
|
|
17
|
+
fixtura.egg-info/requires.txt
|
|
18
|
+
fixtura.egg-info/top_level.txt
|
|
19
|
+
recorder/__init__.py
|
|
20
|
+
recorder/recorder.py
|
|
21
|
+
recorder/sanitizer.py
|
|
22
|
+
recorder/trace_reader.py
|
|
23
|
+
replay/__init__.py
|
|
24
|
+
replay/passive_replay.py
|
|
25
|
+
replay/step_inspector.py
|
|
26
|
+
security/__init__.py
|
|
27
|
+
security/permission_engine.py
|
|
28
|
+
security/rate_limiter.py
|
|
29
|
+
tools/__init__.py
|
|
30
|
+
tools/base.py
|
|
31
|
+
tools/base_tool.py
|
|
32
|
+
tools/filesystem_tool.py
|
|
33
|
+
tools/html_viewer.py
|
|
34
|
+
tools/http_tool.py
|
|
35
|
+
tools/openeval_adapter.py
|
|
36
|
+
tools/sqlite_tool.py
|
|
37
|
+
tools/viewer.py
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=61.0"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "fixtura"
|
|
7
|
+
version = "1.0.0"
|
|
8
|
+
description = "Deterministic execution recording and replay for AI agents"
|
|
9
|
+
authors = [
|
|
10
|
+
{name = "Yashrajsinh Rathod"}
|
|
11
|
+
]
|
|
12
|
+
readme = {file = "README.md", content-type = "text/markdown"}
|
|
13
|
+
license = {text = "MIT"}
|
|
14
|
+
dependencies = [
|
|
15
|
+
"pydantic>=2.0,<3.0",
|
|
16
|
+
"requests>=2.0,<3.0",
|
|
17
|
+
"zstandard>=0.20,<1.0",
|
|
18
|
+
]
|
|
19
|
+
requires-python = ">=3.11"
|
|
20
|
+
|
|
21
|
+
[project.optional-dependencies]
|
|
22
|
+
dev = [
|
|
23
|
+
"pytest>=7.0",
|
|
24
|
+
"mypy>=1.0",
|
|
25
|
+
"types-requests>=2.0",
|
|
26
|
+
]
|
|
27
|
+
|
|
28
|
+
[project.scripts]
|
|
29
|
+
fixtura = "cli.main:main"
|
|
30
|
+
|
|
31
|
+
[tool.setuptools]
|
|
32
|
+
packages = ["recorder", "replay", "security", "tools", "cli"]
|
|
33
|
+
|
|
34
|
+
[tool.mypy]
|
|
35
|
+
strict = true
|
|
36
|
+
|
|
37
|
+
[[tool.mypy.overrides]]
|
|
38
|
+
module = "openeval.*"
|
|
39
|
+
ignore_missing_imports = true
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
# Init file for recorder package
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import json
|
|
2
|
+
import threading
|
|
3
|
+
import zstandard as zstd
|
|
4
|
+
from typing import Dict, Any, Union
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
from recorder.sanitizer import Sanitizer
|
|
7
|
+
|
|
8
|
+
class ExecutionRecorder:
|
|
9
|
+
def __init__(self, trace_file: Union[str, Path]) -> None:
|
|
10
|
+
self.trace_file = Path(trace_file)
|
|
11
|
+
self.sanitizer = Sanitizer()
|
|
12
|
+
self._lock = threading.Lock()
|
|
13
|
+
self._step_counter = 1
|
|
14
|
+
|
|
15
|
+
def record_event(self, event: Dict[str, Any]) -> None:
|
|
16
|
+
"""
|
|
17
|
+
Sanitizes and records a single event to the .trace file.
|
|
18
|
+
No code path is allowed to bypass the sanitizer per TRACE_FORMAT_SPEC.md.
|
|
19
|
+
"""
|
|
20
|
+
with self._lock:
|
|
21
|
+
# Overwrite any incoming step_id with our monotonic generator
|
|
22
|
+
event["step_id"] = f"step-{self._step_counter:06d}"
|
|
23
|
+
self._step_counter += 1
|
|
24
|
+
|
|
25
|
+
# Pass through the sanitizer pipeline
|
|
26
|
+
sanitized_event = self.sanitizer.sanitize(event)
|
|
27
|
+
|
|
28
|
+
# Serialize to JSONL string
|
|
29
|
+
line = json.dumps(sanitized_event) + "\n"
|
|
30
|
+
|
|
31
|
+
# Instantiate compressor locally to avoid shared mutable state
|
|
32
|
+
cctx = zstd.ZstdCompressor()
|
|
33
|
+
compressed_frame = cctx.compress(line.encode("utf-8"))
|
|
34
|
+
|
|
35
|
+
with open(self.trace_file, "ab") as f:
|
|
36
|
+
f.write(compressed_frame)
|