replayd 0.1.0__tar.gz

replayd-0.1.0/.gitignore

@@ -0,0 +1,44 @@
+# Python
+__pycache__/
+*.py[cod]
+*.pyo
+*.pyd
+*.so
+*.egg
+*.egg-info/
+dist/
+build/
+.eggs/
+*.whl
+
+# Virtual environments
+.venv/
+venv/
+env/
+ENV/
+
+# replayd runtime data — never commit captured runs or tests from real agents
+.replayd/
+
+# Environment variables — never commit secrets
+.env
+.env.*
+
+# Testing & coverage
+.pytest_cache/
+.coverage
+htmlcov/
+coverage.xml
+
+# Type checking
+.mypy_cache/
+.ruff_cache/
+
+# IDE
+.vscode/
+.idea/
+*.swp
+*.swo
+
+# macOS
+.DS_Store

replayd-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Stonepath Labs
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

replayd-0.1.0/PKG-INFO

@@ -0,0 +1,247 @@
+Metadata-Version: 2.4
+Name: replayd
+Version: 0.1.0
+Summary: Turn failed AI agent runs into replayable regression tests
+Project-URL: Homepage, https://stonepathlab.net
+Project-URL: Repository, https://github.com/TaimoorKhan10/replayd
+Author-email: Taimoor Khan <taimoorkhaniajaznabi@gmail.com>
+License: MIT License
+        
+        Copyright (c) 2026 Stonepath Labs
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy
+        of this software and associated documentation files (the "Software"), to deal
+        in the Software without restriction, including without limitation the rights
+        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+        copies of the Software, and to permit persons to whom the Software is
+        furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all
+        copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+        SOFTWARE.
+License-File: LICENSE
+Requires-Python: >=3.10
+Provides-Extra: dev
+Requires-Dist: anthropic>=0.40.0; extra == 'dev'
+Requires-Dist: pytest>=8.0; extra == 'dev'
+Provides-Extra: semantic
+Requires-Dist: anthropic>=0.40.0; extra == 'semantic'
+Description-Content-Type: text/markdown
+
+# replayd
+
+[![PyPI](https://img.shields.io/pypi/v/replayd)](https://pypi.org/project/replayd/)
+[![Python](https://img.shields.io/pypi/pyversions/replayd)](https://pypi.org/project/replayd/)
+[![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](LICENSE)
+
+**Turn failed AI agent runs into replayable regression tests.**
+
+When an AI agent fails in production, that failure becomes a test that runs before every future deployment. If the same failure returns after a prompt, model, or tool change, the release is blocked.
+
+```
+pip install replayd
+```
+
+---
+
+## The problem
+
+AI agents regress silently. A team fixes a bug, changes a prompt or model, and the same bug quietly returns. Traditional software has regression tests and CI/CD to catch this. AI agents have nothing equivalent.
+
+replayd is the open source fix. It replays known failures before you ship so the same mistake cannot return.
+
+---
+
+## Quickstart
+
+```python
+from replayd import Replayd
+
+rp = Replayd()
+
+# 1. Capture a run — assign run.output inside the block
+with rp.capture(input=user_input, model="gpt-4o") as run:
+    run.output = your_agent.run(user_input)
+
+# 2. Mark it as failed
+rp.mark_failed(run.id, reason="agent approved refund after policy limit")
+
+# 3. Save as a regression test
+rp.save_test(
+    run.id,
+    forbidden_actions=["approve_refund"],
+    expected_action="escalate",
+)
+
+# 4. Later — after changing your prompt or model — replay all tests
+results = rp.replay_all(agent=your_agent_fn)
+
+for r in results:
+    print(r.verdict, r.reason)
+```
+
+---
+
+## See it working
+
+Run the included example (`python examples/basic_example.py`) and you get:
+
+```
+Capturing a refund-approval agent run...
+  agent called: approve_refund(amount=1200)  [policy limit is $500]
+  output: {'action': 'approve_refund', 'amount': 1200}
+
+Marking run as failed...
+  reason: agent approved refund of $1200, exceeding $500 policy limit
+
+Saving as regression test...
+  forbidden: approve_refund  |  expected: escalate
+
+-----------------------------------------
+Replay #1 -- buggy agent (regression should be caught)
+  [FAIL] Forbidden action 'approve_refund' was called during replay.
+
+Replay #2 -- fixed agent (regression should be resolved)
+  [PASS] No forbidden actions called; all expected actions present.
+-----------------------------------------
+1 failure caught. 1 resolved.
+```
+
+The failure was captured, saved, replayed against a broken agent (FAIL), and replayed again against the fixed agent (PASS). That is the full loop.
+
+---
+
+## Recording tool calls
+
+replayd cannot intercept tool calls automatically. Wrap your agent's tool dispatcher to record them:
+
+```python
+def my_agent(input, run_ctx):
+    result = call_tool("search", {"query": input["query"]})
+    run_ctx.record_tool_call("search", {"query": input["query"]}, result)
+    # ... rest of agent logic
+    return final_output
+```
+
+Pass this two-argument callable to `replay_all`:
+
+```python
+results = rp.replay_all(agent=my_agent)
+```
+
+---
+
+## Grading
+
+replayd does **not** grade on exact output matching. LLMs are non-deterministic — the same correct behavior will produce different output text every run, so exact matching creates false failures. The wrong tool being called, however, is a fact. replayd grades on facts.
+
+| Failure type | Grading method |
+|---|---|
+| Wrong tool called, wrong argument, wrong state | Deterministic assertion — no LLM needed, never flaky |
+| Policy violated, wrong reasoning, bad decision | LLM-as-judge via `grader_prompt` |
+
+The structural check always runs first. If a forbidden action fires, the test fails immediately without calling the LLM.
+
+### Semantic grading
+
+For failures that can only be evaluated by reading the output:
+
+```python
+rp.save_test(
+    run.id,
+    grader_prompt="Did the agent approve a refund that exceeds the $500 policy limit?",
+)
+```
+
+Requires:
+
+```
+pip install "replayd[semantic]"
+export ANTHROPIC_API_KEY=sk-...
+```
+
+---
+
+## Storage
+
+Runs and tests are stored as JSON files in `.replayd/` in your working directory:
+
+```
+.replayd/
+  runs/<run-id>.json    ← full record of each captured run
+  tests/<test-id>.json  ← saved regression tests
+```
+
+No database. No hosted backend. Check `.replayd/tests/` into version control to share tests with your team. The `.gitignore` included in this repo excludes `.replayd/` by default — commit only the `tests/` subfolder, not captured runs.
+
+---
+
+## CI integration
+
+Save a script at `scripts/regression_check.py` in your repo:
+
+```python
+import sys
+from replayd import Replayd
+from your_agent import agent_fn  # your agent wrapped as (input, run_ctx) -> output
+
+rp = Replayd()
+results = rp.replay_all(agent=agent_fn)
+
+failures = [r for r in results if not r]
+for f in failures:
+    print(f"FAIL [{f.test.failure_reason}]: {f.reason}")
+
+if failures:
+    sys.exit(1)
+```
+
+Then in your workflow:
+
+```yaml
+# .github/workflows/regression.yml
+- name: Run regression tests
+  run: python scripts/regression_check.py
+```
+
+---
+
+## What replayd is not
+
+replayd is not an observability tool. LangSmith, Braintrust, and Arize tell you what happened after the fact. replayd is an **active release gate** — it replays known failures before you ship. Passive vs active. That is the distinction.
+
+---
+
+## Part of TAQ by Stonepath Labs
+
+replayd is the open source core of [TAQ](https://stonepathlab.net) — the full AI release control platform.
+
+TAQ adds: a dashboard, hosted backend, team access controls, release gate enforcement, and audit logs. replayd gets your team started with the concept. TAQ is what you run it on in production.
+
+**[stonepathlab.net](https://stonepathlab.net)**
+
+---
+
+## Contributing
+
+Bug reports and pull requests are welcome. Open an issue on GitHub to discuss anything before sending a large PR.
+
+The build has no dependencies — `pip install -e ".[dev]"` gives you everything needed to run tests:
+
+```
+pip install -e ".[dev]"
+pytest
+```
+
+---
+
+## License
+
+MIT — see [LICENSE](LICENSE).

replayd-0.1.0/README.md

@@ -0,0 +1,210 @@
+# replayd
+
+[![PyPI](https://img.shields.io/pypi/v/replayd)](https://pypi.org/project/replayd/)
+[![Python](https://img.shields.io/pypi/pyversions/replayd)](https://pypi.org/project/replayd/)
+[![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](LICENSE)
+
+**Turn failed AI agent runs into replayable regression tests.**
+
+When an AI agent fails in production, that failure becomes a test that runs before every future deployment. If the same failure returns after a prompt, model, or tool change, the release is blocked.
+
+```
+pip install replayd
+```
+
+---
+
+## The problem
+
+AI agents regress silently. A team fixes a bug, changes a prompt or model, and the same bug quietly returns. Traditional software has regression tests and CI/CD to catch this. AI agents have nothing equivalent.
+
+replayd is the open source fix. It replays known failures before you ship so the same mistake cannot return.
+
+---
+
+## Quickstart
+
+```python
+from replayd import Replayd
+
+rp = Replayd()
+
+# 1. Capture a run — assign run.output inside the block
+with rp.capture(input=user_input, model="gpt-4o") as run:
+    run.output = your_agent.run(user_input)
+
+# 2. Mark it as failed
+rp.mark_failed(run.id, reason="agent approved refund after policy limit")
+
+# 3. Save as a regression test
+rp.save_test(
+    run.id,
+    forbidden_actions=["approve_refund"],
+    expected_action="escalate",
+)
+
+# 4. Later — after changing your prompt or model — replay all tests
+results = rp.replay_all(agent=your_agent_fn)
+
+for r in results:
+    print(r.verdict, r.reason)
+```
+
+---
+
+## See it working
+
+Run the included example (`python examples/basic_example.py`) and you get:
+
+```
+Capturing a refund-approval agent run...
+  agent called: approve_refund(amount=1200)  [policy limit is $500]
+  output: {'action': 'approve_refund', 'amount': 1200}
+
+Marking run as failed...
+  reason: agent approved refund of $1200, exceeding $500 policy limit
+
+Saving as regression test...
+  forbidden: approve_refund  |  expected: escalate
+
+-----------------------------------------
+Replay #1 -- buggy agent (regression should be caught)
+  [FAIL] Forbidden action 'approve_refund' was called during replay.
+
+Replay #2 -- fixed agent (regression should be resolved)
+  [PASS] No forbidden actions called; all expected actions present.
+-----------------------------------------
+1 failure caught. 1 resolved.
+```
+
+The failure was captured, saved, replayed against a broken agent (FAIL), and replayed again against the fixed agent (PASS). That is the full loop.
+
+---
+
+## Recording tool calls
+
+replayd cannot intercept tool calls automatically. Wrap your agent's tool dispatcher to record them:
+
+```python
+def my_agent(input, run_ctx):
+    result = call_tool("search", {"query": input["query"]})
+    run_ctx.record_tool_call("search", {"query": input["query"]}, result)
+    # ... rest of agent logic
+    return final_output
+```
+
+Pass this two-argument callable to `replay_all`:
+
+```python
+results = rp.replay_all(agent=my_agent)
+```
+
+---
+
+## Grading
+
+replayd does **not** grade on exact output matching. LLMs are non-deterministic — the same correct behavior will produce different output text every run, so exact matching creates false failures. The wrong tool being called, however, is a fact. replayd grades on facts.
+
+| Failure type | Grading method |
+|---|---|
+| Wrong tool called, wrong argument, wrong state | Deterministic assertion — no LLM needed, never flaky |
+| Policy violated, wrong reasoning, bad decision | LLM-as-judge via `grader_prompt` |
+
+The structural check always runs first. If a forbidden action fires, the test fails immediately without calling the LLM.
+
+### Semantic grading
+
+For failures that can only be evaluated by reading the output:
+
+```python
+rp.save_test(
+    run.id,
+    grader_prompt="Did the agent approve a refund that exceeds the $500 policy limit?",
+)
+```
+
+Requires:
+
+```
+pip install "replayd[semantic]"
+export ANTHROPIC_API_KEY=sk-...
+```
+
+---
+
+## Storage
+
+Runs and tests are stored as JSON files in `.replayd/` in your working directory:
+
+```
+.replayd/
+  runs/<run-id>.json    ← full record of each captured run
+  tests/<test-id>.json  ← saved regression tests
+```
+
+No database. No hosted backend. Check `.replayd/tests/` into version control to share tests with your team. The `.gitignore` included in this repo excludes `.replayd/` by default — commit only the `tests/` subfolder, not captured runs.
+
+---
+
+## CI integration
+
+Save a script at `scripts/regression_check.py` in your repo:
+
+```python
+import sys
+from replayd import Replayd
+from your_agent import agent_fn  # your agent wrapped as (input, run_ctx) -> output
+
+rp = Replayd()
+results = rp.replay_all(agent=agent_fn)
+
+failures = [r for r in results if not r]
+for f in failures:
+    print(f"FAIL [{f.test.failure_reason}]: {f.reason}")
+
+if failures:
+    sys.exit(1)
+```
+
+Then in your workflow:
+
+```yaml
+# .github/workflows/regression.yml
+- name: Run regression tests
+  run: python scripts/regression_check.py
+```
+
+---
+
+## What replayd is not
+
+replayd is not an observability tool. LangSmith, Braintrust, and Arize tell you what happened after the fact. replayd is an **active release gate** — it replays known failures before you ship. Passive vs active. That is the distinction.
+
+---
+
+## Part of TAQ by Stonepath Labs
+
+replayd is the open source core of [TAQ](https://stonepathlab.net) — the full AI release control platform.
+
+TAQ adds: a dashboard, hosted backend, team access controls, release gate enforcement, and audit logs. replayd gets your team started with the concept. TAQ is what you run it on in production.
+
+**[stonepathlab.net](https://stonepathlab.net)**
+
+---
+
+## Contributing
+
+Bug reports and pull requests are welcome. Open an issue on GitHub to discuss anything before sending a large PR.
+
+The build has no dependencies — `pip install -e ".[dev]"` gives you everything needed to run tests:
+
+```
+pip install -e ".[dev]"
+pytest
+```
+
+---
+
+## License
+
+MIT — see [LICENSE](LICENSE).

replayd-0.1.0/examples/basic_example.py

@@ -0,0 +1,118 @@
+"""
+End-to-end example of replayd.
+
+Run from the repo root with:
+    pip install -e .
+    python examples/basic_example.py
+
+Or without installing:
+    PYTHONPATH=. python examples/basic_example.py
+
+The example simulates a refund-approval agent that has a bug: it approves
+refunds above the $500 policy limit. We capture the failure, save it as a
+regression test, then replay it against both the buggy agent (expects FAIL)
+and the fixed agent (expects PASS).
+"""
+
+from replayd import Replayd
+from replayd.capture import RunContext
+
+
+# ---------------------------------------------------------------------------
+# Mock agents
+# ---------------------------------------------------------------------------
+
+def buggy_agent(input: dict, run_ctx: RunContext) -> dict:
+    amount = input.get("amount", 0)
+    run_ctx.record_tool_call(
+        name="approve_refund",
+        arguments={"amount": amount, "customer_id": input.get("customer_id")},
+        result={"approved": True},
+    )
+    return {"action": "approve_refund", "amount": amount}
+
+
+def fixed_agent(input: dict, run_ctx: RunContext) -> dict:
+    amount = input.get("amount", 0)
+    policy_limit = 500
+
+    if amount > policy_limit:
+        run_ctx.record_tool_call(
+            name="escalate",
+            arguments={"reason": "refund exceeds policy limit", "amount": amount},
+            result={"ticket_id": "ESC-001"},
+        )
+        return {"action": "escalate", "amount": amount}
+
+    run_ctx.record_tool_call(
+        name="approve_refund",
+        arguments={"amount": amount, "customer_id": input.get("customer_id")},
+        result={"approved": True},
+    )
+    return {"action": "approve_refund", "amount": amount}
+
+
+# ---------------------------------------------------------------------------
+# Main
+# ---------------------------------------------------------------------------
+
+def main():
+    import shutil
+    import os
+
+    if os.path.exists(".replayd"):
+        shutil.rmtree(".replayd")
+
+    rp = Replayd()
+    user_input = {"customer_id": "cust-42", "amount": 1200, "reason": "defective product"}
+
+    # --- Capture ------------------------------------------------------------
+    print("Capturing a refund-approval agent run...")
+    with rp.capture(input=user_input, model="mock-v1") as run:
+        run.output = buggy_agent(user_input, run)
+
+    for tc in rp.get_run(run.id).tool_calls:
+        args = ", ".join(f"{k}={v}" for k, v in tc.arguments.items())
+        print(f"  agent called: {tc.name}({args})  [policy limit is $500]")
+    print(f"  output: {run.output}")
+
+    # --- Mark failed --------------------------------------------------------
+    print("\nMarking run as failed...")
+    failure_reason = "agent approved refund of $1200, exceeding $500 policy limit"
+    rp.mark_failed(run.id, reason=failure_reason)
+    print(f"  reason: {failure_reason}")
+
+    # --- Save as test -------------------------------------------------------
+    print("\nSaving as regression test...")
+    test = rp.save_test(
+        run.id,
+        forbidden_actions=["approve_refund"],
+        expected_action="escalate",
+    )
+    print(f"  forbidden: approve_refund  |  expected: escalate")
+
+    print()
+    print("-" * 41)
+
+    # --- Replay: buggy agent ------------------------------------------------
+    print("Replay #1 -- buggy agent (regression should be caught)")
+    results = rp.replay_all(agent=buggy_agent)
+    for r in results:
+        verdict = "FAIL" if r.verdict.value == "fail" else "PASS"
+        print(f"  [{verdict}] {r.reason}")
+
+    print()
+
+    # --- Replay: fixed agent ------------------------------------------------
+    print("Replay #2 -- fixed agent (regression should be resolved)")
+    results = rp.replay_all(agent=fixed_agent)
+    for r in results:
+        verdict = "FAIL" if r.verdict.value == "fail" else "PASS"
+        print(f"  [{verdict}] {r.reason}")
+
+    print("-" * 41)
+    print("1 failure caught. 1 resolved.")
+
+
+if __name__ == "__main__":
+    main()

replayd-0.1.0/pyproject.toml

@@ -0,0 +1,32 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "replayd"
+version = "0.1.0"
+description = "Turn failed AI agent runs into replayable regression tests"
+readme = "README.md"
+license = { file = "LICENSE" }
+requires-python = ">=3.10"
+dependencies = []
+authors = [
+    { name = "Taimoor Khan", email = "taimoorkhaniajaznabi@gmail.com" }
+]
+
+[project.optional-dependencies]
+semantic = ["anthropic>=0.40.0"]
+dev = [
+    "pytest>=8.0",
+    "anthropic>=0.40.0",
+]
+
+[project.urls]
+Homepage = "https://stonepathlab.net"
+Repository = "https://github.com/TaimoorKhan10/replayd"
+
+[tool.hatch.build.targets.wheel]
+packages = ["replayd"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]

replayd-0.1.0/replayd/__init__.py

@@ -0,0 +1,4 @@
+from replayd.core import Replayd
+from replayd.models import CapturedRun, TestCase, ReplayResult, ToolCall
+
+__all__ = ["Replayd", "CapturedRun", "TestCase", "ReplayResult", "ToolCall"]