runledger 0.2.0__tar.gz → 0.2.2__tar.gz

{runledger-0.2.0 → runledger-0.2.2}/PKG-INFO

@@ -1,9 +1,13 @@
 Metadata-Version: 2.4
 Name: runledger
-Version: 0.2.0
+Version: 0.2.2
 Summary: CI for tool-using agents: deterministic eval suites with record/replay, hard assertions, budgets, and PR regression gates.
 Author: RunLedger
 License-Expression: MIT
+Project-URL: Homepage, https://runledger.io
+Project-URL: Repository, https://github.com/runledger/Runledger
+Project-URL: Documentation, https://github.com/runledger/Runledger/tree/main/docs
+Project-URL: Changelog, https://github.com/runledger/Runledger/blob/main/CHANGELOG.md
 Keywords: llm,agents,evaluation,ci,github-actions,replay,testing,junit,jsonschema
 Requires-Python: >=3.9
 Description-Content-Type: text/markdown
@@ -30,25 +34,57 @@ Dynamic: license-file
 
 Website: https://runledger.io
 
-**Deterministic CI regression checks for tool-using agents.** Record tool calls once, replay in CI, and fail PRs on mismatches, contract violations, or budget regressions.
+**Deterministic CI regression checks for tool-using agents.** Record tool calls once, replay them in CI, and fail the run when a schema, tool-order, or tool-call budget gate fails.
 
-- Demo repo: https://github.com/runledger/runledger-demo (main is green; regression branches fail for real reasons)
+- Demo: https://github.com/ZackMitchell910/runledger-demo/pull/1 (intentionally left open to show RunLedger failing a bad agent change)
 - If this is useful, please star the repo (it helps a lot).
 
-RunLedger is a **CI harness**, not an "eval metrics framework":
-- **DeepEval-style tools** help you *score* behavior.
-- **RunLedger** helps you *ship safely* by making agent tests deterministic and merge-gated.
+## Five-minute quickstart
 
----
-
-## Try it now (60 seconds)
+From an empty directory, with Python 3.9+:
 
 ```bash
-pipx install runledger
+python3 -m pip install runledger
 runledger init
 runledger run ./evals/demo --mode replay --baseline baselines/demo.json
 ```
 
+`runledger init` writes:
+
+- `evals/demo/` — replay suite, cassette, JSON Schema, and a small agent that calls `search_docs`
+- `baselines/demo.json` — baseline from that passing run
+- `.github/workflows/runledger.yml` — pull-request workflow that runs the same command
+
+A second `runledger init` leaves those files unchanged. `--force` regenerates them.
+
+The demo exits non-zero unless all three of these hold:
+
+- **Schema.** The final output matches `evals/demo/schema.json` (`category` and `reply`).
+- **Tool order.** The agent calls `search_docs` (`call_order` in `evals/demo/cases/t1.yaml`).
+- **Tool-call budget.** The run stays within `max_tool_calls: 1`.
+
+A passing run prints `PASS` and writes `runledger_out/demo/<run_id>/` (`summary.json`, `junit.xml`, `report.html`, `run.jsonl`). Commit the init files to run that gate on pull requests.
+
+The same check as a GitHub Action. `suite.yaml` already points at `baselines/demo.json`, so the action compares against that baseline:
+
+```yaml
+- id: evals
+  uses: runledger/Runledger@v0.2
+  with:
+    path: ./evals/demo
+    mode: replay
+```
+
+This repo's `action.yml` also accepts a `baseline` input and sets `result` (`pass` or `fail`), `exit-code`, and `artifacts-path`. Those fields ship when a release containing this file is tagged. The in-repo self-test calls the action with `uses: ./`.
+
+Page copy for runledger.io lives in `docs/site-quickstart.md`.
+
+---
+
+RunLedger is a **CI harness**, not an "eval metrics framework":
+- **DeepEval-style tools** help you *score* behavior.
+- **RunLedger** helps you *ship safely* by making agent tests deterministic and merge-gated.
+
 ## Integrations (merged)
 
 - hoangsonww/Agentic-AI-Pipeline - https://github.com/hoangsonww/Agentic-AI-Pipeline/pull/24
@@ -73,7 +109,7 @@ RunLedger stops regressions by shifting from "vibes-based" evaluation to determi
 
 - **Record & replay:** record tool outputs once; replay them in CI for instant, stable tests.
 - **Strict contracts:** enforce tool schemas, calling order, and allowlists.
-- **Budget gating:** fail PRs automatically if wall time, tool-call count, or tool errors exceed defined limits.
+- **Budget gating:** fail the run when tool-call count exceeds `max_tool_calls`. Wall time and tool-error caps are separate gates, configured with `max_wall_ms` and `max_tool_errors`.
 
 ### RunLedger vs evaluation frameworks
 
@@ -352,18 +388,18 @@ jobs:
       - uses: actions/checkout@v4
 
       - name: Run deterministic evals (replay)
+        id: evals
         uses: runledger/Runledger@v0.2
         with:
           path: ./evals/demo
           mode: replay
 
-      - name: Upload eval artifacts
-        uses: actions/upload-artifact@v4
-        with:
-          name: agent-eval-artifacts
-          path: runledger_out/**
+      - name: Show the action result
+        run: echo "result=${{ steps.evals.outputs.result }} exit=${{ steps.evals.outputs.exit-code }}"
 ```
 
+`runledger init` writes `.github/workflows/runledger.yml`, which installs the CLI and runs the suite with `--baseline`. The action above is the Marketplace entry in `action.yml`. It uploads `runledger_out/**` itself. On a release that contains this `action.yml`, the step also sets `result`, `exit-code`, and `artifacts-path`, and you can pass `baseline:` to override `suite.yaml`.
+
 ---
 
 ## Determinism guide (rules of the road)
@@ -386,6 +422,7 @@ jobs:
 
 ## Docs
 
+- `docs/site-quickstart.md` -- five-minute copy for runledger.io
 - `docs/quickstart.md` -- install + first run
 - `docs/assertions.md` -- assertions, tool contracts, budgets
 - `docs/baselines.md` -- baselines, diffing, promotion
@@ -398,7 +435,6 @@ jobs:
 ## Roadmap
 
 * `init` templates for Python (OpenAI SDK), LangGraph/LangChain, and Node/TS
-* richer budgets (tokens/cost) via optional `task_metrics`
 * PR comments bot for regression summaries
 * plugin system for custom assertions
 * HTML report trace viewer improvements

runledger-0.2.0/src/runledger.egg-info/PKG-INFO → runledger-0.2.2/README.md

@@ -1,27 +1,3 @@
-Metadata-Version: 2.4
-Name: runledger
-Version: 0.2.0
-Summary: CI for tool-using agents: deterministic eval suites with record/replay, hard assertions, budgets, and PR regression gates.
-Author: RunLedger
-License-Expression: MIT
-Keywords: llm,agents,evaluation,ci,github-actions,replay,testing,junit,jsonschema
-Requires-Python: >=3.9
-Description-Content-Type: text/markdown
-License-File: LICENSE
-Requires-Dist: typer>=0.9.0
-Requires-Dist: rich>=13.0.0
-Requires-Dist: pydantic>=2.0.0
-Requires-Dist: pyyaml>=6.0.0
-Requires-Dist: jsonschema>=4.0.0
-Requires-Dist: jinja2>=3.1.0
-Requires-Dist: eval_type_backport>=0.2.0; python_version < "3.10"
-Provides-Extra: dev
-Requires-Dist: pytest>=7.0.0; extra == "dev"
-Requires-Dist: pytest-timeout>=2.1.0; extra == "dev"
-Requires-Dist: ruff>=0.1.0; extra == "dev"
-Requires-Dist: mypy>=1.0.0; extra == "dev"
-Dynamic: license-file
-
 # RunLedger
 
 [![CI](https://github.com/runledger/Runledger/actions/workflows/ci.yml/badge.svg)](https://github.com/runledger/Runledger/actions/workflows/ci.yml)
@@ -30,25 +6,57 @@ Dynamic: license-file
 
 Website: https://runledger.io
 
-**Deterministic CI regression checks for tool-using agents.** Record tool calls once, replay in CI, and fail PRs on mismatches, contract violations, or budget regressions.
+**Deterministic CI regression checks for tool-using agents.** Record tool calls once, replay them in CI, and fail the run when a schema, tool-order, or tool-call budget gate fails.
 
-- Demo repo: https://github.com/runledger/runledger-demo (main is green; regression branches fail for real reasons)
+- Demo: https://github.com/ZackMitchell910/runledger-demo/pull/1 (intentionally left open to show RunLedger failing a bad agent change)
 - If this is useful, please star the repo (it helps a lot).
 
-RunLedger is a **CI harness**, not an "eval metrics framework":
-- **DeepEval-style tools** help you *score* behavior.
-- **RunLedger** helps you *ship safely* by making agent tests deterministic and merge-gated.
+## Five-minute quickstart
 
----
-
-## Try it now (60 seconds)
+From an empty directory, with Python 3.9+:
 
 ```bash
-pipx install runledger
+python3 -m pip install runledger
 runledger init
 runledger run ./evals/demo --mode replay --baseline baselines/demo.json
 ```
 
+`runledger init` writes:
+
+- `evals/demo/` — replay suite, cassette, JSON Schema, and a small agent that calls `search_docs`
+- `baselines/demo.json` — baseline from that passing run
+- `.github/workflows/runledger.yml` — pull-request workflow that runs the same command
+
+A second `runledger init` leaves those files unchanged. `--force` regenerates them.
+
+The demo exits non-zero unless all three of these hold:
+
+- **Schema.** The final output matches `evals/demo/schema.json` (`category` and `reply`).
+- **Tool order.** The agent calls `search_docs` (`call_order` in `evals/demo/cases/t1.yaml`).
+- **Tool-call budget.** The run stays within `max_tool_calls: 1`.
+
+A passing run prints `PASS` and writes `runledger_out/demo/<run_id>/` (`summary.json`, `junit.xml`, `report.html`, `run.jsonl`). Commit the init files to run that gate on pull requests.
+
+The same check as a GitHub Action. `suite.yaml` already points at `baselines/demo.json`, so the action compares against that baseline:
+
+```yaml
+- id: evals
+  uses: runledger/Runledger@v0.2
+  with:
+    path: ./evals/demo
+    mode: replay
+```
+
+This repo's `action.yml` also accepts a `baseline` input and sets `result` (`pass` or `fail`), `exit-code`, and `artifacts-path`. Those fields ship when a release containing this file is tagged. The in-repo self-test calls the action with `uses: ./`.
+
+Page copy for runledger.io lives in `docs/site-quickstart.md`.
+
+---
+
+RunLedger is a **CI harness**, not an "eval metrics framework":
+- **DeepEval-style tools** help you *score* behavior.
+- **RunLedger** helps you *ship safely* by making agent tests deterministic and merge-gated.
+
 ## Integrations (merged)
 
 - hoangsonww/Agentic-AI-Pipeline - https://github.com/hoangsonww/Agentic-AI-Pipeline/pull/24
@@ -73,7 +81,7 @@ RunLedger stops regressions by shifting from "vibes-based" evaluation to determi
 
 - **Record & replay:** record tool outputs once; replay them in CI for instant, stable tests.
 - **Strict contracts:** enforce tool schemas, calling order, and allowlists.
-- **Budget gating:** fail PRs automatically if wall time, tool-call count, or tool errors exceed defined limits.
+- **Budget gating:** fail the run when tool-call count exceeds `max_tool_calls`. Wall time and tool-error caps are separate gates, configured with `max_wall_ms` and `max_tool_errors`.
 
 ### RunLedger vs evaluation frameworks
 
@@ -352,18 +360,18 @@ jobs:
       - uses: actions/checkout@v4
 
       - name: Run deterministic evals (replay)
+        id: evals
         uses: runledger/Runledger@v0.2
         with:
           path: ./evals/demo
           mode: replay
 
-      - name: Upload eval artifacts
-        uses: actions/upload-artifact@v4
-        with:
-          name: agent-eval-artifacts
-          path: runledger_out/**
+      - name: Show the action result
+        run: echo "result=${{ steps.evals.outputs.result }} exit=${{ steps.evals.outputs.exit-code }}"
 ```
 
+`runledger init` writes `.github/workflows/runledger.yml`, which installs the CLI and runs the suite with `--baseline`. The action above is the Marketplace entry in `action.yml`. It uploads `runledger_out/**` itself. On a release that contains this `action.yml`, the step also sets `result`, `exit-code`, and `artifacts-path`, and you can pass `baseline:` to override `suite.yaml`.
+
 ---
 
 ## Determinism guide (rules of the road)
@@ -386,6 +394,7 @@ jobs:
 
 ## Docs
 
+- `docs/site-quickstart.md` -- five-minute copy for runledger.io
 - `docs/quickstart.md` -- install + first run
 - `docs/assertions.md` -- assertions, tool contracts, budgets
 - `docs/baselines.md` -- baselines, diffing, promotion
@@ -398,7 +407,6 @@ jobs:
 ## Roadmap
 
 * `init` templates for Python (OpenAI SDK), LangGraph/LangChain, and Node/TS
-* richer budgets (tokens/cost) via optional `task_metrics`
 * PR comments bot for regression summaries
 * plugin system for custom assertions
 * HTML report trace viewer improvements

{runledger-0.2.0 → runledger-0.2.2}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "runledger"
-version = "0.2.0"
+version = "0.2.2"
 description = "CI for tool-using agents: deterministic eval suites with record/replay, hard assertions, budgets, and PR regression gates."
 readme = "README.md"
 requires-python = ">=3.9"
@@ -34,6 +34,12 @@ dependencies = [
   "eval_type_backport>=0.2.0; python_version < \"3.10\"",
 ]
 
+[project.urls]
+Homepage = "https://runledger.io"
+Repository = "https://github.com/runledger/Runledger"
+Documentation = "https://github.com/runledger/Runledger/tree/main/docs"
+Changelog = "https://github.com/runledger/Runledger/blob/main/CHANGELOG.md"
+
 [project.optional-dependencies]
 dev = [
   "pytest>=7.0.0",

{runledger-0.2.0 → runledger-0.2.2}/src/runledger/__init__.py

@@ -1,3 +1,3 @@
 __all__ = ["__version__"]
 
-__version__ = "0.2.0"
+__version__ = "0.2.2"