understudy 0.4.0__tar.gz → 0.6.0__tar.gz

understudy-0.6.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Gaurav Sood
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

{understudy-0.4.0 → understudy-0.6.0}/PKG-INFO

@@ -1,35 +1,30 @@
 Metadata-Version: 2.4
 Name: understudy
-Version: 0.4.0
+Version: 0.6.0
 Summary: Simulation and trace-based evaluation for agentic systems
 Keywords: agent,evaluation,simulation,testing,llm
 Author: Gaurav Sood
 License-Expression: MIT
+License-File: LICENSE
 Classifier: Development Status :: 3 - Alpha
 Classifier: Intended Audience :: Developers
-Classifier: License :: OSI Approved :: MIT License
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
 Classifier: Topic :: Software Development :: Testing
 Requires-Dist: pydantic>=2.0
 Requires-Dist: pyyaml>=6.0
 Requires-Dist: litellm>=1.50
-Requires-Dist: google-adk>=1.0 ; extra == 'adk'
+Requires-Dist: google-adk>=2.11 ; extra == 'adk'
 Requires-Dist: python-dotenv>=1.0 ; extra == 'adk'
-Requires-Dist: understudy[adk,http,langgraph,server,reports] ; extra == 'all'
-Requires-Dist: understudy[all,docs] ; extra == 'dev'
-Requires-Dist: pytest>=8.0 ; extra == 'dev'
-Requires-Dist: pytest-asyncio>=0.23 ; extra == 'dev'
-Requires-Dist: pytest-cov>=4.0 ; extra == 'dev'
-Requires-Dist: ruff>=0.4 ; extra == 'dev'
-Requires-Dist: sphinx>=7.0 ; extra == 'docs'
-Requires-Dist: furo>=2024.0 ; extra == 'docs'
-Requires-Dist: myst-parser>=3.0 ; extra == 'docs'
-Requires-Dist: sphinx-autodoc-typehints>=2.0 ; extra == 'docs'
+Requires-Dist: understudy[adk,http,langgraph,langchain,server,reports] ; extra == 'all'
 Requires-Dist: httpx>=0.27 ; extra == 'http'
-Requires-Dist: langgraph>=0.2 ; extra == 'langgraph'
-Requires-Dist: langchain-core>=0.3 ; extra == 'langgraph'
+Requires-Dist: langchain>=1.4.4 ; extra == 'langchain'
+Requires-Dist: langgraph>=1.2.14 ; extra == 'langchain'
+Requires-Dist: langchain-core>=1.6.9 ; extra == 'langchain'
+Requires-Dist: langgraph>=1.2.14 ; extra == 'langgraph'
+Requires-Dist: langchain-core>=1.6.9 ; extra == 'langgraph'
 Requires-Dist: langchain-openai>=0.3 ; extra == 'langgraph'
 Requires-Dist: jinja2>=3.0 ; extra == 'reports'
 Requires-Dist: click>=8.0 ; extra == 'reports'
@@ -42,9 +37,8 @@ Project-URL: Repository, https://github.com/gojiplus/understudy
 Project-URL: Issues, https://github.com/gojiplus/understudy/issues
 Provides-Extra: adk
 Provides-Extra: all
-Provides-Extra: dev
-Provides-Extra: docs
 Provides-Extra: http
+Provides-Extra: langchain
 Provides-Extra: langgraph
 Provides-Extra: reports
 Provides-Extra: server
@@ -65,22 +59,50 @@ Understudy is a scenario-driven testing framework for AI agents that simulates r
 
 Testing with understudy is **4 steps**:
 
-1. **Wrap your agent** — Adapt your agent (ADK, LangGraph, HTTP) to understudy's interface
+1. **Wrap your agent** — Adapt your agent (ADK, LangGraph, LangChain, HTTP) to understudy's interface
 2. **Mock your tools** — Register handlers that return test data instead of calling real services
 3. **Write scenes** — YAML files defining what the simulated user wants and what you expect
 4. **Run and assert** — Execute simulations, check traces, generate reports
 
 The key insight: **assert against the trace, not the prose**. Don't check what the agent said—check what it did (tool calls).
 
+## Two Evaluation Paradigms
+
+### Conversational Agent Evaluation
+Simulate multi-turn conversations with personas to test dialog agents.
+- Use case: Customer service bots, assistants, chatbots
+- Assert on tool calls: `trace.called("tool_name")`
+
+### Agentic Flow Evaluation
+Evaluate autonomous agents executing multi-step tasks.
+- Use case: Code agents, research agents, task automation
+- Assert on actions: `trace.performed("action")`
+
+See [examples/README.md](https://github.com/gojiplus/understudy/blob/main/examples/README.md) for complete examples of both paradigms.
+
 **See real examples:**
-- [Example scene](https://github.com/gojiplus/understudy/blob/main/example/scenes/return_eligible_backpack.yaml) — YAML defining a test scenario
-- [ADK test file](https://github.com/gojiplus/understudy/blob/main/example/adk/test_returns.py) — pytest assertions against traces
-- [LangGraph test file](https://github.com/gojiplus/understudy/blob/main/example/langgraph/test_returns.py) — same tests, different framework
+- [Example scene](https://github.com/gojiplus/understudy/blob/main/examples/scenes/return_eligible_backpack.yaml) — YAML defining a test scenario
+- [ADK test file](https://github.com/gojiplus/understudy/blob/main/examples/adk/test_adk_returns.py) — pytest assertions against traces
+- [LangGraph test file](https://github.com/gojiplus/understudy/blob/main/examples/langgraph/test_langgraph_returns.py) — same tests, different framework
+- [Agentic test file](https://github.com/gojiplus/understudy/blob/main/examples/agentic/test_agentic.py) — agentic flow evaluation
+- [Agentic scene](https://github.com/gojiplus/understudy/blob/main/examples/agentic_scenes/code_review_task.yaml) — task-based scenario
+- [Example report](https://htmlpreview.github.io/?https://github.com/gojiplus/understudy/blob/main/examples/langgraph/report/index.html) — HTML report with metrics and transcripts
+
+## State injection and next-turn tests
+
+Restore a completed checkpoint, inject synthetic history/state, or replay through
+exchange X, then execute and assert only against exchange X+1. ADK and LangGraph
+support isolated checkpoint forks; `LangChainApp` builds an instrumented LangChain
+agent with tool middleware. Traces retain nested invocation identities.
+
+See the [state-testing guide](https://gojiplus.github.io/understudy/state-testing.html)
+for `run_turn`, `replay_to`, scoped tool mocks, pytest fixtures, CLI commands, and
+framework-specific restoration limits.
 
 ## Installation
 
 ```bash
-pip install understudy[all]
+pip install "understudy[all]"
 ```
 
 ## Quick Start
@@ -103,9 +125,15 @@ from understudy.mocks import MockToolkit
 
 mocks = MockToolkit()
 
+
 @mocks.handle("lookup_order")
 def lookup_order(order_id: str) -> dict:
-    return {"order_id": order_id, "items": [...], "status": "delivered"}
+    return {
+        "order_id": order_id,
+        "items": [{"sku": "BACKPACK-01", "quantity": 1}],
+        "status": "delivered",
+    }
+
 
 @mocks.handle("create_return")
 def create_return(order_id: str, item_sku: str, reason: str) -> dict:
@@ -282,7 +310,7 @@ assert result.score == 1
 Built-in rubrics:
 
 ```python
-from understudy.judges import (
+from understudy.prompts import (
     TOOL_USAGE_CORRECTNESS,
     POLICY_COMPLIANCE,
     TONE_EMPATHY,

{understudy-0.4.0 → understudy-0.6.0}/README.md

@@ -13,22 +13,50 @@ Understudy is a scenario-driven testing framework for AI agents that simulates r
 
 Testing with understudy is **4 steps**:
 
-1. **Wrap your agent** — Adapt your agent (ADK, LangGraph, HTTP) to understudy's interface
+1. **Wrap your agent** — Adapt your agent (ADK, LangGraph, LangChain, HTTP) to understudy's interface
 2. **Mock your tools** — Register handlers that return test data instead of calling real services
 3. **Write scenes** — YAML files defining what the simulated user wants and what you expect
 4. **Run and assert** — Execute simulations, check traces, generate reports
 
 The key insight: **assert against the trace, not the prose**. Don't check what the agent said—check what it did (tool calls).
 
+## Two Evaluation Paradigms
+
+### Conversational Agent Evaluation
+Simulate multi-turn conversations with personas to test dialog agents.
+- Use case: Customer service bots, assistants, chatbots
+- Assert on tool calls: `trace.called("tool_name")`
+
+### Agentic Flow Evaluation
+Evaluate autonomous agents executing multi-step tasks.
+- Use case: Code agents, research agents, task automation
+- Assert on actions: `trace.performed("action")`
+
+See [examples/README.md](https://github.com/gojiplus/understudy/blob/main/examples/README.md) for complete examples of both paradigms.
+
 **See real examples:**
-- [Example scene](https://github.com/gojiplus/understudy/blob/main/example/scenes/return_eligible_backpack.yaml) — YAML defining a test scenario
-- [ADK test file](https://github.com/gojiplus/understudy/blob/main/example/adk/test_returns.py) — pytest assertions against traces
-- [LangGraph test file](https://github.com/gojiplus/understudy/blob/main/example/langgraph/test_returns.py) — same tests, different framework
+- [Example scene](https://github.com/gojiplus/understudy/blob/main/examples/scenes/return_eligible_backpack.yaml) — YAML defining a test scenario
+- [ADK test file](https://github.com/gojiplus/understudy/blob/main/examples/adk/test_adk_returns.py) — pytest assertions against traces
+- [LangGraph test file](https://github.com/gojiplus/understudy/blob/main/examples/langgraph/test_langgraph_returns.py) — same tests, different framework
+- [Agentic test file](https://github.com/gojiplus/understudy/blob/main/examples/agentic/test_agentic.py) — agentic flow evaluation
+- [Agentic scene](https://github.com/gojiplus/understudy/blob/main/examples/agentic_scenes/code_review_task.yaml) — task-based scenario
+- [Example report](https://htmlpreview.github.io/?https://github.com/gojiplus/understudy/blob/main/examples/langgraph/report/index.html) — HTML report with metrics and transcripts
+
+## State injection and next-turn tests
+
+Restore a completed checkpoint, inject synthetic history/state, or replay through
+exchange X, then execute and assert only against exchange X+1. ADK and LangGraph
+support isolated checkpoint forks; `LangChainApp` builds an instrumented LangChain
+agent with tool middleware. Traces retain nested invocation identities.
+
+See the [state-testing guide](https://gojiplus.github.io/understudy/state-testing.html)
+for `run_turn`, `replay_to`, scoped tool mocks, pytest fixtures, CLI commands, and
+framework-specific restoration limits.
 
 ## Installation
 
 ```bash
-pip install understudy[all]
+pip install "understudy[all]"
 ```
 
 ## Quick Start
@@ -51,9 +79,15 @@ from understudy.mocks import MockToolkit
 
 mocks = MockToolkit()
 
+
 @mocks.handle("lookup_order")
 def lookup_order(order_id: str) -> dict:
-    return {"order_id": order_id, "items": [...], "status": "delivered"}
+    return {
+        "order_id": order_id,
+        "items": [{"sku": "BACKPACK-01", "quantity": 1}],
+        "status": "delivered",
+    }
+
 
 @mocks.handle("create_return")
 def create_return(order_id: str, item_sku: str, reason: str) -> dict:
@@ -230,7 +264,7 @@ assert result.score == 1
 Built-in rubrics:
 
 ```python
-from understudy.judges import (
+from understudy.prompts import (
     TOOL_USAGE_CORRECTNESS,
     POLICY_COMPLIANCE,
     TONE_EMPATHY,

understudy-0.6.0/pyproject.toml

@@ -0,0 +1,207 @@
+[build-system]
+requires = ["uv_build>=0.12.5,<0.13"]
+build-backend = "uv_build"
+
+[project]
+name = "understudy"
+version = "0.6.0"
+description = "Simulation and trace-based evaluation for agentic systems"
+readme = "README.md"
+license = "MIT"
+license-files = ["LICENSE"]
+requires-python = ">=3.12"
+keywords = [
+    "agent",
+    "evaluation",
+    "simulation",
+    "testing",
+    "llm",
+]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Intended Audience :: Developers",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Programming Language :: Python :: 3.14",
+    "Topic :: Software Development :: Testing",
+]
+dependencies = [
+    "pydantic>=2.0",
+    "pyyaml>=6.0",
+    "litellm>=1.50",
+]
+
+[[project.authors]]
+name = "Gaurav Sood"
+
+[project.optional-dependencies]
+adk = [
+    "google-adk>=2.11",
+    "python-dotenv>=1.0",
+]
+http = ["httpx>=0.27"]
+langgraph = [
+    "langgraph>=1.2.14",
+    "langchain-core>=1.6.9",
+    "langchain-openai>=0.3",
+]
+server = [
+    "fastapi>=0.115",
+    "uvicorn>=0.32",
+]
+reports = [
+    "jinja2>=3.0",
+    "click>=8.0",
+]
+all = ["understudy[adk,http,langgraph,langchain,server,reports]"]
+langchain = [
+    "langchain>=1.4.4",
+    "langgraph>=1.2.14",
+    "langchain-core>=1.6.9",
+]
+
+[project.scripts]
+understudy = "understudy.cli:main"
+
+[project.entry-points.pytest11]
+understudy = "understudy.pytest_plugin"
+
+[project.urls]
+Homepage = "https://github.com/gojiplus/understudy"
+Documentation = "https://gojiplus.github.io/understudy"
+Repository = "https://github.com/gojiplus/understudy"
+Issues = "https://github.com/gojiplus/understudy/issues"
+
+[tool.ruff]
+line-length = 88
+target-version = "py312"
+
+[tool.ruff.lint]
+external = ["DOC"]
+select = [
+    "E",
+    "W",
+    "F",
+    "I",
+    "B",
+    "C4",
+    "UP",
+    "N",
+    "D",
+    "S",
+    "SIM",
+    "T20",
+    "PT",
+    "RUF",
+    "PTH",
+    "RET",
+    "PIE",
+    "FURB",
+    "PERF",
+    "DTZ",
+    "LOG",
+    "G",
+    "TC",
+    "FLY",
+    "RSE",
+    "SLOT",
+    "FA",
+    "A",
+    "EXE",
+    "ICN",
+    "PGH",
+    "PLE",
+    "ARG",
+    "SLF",
+]
+ignore = [
+    "D203",
+    "D213",
+    "W191",
+    "D206",
+    "D300",
+]
+
+[tool.ruff.lint.pydocstyle]
+convention = "google"
+
+[tool.ruff.lint.per-file-ignores]
+"tests/**" = [
+    "S101",
+    "D",
+    "ARG",
+    "SLF",
+    "S108",
+]
+"docs/**" = ["D"]
+"examples/**" = [
+    "S101",
+    "T20",
+    "D",
+    "ARG",
+    "A002",
+]
+"src/understudy/server/models.py" = ["N815"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+asyncio_mode = "auto"
+minversion = "6"
+log_level = "INFO"
+xfail_strict = true
+filterwarnings = ["error"]
+addopts = [
+    "--strict-config",
+    "--strict-markers",
+    "-ra",
+]
+
+[tool.coverage.run]
+source = ["src/understudy"]
+branch = true
+
+[tool.coverage.report]
+show_missing = true
+fail_under = 80
+
+[tool.pyright]
+include = ["src"]
+typeCheckingMode = "standard"
+
+[tool.pydoclint]
+style = "google"
+arg-type-hints-in-docstring = false
+check-return-types = false
+check-class-attributes = false
+allow-init-docstring = true
+exclude = '\.venv|tests|docs'
+
+[tool.codespell]
+builtin = "clear,rare,en-GB_to_en-US"
+
+[dependency-groups]
+docs = [
+    "sphinx>=7.0",
+    "furo>=2024.0",
+    "myst-parser>=3.0",
+    "sphinx-autodoc-typehints>=2.0",
+    "sphinx-copybutton",
+    "py-canon @ git+https://github.com/gojiplus/py-canon@v1",
+]
+test = [
+    "pytest>=8.0",
+    "pytest-asyncio>=0.23",
+    "pytest-cov>=4.0",
+    "httpx2>=2.12",
+]
+dev = [
+    "pytest>=8.0",
+    "pytest-asyncio>=0.23",
+    "pytest-cov>=4.0",
+    "ruff>=0.4",
+    { include-group = "test" },
+    { include-group = "docs" },
+    "pyright>=1.1.390",
+    "pydoclint>=0.5",
+]

understudy-0.6.0/pyproject.toml.orig

@@ -0,0 +1,151 @@
+[build-system]
+requires = ["uv_build>=0.12.5,<0.13"]
+build-backend = "uv_build"
+
+[project]
+name = "understudy"
+version = "0.6.0"
+description = "Simulation and trace-based evaluation for agentic systems"
+readme = "README.md"
+license = "MIT"
+license-files = ["LICENSE"]
+requires-python = ">=3.12"
+authors = [
+    { name = "Gaurav Sood" },
+]
+keywords = ["agent", "evaluation", "simulation", "testing", "llm"]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Intended Audience :: Developers",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Programming Language :: Python :: 3.14",
+    "Topic :: Software Development :: Testing",
+]
+dependencies = [
+    "pydantic>=2.0",
+    "pyyaml>=6.0",
+    "litellm>=1.50",
+]
+
+[project.optional-dependencies]
+adk = ["google-adk>=2.11", "python-dotenv>=1.0"]
+http = ["httpx>=0.27"]
+langgraph = ["langgraph>=1.2.14", "langchain-core>=1.6.9", "langchain-openai>=0.3"]
+server = ["fastapi>=0.115", "uvicorn>=0.32"]
+reports = ["jinja2>=3.0", "click>=8.0"]
+all = ["understudy[adk,http,langgraph,langchain,server,reports]"]
+langchain = [
+    "langchain>=1.4.4",
+    "langgraph>=1.2.14",
+    "langchain-core>=1.6.9",
+]
+
+[project.scripts]
+understudy = "understudy.cli:main"
+
+[project.entry-points.pytest11]
+understudy = "understudy.pytest_plugin"
+
+[project.urls]
+Homepage = "https://github.com/gojiplus/understudy"
+Documentation = "https://gojiplus.github.io/understudy"
+Repository = "https://github.com/gojiplus/understudy"
+Issues = "https://github.com/gojiplus/understudy/issues"
+
+[tool.ruff]
+line-length = 88
+target-version = "py312"
+
+[tool.ruff.lint]
+external = ["DOC"]
+select = [
+    "E", "W", "F", "I", "B", "C4", "UP", "N", "D", "S", "SIM", "T20", "PT", "RUF",
+    "PTH", "RET", "PIE", "FURB", "PERF", "DTZ", "LOG", "G", "TC", "FLY",
+    "RSE", "SLOT", "FA", "A", "EXE", "ICN", "PGH", "PLE", "ARG", "SLF",
+]
+# D203/D213: the google convention the fleet standard picks. W191/D206/D300:
+# ruff's own docs list these as always incompatible with `ruff format`, which
+# the standard also runs.
+ignore = ["D203", "D213", "W191", "D206", "D300"]
+
+[tool.ruff.lint.pydocstyle]
+convention = "google"
+
+[tool.ruff.lint.per-file-ignores]
+# S108: "/tmp/..." strings in tests are inert fixture data, never opened.
+"tests/**" = ["S101", "D", "ARG", "SLF", "S108"]
+"docs/**" = ["D"]
+# Runnable demo scripts: asserts and prints are their idiom, unused callback
+# arguments mirror the framework signatures they demonstrate, and mock tool
+# handlers must reuse the tool's own argument names (`format` included)
+# because mocks are invoked with keyword arguments from the scene.
+"examples/**" = ["S101", "T20", "D", "ARG", "A002"]
+# Request/response models mirror the UI client's camelCase wire schema.
+"src/understudy/server/models.py" = ["N815"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+asyncio_mode = "auto"
+minversion = "6"
+log_level = "INFO"
+xfail_strict = true
+filterwarnings = ["error"]
+addopts = ["--strict-config", "--strict-markers", "-ra"]
+
+[tool.coverage.run]
+source = ["src/understudy"]
+branch = true
+
+[tool.coverage.report]
+show_missing = true
+fail_under = 80
+
+[tool.pyright]
+include = ["src"]
+typeCheckingMode = "standard"
+
+[tool.pydoclint]
+style = "google"
+arg-type-hints-in-docstring = false
+check-return-types = false
+check-class-attributes = false
+allow-init-docstring = true
+exclude = '\.venv|tests|docs'
+
+[dependency-groups]
+# Development concerns, not user-facing optional features -- extras are for
+# what a consumer might install, and nobody installs understudy to build its
+# docs. CI syncs `--all-extras --all-groups`.
+docs = [
+    "sphinx>=7.0",
+    "furo>=2024.0",
+    "myst-parser>=3.0",
+    "sphinx-autodoc-typehints>=2.0",
+    "sphinx-copybutton",
+    "py-canon @ git+https://github.com/gojiplus/py-canon@v1",
+]
+test = [
+    "pytest>=8.0",
+    "pytest-asyncio>=0.23",
+    "pytest-cov>=4.0",
+    # starlette's TestClient (reached through fastapi.testclient) warns that
+    # driving it with httpx is deprecated and names httpx2 as the replacement.
+    # Under filterwarnings = ["error"] that notice fails collection of
+    # tests/test_server.py outright.
+    "httpx2>=2.12",
+]
+dev = [
+    "pytest>=8.0",
+    "pytest-asyncio>=0.23",
+    "pytest-cov>=4.0",
+    "ruff>=0.4",
+    { include-group = "test" },
+    { include-group = "docs" },
+    "pyright>=1.1.390",
+    "pydoclint>=0.5",
+]
+
+[tool.codespell]
+builtin = "clear,rare,en-GB_to_en-US"