understudy 0.4.0__tar.gz → 0.5.0__tar.gz

{understudy-0.4.0 → understudy-0.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: understudy
-Version: 0.4.0
+Version: 0.5.0
 Summary: Simulation and trace-based evaluation for agentic systems
 Keywords: agent,evaluation,simulation,testing,llm
 Author: Gaurav Sood
@@ -72,10 +72,27 @@ Testing with understudy is **4 steps**:
 
 The key insight: **assert against the trace, not the prose**. Don't check what the agent said—check what it did (tool calls).
 
+## Two Evaluation Paradigms
+
+### Conversational Agent Evaluation
+Simulate multi-turn conversations with personas to test dialogue agents.
+- Use case: Customer service bots, assistants, chatbots
+- Assert on tool calls: `trace.called("tool_name")`
+
+### Agentic Flow Evaluation
+Evaluate autonomous agents executing multi-step tasks.
+- Use case: Code agents, research agents, task automation
+- Assert on actions: `trace.performed("action")`
+
+See [examples/README.md](examples/README.md) for complete examples of both paradigms.
+
 **See real examples:**
-- [Example scene](https://github.com/gojiplus/understudy/blob/main/example/scenes/return_eligible_backpack.yaml) — YAML defining a test scenario
-- [ADK test file](https://github.com/gojiplus/understudy/blob/main/example/adk/test_returns.py) — pytest assertions against traces
-- [LangGraph test file](https://github.com/gojiplus/understudy/blob/main/example/langgraph/test_returns.py) — same tests, different framework
+- [Example scene](https://github.com/gojiplus/understudy/blob/main/examples/scenes/return_eligible_backpack.yaml) — YAML defining a test scenario
+- [ADK test file](https://github.com/gojiplus/understudy/blob/main/examples/adk/test_adk_returns.py) — pytest assertions against traces
+- [LangGraph test file](https://github.com/gojiplus/understudy/blob/main/examples/langgraph/test_langgraph_returns.py) — same tests, different framework
+- [Agentic test file](https://github.com/gojiplus/understudy/blob/main/examples/agentic/test_agentic.py) — agentic flow evaluation
+- [Agentic scene](https://github.com/gojiplus/understudy/blob/main/examples/agentic_scenes/code_review_task.yaml) — task-based scenario
+- [Example report](https://htmlpreview.github.io/?https://github.com/gojiplus/understudy/blob/main/examples/langgraph/report/index.html) — HTML report with metrics and transcripts
 
 ## Installation
 

{understudy-0.4.0 → understudy-0.5.0}/README.md

@@ -20,10 +20,27 @@ Testing with understudy is **4 steps**:
 
 The key insight: **assert against the trace, not the prose**. Don't check what the agent said—check what it did (tool calls).
 
+## Two Evaluation Paradigms
+
+### Conversational Agent Evaluation
+Simulate multi-turn conversations with personas to test dialogue agents.
+- Use case: Customer service bots, assistants, chatbots
+- Assert on tool calls: `trace.called("tool_name")`
+
+### Agentic Flow Evaluation
+Evaluate autonomous agents executing multi-step tasks.
+- Use case: Code agents, research agents, task automation
+- Assert on actions: `trace.performed("action")`
+
+See [examples/README.md](examples/README.md) for complete examples of both paradigms.
+
 **See real examples:**
-- [Example scene](https://github.com/gojiplus/understudy/blob/main/example/scenes/return_eligible_backpack.yaml) — YAML defining a test scenario
-- [ADK test file](https://github.com/gojiplus/understudy/blob/main/example/adk/test_returns.py) — pytest assertions against traces
-- [LangGraph test file](https://github.com/gojiplus/understudy/blob/main/example/langgraph/test_returns.py) — same tests, different framework
+- [Example scene](https://github.com/gojiplus/understudy/blob/main/examples/scenes/return_eligible_backpack.yaml) — YAML defining a test scenario
+- [ADK test file](https://github.com/gojiplus/understudy/blob/main/examples/adk/test_adk_returns.py) — pytest assertions against traces
+- [LangGraph test file](https://github.com/gojiplus/understudy/blob/main/examples/langgraph/test_langgraph_returns.py) — same tests, different framework
+- [Agentic test file](https://github.com/gojiplus/understudy/blob/main/examples/agentic/test_agentic.py) — agentic flow evaluation
+- [Agentic scene](https://github.com/gojiplus/understudy/blob/main/examples/agentic_scenes/code_review_task.yaml) — task-based scenario
+- [Example report](https://htmlpreview.github.io/?https://github.com/gojiplus/understudy/blob/main/examples/langgraph/report/index.html) — HTML report with metrics and transcripts
 
 ## Installation
 

{understudy-0.4.0 → understudy-0.5.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "uv_build"
 
 [project]
 name = "understudy"
-version = "0.4.0"
+version = "0.5.0"
 description = "Simulation and trace-based evaluation for agentic systems"
 readme = "README.md"
 license = "MIT"
@@ -52,6 +52,9 @@ dev = [
 [project.scripts]
 understudy = "understudy.cli:main"
 
+[project.entry-points.pytest11]
+understudy = "understudy.pytest_plugin"
+
 [project.urls]
 Homepage = "https://github.com/gojiplus/understudy"
 Documentation = "https://gojiplus.github.io/understudy"

{understudy-0.4.0 → understudy-0.5.0}/src/understudy/__init__.py

@@ -7,8 +7,26 @@ not by reading the script, but by inspecting what actually happened.
 
 from importlib.metadata import version
 
+from . import agentic
+from .agentic import (
+    AgenticApp,
+    AgenticCheckItem,
+    AgenticCheckResult,
+    AgenticExpectations,
+    AgenticMetricResult,
+    AgenticScene,
+    AgenticTrace,
+    Artifact,
+    Step,
+    StepResult,
+    Task,
+    check_agentic,
+    run_agentic,
+)
 from .check import CheckItem, CheckResult, EvaluationResult, check, evaluate, evaluate_batch
 from .compare import ComparisonResult, SceneComparison, compare_runs
+from .diff import ToolCallDiff, TraceDiff, diff_tool_sequences, diff_traces
+from .judge_backends import CallbackBackend, JudgeBackend, LiteLLMBackend
 from .judges import FailureAnalysis, FailureAnalyzer, Judge, JudgeResult
 from .metrics import MetricRegistry, MetricResult
 from .mocks import MockToolkit, ToolError
@@ -22,11 +40,14 @@ from .prompts import (
     TONE_EMPATHY,
     TOOL_USAGE_CORRECTNESS,
 )
-from .runner import AgentApp, AgentResponse, LiteLLMBackend, run, simulate, simulate_batch
+from .pytest_plugin import AssertionHelpers
+from .replay import ReplayResult, create_replay_scene, load_trace, replay
+from .runner import AgentApp, AgentResponse, run, simulate, simulate_batch
 from .simulator import Simulator, SimulatorBackend
 from .storage import EvaluationStorage, RunStorage, TraceStorage
 from .suite import SceneResult, Suite, SuiteResults
 from .trace import AgentTransfer, StateSnapshot, ToolCall, Trace, TraceMetrics, Turn, TurnMetrics
+from .validation import SceneValidationError
 
 __version__ = version("understudy")
 
@@ -73,14 +94,32 @@ __all__ = [
     "compare_runs",
     "ComparisonResult",
     "SceneComparison",
+    # diff
+    "diff_traces",
+    "diff_tool_sequences",
+    "TraceDiff",
+    "ToolCallDiff",
+    # replay
+    "replay",
+    "load_trace",
+    "create_replay_scene",
+    "ReplayResult",
     # judges
     "Judge",
     "JudgeResult",
     "FailureAnalyzer",
     "FailureAnalysis",
+    # judge backends
+    "JudgeBackend",
+    "LiteLLMBackend",
+    "CallbackBackend",
     # mocks
     "MockToolkit",
     "ToolError",
+    # validation
+    "SceneValidationError",
+    # pytest
+    "AssertionHelpers",
     # simulator
     "Simulator",
     "SimulatorBackend",
@@ -92,4 +131,19 @@ __all__ = [
     "TASK_COMPLETION",
     "FACTUAL_GROUNDING",
     "INSTRUCTION_FOLLOWING",
+    # agentic module
+    "agentic",
+    "AgenticApp",
+    "AgenticCheckItem",
+    "AgenticCheckResult",
+    "AgenticExpectations",
+    "AgenticMetricResult",
+    "AgenticScene",
+    "AgenticTrace",
+    "Artifact",
+    "Step",
+    "StepResult",
+    "Task",
+    "check_agentic",
+    "run_agentic",
 ]

understudy-0.5.0/src/understudy/agentic/__init__.py

@@ -0,0 +1,53 @@
+"""Agentic flow evaluation module for understudy.
+
+This module provides evaluation capabilities for autonomous agent flows,
+complementing the conversational agent evaluation in the main understudy module.
+"""
+
+from .check import (
+    AgenticCheckItem,
+    AgenticCheckResult,
+    check_agentic,
+)
+from .metrics import (
+    AgenticMetricResult,
+    action_efficiency,
+    action_trajectory,
+    compute_all_metrics,
+    goal_completion,
+    reasoning_quality,
+)
+from .models import (
+    AgenticExpectations,
+    AgenticScene,
+    AgenticTrace,
+    Artifact,
+    Step,
+    Task,
+)
+from .runner import (
+    AgenticApp,
+    StepResult,
+    run_agentic,
+)
+
+__all__ = [
+    "AgenticApp",
+    "AgenticCheckItem",
+    "AgenticCheckResult",
+    "AgenticExpectations",
+    "AgenticMetricResult",
+    "AgenticScene",
+    "AgenticTrace",
+    "Artifact",
+    "Step",
+    "StepResult",
+    "Task",
+    "action_efficiency",
+    "action_trajectory",
+    "check_agentic",
+    "compute_all_metrics",
+    "goal_completion",
+    "reasoning_quality",
+    "run_agentic",
+]

understudy-0.5.0/src/understudy/agentic/check.py

@@ -0,0 +1,198 @@
+"""Check: validate an agentic trace against expectations."""
+
+import fnmatch
+from dataclasses import dataclass, field
+from typing import Any
+
+from .models import AgenticExpectations, AgenticTrace
+
+
+@dataclass
+class AgenticCheckItem:
+    """A single check result for agentic evaluation."""
+
+    label: str
+    passed: bool
+    detail: str
+
+
+@dataclass
+class AgenticCheckResult:
+    """Result of checking an agentic trace against expectations."""
+
+    checks: list[AgenticCheckItem] = field(default_factory=list)
+    metrics: dict[str, Any] = field(default_factory=dict)
+
+    @property
+    def passed(self) -> bool:
+        return all(c.passed for c in self.checks)
+
+    @property
+    def failed_checks(self) -> list[AgenticCheckItem]:
+        return [c for c in self.checks if not c.passed]
+
+    def summary(self) -> str:
+        lines = []
+        for c in self.checks:
+            mark = "+" if c.passed else "-"
+            lines.append(f"  {mark} {c.label}: {c.detail}")
+        for name, value in self.metrics.items():
+            lines.append(f"  * {name}: {value}")
+        return "\n".join(lines)
+
+    def __repr__(self) -> str:
+        n_checks = len(self.checks)
+        n_pass = sum(1 for c in self.checks if c.passed)
+        return f"AgenticCheckResult({n_pass}/{n_checks} passed)"
+
+
+def _matches_pattern(action: str, pattern: str) -> bool:
+    """Check if an action matches a pattern (supports wildcards)."""
+    if "*" in pattern or "?" in pattern:
+        return fnmatch.fnmatch(action, pattern)
+    return action == pattern
+
+
+def _evaluate_predicate(predicate: str, trace: AgenticTrace) -> bool:
+    """Evaluate a goal predicate against the trace.
+
+    Supports:
+    - Simple expressions: "outcome == 'success'"
+    - Access to trace fields: outcome, total_steps, total_tokens, final_state
+    """
+    local_vars = {
+        "outcome": trace.outcome,
+        "total_steps": trace.total_steps,
+        "total_tokens": trace.total_tokens,
+        "final_state": trace.final_state,
+        "artifacts": [a.model_dump() for a in trace.artifacts],
+        "steps": trace.steps,
+    }
+    try:
+        result = eval(predicate, {"__builtins__": {}}, local_vars)
+        return bool(result)
+    except Exception:
+        return False
+
+
+def _compare_outputs(actual: dict, expected: dict) -> tuple[bool, str]:
+    """Compare actual output against golden output."""
+    differences = []
+
+    for key, expected_value in expected.items():
+        if key not in actual:
+            differences.append(f"missing key: {key}")
+        elif actual[key] != expected_value:
+            differences.append(f"{key}: expected {expected_value}, got {actual[key]}")
+
+    if differences:
+        return False, "; ".join(differences)
+    return True, "output matches expected"
+
+
+def check_agentic(trace: AgenticTrace, expectations: AgenticExpectations) -> AgenticCheckResult:
+    """Validate an agentic trace against expectations.
+
+    Args:
+        trace: The agentic execution trace.
+        expectations: The expectations to check against.
+
+    Returns:
+        An AgenticCheckResult with individual check outcomes.
+    """
+    result = AgenticCheckResult()
+    performed_actions = set(trace.action_sequence())
+
+    if expectations.goal_predicate:
+        predicate_passed = _evaluate_predicate(expectations.goal_predicate, trace)
+        result.checks.append(
+            AgenticCheckItem(
+                label="goal_predicate",
+                passed=predicate_passed,
+                detail=(
+                    f"predicate '{expectations.goal_predicate}' "
+                    f"{'satisfied' if predicate_passed else 'NOT satisfied'}"
+                ),
+            )
+        )
+
+    if expectations.golden_output:
+        output_passed, output_detail = _compare_outputs(
+            trace.final_state, expectations.golden_output
+        )
+        result.checks.append(
+            AgenticCheckItem(
+                label="golden_output",
+                passed=output_passed,
+                detail=output_detail,
+            )
+        )
+
+    if expectations.max_steps is not None:
+        steps_ok = trace.total_steps <= expectations.max_steps
+        result.checks.append(
+            AgenticCheckItem(
+                label="max_steps",
+                passed=steps_ok,
+                detail=(
+                    f"steps={trace.total_steps}, max={expectations.max_steps} "
+                    f"({'OK' if steps_ok else 'EXCEEDED'})"
+                ),
+            )
+        )
+
+    if expectations.max_tokens is not None:
+        tokens_ok = trace.total_tokens <= expectations.max_tokens
+        result.checks.append(
+            AgenticCheckItem(
+                label="max_tokens",
+                passed=tokens_ok,
+                detail=(
+                    f"tokens={trace.total_tokens}, max={expectations.max_tokens} "
+                    f"({'OK' if tokens_ok else 'EXCEEDED'})"
+                ),
+            )
+        )
+
+    if expectations.max_retries is not None:
+        retries = trace.retry_count()
+        retries_ok = retries <= expectations.max_retries
+        result.checks.append(
+            AgenticCheckItem(
+                label="max_retries",
+                passed=retries_ok,
+                detail=(
+                    f"retries={retries}, max={expectations.max_retries} "
+                    f"({'OK' if retries_ok else 'EXCEEDED'})"
+                ),
+            )
+        )
+
+    for action in expectations.required_actions:
+        found = any(_matches_pattern(a, action) for a in performed_actions)
+        result.checks.append(
+            AgenticCheckItem(
+                label="required_action",
+                passed=found,
+                detail=f"{action} {'performed' if found else 'NOT performed'}",
+            )
+        )
+
+    for pattern in expectations.forbidden_actions:
+        violations = [a for a in performed_actions if _matches_pattern(a, pattern)]
+        passed = len(violations) == 0
+        result.checks.append(
+            AgenticCheckItem(
+                label="forbidden_action",
+                passed=passed,
+                detail=f"{pattern} VIOLATED by {violations}" if violations else f"{pattern} OK",
+            )
+        )
+
+    result.metrics["outcome"] = trace.outcome
+    result.metrics["total_steps"] = trace.total_steps
+    result.metrics["total_tokens"] = trace.total_tokens
+    result.metrics["total_latency_ms"] = trace.total_latency_ms
+    result.metrics["retry_count"] = trace.retry_count()
+
+    return result

understudy-0.5.0/src/understudy/agentic/metrics.py

@@ -0,0 +1,170 @@
+"""Metrics: agentic-specific evaluation metrics."""
+
+from dataclasses import dataclass
+from typing import Any
+
+from .models import AgenticExpectations, AgenticTrace
+
+
+@dataclass
+class AgenticMetricResult:
+    """Result of computing an agentic metric."""
+
+    name: str
+    value: Any
+    passed: bool | None = None
+    detail: str = ""
+
+    def __repr__(self) -> str:
+        if self.passed is not None:
+            status = "passed" if self.passed else "failed"
+            return f"AgenticMetricResult({self.name}: {status})"
+        return f"AgenticMetricResult({self.name}: {self.value})"
+
+
+def goal_completion(trace: AgenticTrace) -> AgenticMetricResult:
+    """Compute goal completion metric based on outcome.
+
+    Returns 1.0 for success, 0.0 for failure/error.
+    """
+    success_outcomes = {"success", "completed", "done", "finished"}
+    completed = trace.outcome.lower() in success_outcomes
+
+    return AgenticMetricResult(
+        name="goal_completion",
+        value=1.0 if completed else 0.0,
+        passed=completed,
+        detail=f"outcome='{trace.outcome}'",
+    )
+
+
+def reasoning_quality(trace: AgenticTrace) -> AgenticMetricResult:
+    """Analyze the quality of reasoning steps.
+
+    Computes basic metrics about the reasoning process:
+    - Number of thinking steps
+    - Average reasoning length
+    - Think-before-act ratio
+    """
+    thinking_steps = trace.thinking_steps()
+    action_steps = [s for s in trace.steps if s.step_type == "act"]
+
+    n_thinking = len(thinking_steps)
+    n_actions = len(action_steps)
+
+    if n_thinking == 0:
+        return AgenticMetricResult(
+            name="reasoning_quality",
+            value={
+                "thinking_steps": 0,
+                "action_steps": n_actions,
+                "think_act_ratio": 0.0,
+                "avg_reasoning_length": 0,
+            },
+            detail="No thinking steps recorded",
+        )
+
+    total_reasoning_length = sum(len(s.reasoning or "") for s in thinking_steps)
+    avg_reasoning_length = total_reasoning_length / n_thinking
+
+    think_act_ratio = n_thinking / max(n_actions, 1)
+
+    return AgenticMetricResult(
+        name="reasoning_quality",
+        value={
+            "thinking_steps": n_thinking,
+            "action_steps": n_actions,
+            "think_act_ratio": round(think_act_ratio, 2),
+            "avg_reasoning_length": int(avg_reasoning_length),
+        },
+        detail=f"{n_thinking} thinking steps, ratio={think_act_ratio:.2f}",
+    )
+
+
+def action_efficiency(
+    trace: AgenticTrace, expectations: AgenticExpectations | None = None
+) -> AgenticMetricResult:
+    """Compute action efficiency metrics.
+
+    Includes:
+    - Steps per action (lower is better)
+    - Retry rate (lower is better)
+    - Token efficiency
+    """
+    total_steps = trace.total_steps
+    action_steps = [s for s in trace.steps if s.step_type == "act"]
+    n_actions = len(action_steps)
+    n_retries = trace.retry_count()
+    total_tokens = trace.total_tokens
+
+    if n_actions == 0:
+        return AgenticMetricResult(
+            name="action_efficiency",
+            value={
+                "total_steps": total_steps,
+                "action_steps": 0,
+                "retry_count": 0,
+                "retry_rate": 0.0,
+                "tokens_per_action": 0,
+            },
+            detail="No actions performed",
+        )
+
+    retry_rate = n_retries / n_actions
+    tokens_per_action = total_tokens / n_actions if total_tokens > 0 else 0
+
+    efficiency_passed = None
+    if expectations:
+        checks = []
+        if expectations.max_steps is not None:
+            checks.append(total_steps <= expectations.max_steps)
+        if expectations.max_retries is not None:
+            checks.append(n_retries <= expectations.max_retries)
+        if expectations.max_tokens is not None:
+            checks.append(total_tokens <= expectations.max_tokens)
+        if checks:
+            efficiency_passed = all(checks)
+
+    return AgenticMetricResult(
+        name="action_efficiency",
+        value={
+            "total_steps": total_steps,
+            "action_steps": n_actions,
+            "retry_count": n_retries,
+            "retry_rate": round(retry_rate, 3),
+            "tokens_per_action": int(tokens_per_action),
+            "total_tokens": total_tokens,
+            "total_latency_ms": trace.total_latency_ms,
+        },
+        passed=efficiency_passed,
+        detail=f"{n_actions} actions, {n_retries} retries, {total_tokens} tokens",
+    )
+
+
+def action_trajectory(trace: AgenticTrace) -> AgenticMetricResult:
+    """Compute action trajectory metrics."""
+    sequence = trace.action_sequence()
+    unique_actions = list(set(sequence))
+
+    return AgenticMetricResult(
+        name="action_trajectory",
+        value={
+            "sequence": sequence,
+            "unique_actions": unique_actions,
+            "total_actions": len(sequence),
+            "unique_count": len(unique_actions),
+        },
+        detail=f"{len(sequence)} actions, {len(unique_actions)} unique",
+    )
+
+
+def compute_all_metrics(
+    trace: AgenticTrace, expectations: AgenticExpectations | None = None
+) -> dict[str, AgenticMetricResult]:
+    """Compute all agentic metrics."""
+    return {
+        "goal_completion": goal_completion(trace),
+        "reasoning_quality": reasoning_quality(trace),
+        "action_efficiency": action_efficiency(trace, expectations),
+        "action_trajectory": action_trajectory(trace),
+    }