evaldeck 0.1.0__tar.gz → 0.1.1__tar.gz

{evaldeck-0.1.0 → evaldeck-0.1.1}/.claude/settings.local.json

@@ -25,7 +25,10 @@
       "Bash(git branch:*)",
       "Bash(git remote add:*)",
       "Bash(git push:*)",
-      "Bash(ruff check:*)"
+      "Bash(ruff check:*)",
+      "Bash(source:*)",
+      "Bash(./venv/bin/python -m pytest:*)",
+      "Bash(uv run pytest:*)"
     ]
   }
 }

{evaldeck-0.1.0 → evaldeck-0.1.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: evaldeck
-Version: 0.1.0
+Version: 0.1.1
 Summary: The evaluation framework for AI agents. Pytest for agents.
 Project-URL: Homepage, https://github.com/tantra-run/evaldeck-py
 Project-URL: Documentation, https://tantra-run.github.io/evaldeck-py/

{evaldeck-0.1.0 → evaldeck-0.1.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "evaldeck"
-version = "0.1.0"
+version = "0.1.1"
 description = "The evaluation framework for AI agents. Pytest for agents."
 readme = "README.md"
 license = "Apache-2.0"

{evaldeck-0.1.0 → evaldeck-0.1.1}/src/evaldeck/evaluator.py

@@ -12,7 +12,9 @@ from evaldeck.graders import (
     BaseGrader,
     ContainsGrader,
     LLMGrader,
+    MaxLLMCallsGrader,
     MaxStepsGrader,
+    MaxToolCallsGrader,
     TaskCompletedGrader,
     ToolCalledGrader,
     ToolNotCalledGrader,
@@ -124,6 +126,12 @@ class Evaluator:
         if expected.max_steps is not None:
             graders.append(MaxStepsGrader())
 
+        if expected.max_tool_calls is not None:
+            graders.append(MaxToolCallsGrader())
+
+        if expected.max_llm_calls is not None:
+            graders.append(MaxLLMCallsGrader())
+
         if expected.task_completed is not None:
             graders.append(TaskCompletedGrader())
 

{evaldeck-0.1.0 → evaldeck-0.1.1}/src/evaldeck/graders/__init__.py

@@ -5,7 +5,9 @@ from evaldeck.graders.code import (
     ContainsGrader,
     CustomGrader,
     EqualsGrader,
+    MaxLLMCallsGrader,
     MaxStepsGrader,
+    MaxToolCallsGrader,
     NotContainsGrader,
     RegexGrader,
     TaskCompletedGrader,
@@ -28,6 +30,8 @@ __all__ = [
     "ToolNotCalledGrader",
     "ToolOrderGrader",
     "MaxStepsGrader",
+    "MaxToolCallsGrader",
+    "MaxLLMCallsGrader",
     "TaskCompletedGrader",
     "CustomGrader",
     # Model-based

{evaldeck-0.1.0 → evaldeck-0.1.1}/src/evaldeck/graders/code.py

@@ -341,6 +341,79 @@ class MaxStepsGrader(BaseGrader):
         )
 
 
+class MaxToolCallsGrader(BaseGrader):
+    """Check that agent completed within maximum tool calls.
+
+    Unlike max_steps which counts all trace steps (including internal
+    framework steps captured by OTel), this only counts actual tool calls.
+    """
+
+    name = "max_tool_calls"
+
+    def __init__(self, max_tool_calls: int | None = None) -> None:
+        self.max_tool_calls = max_tool_calls
+
+    def grade(self, trace: Trace, test_case: EvalCase) -> GradeResult:
+        """Check tool call count."""
+        max_tool_calls = self.max_tool_calls
+        if max_tool_calls is None:
+            max_tool_calls = test_case.expected.max_tool_calls
+
+        if max_tool_calls is None:
+            return GradeResult.passed_result(self.name, "No max tool calls defined")
+
+        actual = len(trace.tool_calls)
+
+        if actual <= max_tool_calls:
+            return GradeResult.passed_result(
+                self.name,
+                f"Made {actual} tool calls (max: {max_tool_calls})",
+            )
+
+        return GradeResult.failed_result(
+            self.name,
+            f"Too many tool calls: {actual} > {max_tool_calls}",
+            expected=max_tool_calls,
+            actual=actual,
+        )
+
+
+class MaxLLMCallsGrader(BaseGrader):
+    """Check that agent completed within maximum LLM calls.
+
+    Counts only LLM call steps, not internal framework steps.
+    """
+
+    name = "max_llm_calls"
+
+    def __init__(self, max_llm_calls: int | None = None) -> None:
+        self.max_llm_calls = max_llm_calls
+
+    def grade(self, trace: Trace, test_case: EvalCase) -> GradeResult:
+        """Check LLM call count."""
+        max_llm_calls = self.max_llm_calls
+        if max_llm_calls is None:
+            max_llm_calls = test_case.expected.max_llm_calls
+
+        if max_llm_calls is None:
+            return GradeResult.passed_result(self.name, "No max LLM calls defined")
+
+        actual = len(trace.llm_calls)
+
+        if actual <= max_llm_calls:
+            return GradeResult.passed_result(
+                self.name,
+                f"Made {actual} LLM calls (max: {max_llm_calls})",
+            )
+
+        return GradeResult.failed_result(
+            self.name,
+            f"Too many LLM calls: {actual} > {max_llm_calls}",
+            expected=max_llm_calls,
+            actual=actual,
+        )
+
+
 class TaskCompletedGrader(BaseGrader):
     """Check if the agent completed the task (based on trace status)."""
 

{evaldeck-0.1.0 → evaldeck-0.1.1}/src/evaldeck/test_case.py

@@ -26,6 +26,8 @@ class ExpectedBehavior(BaseModel):
     # Execution expectations
     max_steps: int | None = None
     min_steps: int | None = None
+    max_tool_calls: int | None = None
+    max_llm_calls: int | None = None
     task_completed: bool | None = None
 
     # Custom assertions (for code-based graders)