agentdog 0.1.0__py3-none-any.whl

agentdog/__init__.py

@@ -0,0 +1,104 @@
+"""
+agentdog: lightweight evaluation toolkit for AI agents.
+"""
+
+from .case import EvalRun, TestCase
+from .report import CaseResult, Report, ScorerResult
+from .runner import run
+from .scorers import (
+    AnswerNotEmpty,
+    AvoidedTools,
+    CitedSource,
+    ContainsAnswer,
+    ExactAnswer,
+    ForbiddenContent,
+    GroundedInContext,
+    LLMJudge,
+    MaxRetries,
+    MaxToolCalls,
+    NoContextHallucination,
+    NoRiskyActionTaken,
+    NoSensitiveDataLeaked,
+    PromptInjectionResisted,
+    RegexAnswer,
+    ScoreResult,
+    Scorer,
+    ToolArgContains,
+    ToolArgEquals,
+    ToolCallOrder,
+    UnderCostLimit,
+    UnderLatencyLimit,
+    UnderTokenLimit,
+    UsedTools,
+)
+from .trace import AgentTrace, ToolCall
+
+__version__ = "0.1.0"
+
+__all__ = [
+    # trace
+    "AgentTrace",
+    "ToolCall",
+    # case
+    "TestCase",
+    "EvalRun",
+    # runner
+    "run",
+    # report
+    "Report",
+    "CaseResult",
+    "ScorerResult",
+    # scorers — base
+    "Scorer",
+    "ScoreResult",
+    # scorers — answer
+    "ContainsAnswer",
+    "ExactAnswer",
+    "RegexAnswer",
+    "ForbiddenContent",
+    "AnswerNotEmpty",
+    # scorers — tools
+    "UsedTools",
+    "AvoidedTools",
+    "ToolCallOrder",
+    "MaxToolCalls",
+    "ToolArgContains",
+    "ToolArgEquals",
+    # scorers — grounding
+    "GroundedInContext",
+    "CitedSource",
+    "NoContextHallucination",
+    # scorers — safety
+    "NoSensitiveDataLeaked",
+    "NoRiskyActionTaken",
+    "PromptInjectionResisted",
+    # scorers — efficiency
+    "UnderTokenLimit",
+    "UnderCostLimit",
+    "UnderLatencyLimit",
+    "MaxRetries",
+    # scorers — llm judge
+    "LLMJudge",
+]
+
+# ---------------------------------------------------------------------------
+# Aspirational features — tracked here until implemented
+# ---------------------------------------------------------------------------
+#
+# v2 candidates (medium effort, clear value):
+#   - human_in_the_loop_review: structured queue for human-reviewed cases
+#   - evaluation_packs: pre-built scorer bundles (rag_pack, security_pack, etc.)
+#   - html_reports: rich HTML report with drill-down per case
+#   - github_action: official GHA for agentdog run with PR comment integration
+#   - async_runner: parallel trace evaluation for large eval sets
+#
+# v3 candidates (higher effort or less certain):
+#   - framework_adapters: native converters for LangChain, LlamaIndex, OpenAI SDK traces
+#   - model_comparison: run same cases against multiple models, diff results
+#   - prompt_version_comparison: A/B eval across prompt variants
+#   - trace_replay: re-run a captured trace through a new model/prompt
+#   - synthetic_eval_generation: auto-generate eval cases from a prompt + schema
+#   - dataset_quality_checks: flag low-quality or duplicate eval cases
+#   - safety_governance_templates: pre-built packs for HIPAA, PCI, SOC2 patterns
+#   - rag_advanced: retrieval-specific metrics (NDCG, MRR, recall@k)
+#   - streaming_trace_support: capture and eval streaming agent runs

agentdog/case.py

@@ -0,0 +1,26 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from agentdog.scorers.base import Scorer
+    from agentdog.trace import AgentTrace
+
+
+@dataclass
+class TestCase:
+    """Defines what to evaluate and how to evaluate it for a single agent input."""
+
+    name: str
+    scorers: list["Scorer"]
+    description: str = ""
+    tags: list[str] = field(default_factory=list)
+
+
+@dataclass
+class EvalRun:
+    """A test case paired with the agent trace to evaluate against."""
+
+    case: TestCase
+    trace: "AgentTrace"

agentdog/cli.py

@@ -0,0 +1,130 @@
+from __future__ import annotations
+
+import importlib.util
+import json
+import sys
+from pathlib import Path
+
+import click
+
+from .case import EvalRun
+from .runner import run
+from .trace import AgentTrace
+
+
+def _load_module(path: Path):
+    spec = importlib.util.spec_from_file_location("_agentdog_module", path)
+    if spec is None or spec.loader is None:
+        raise click.ClickException(f"Cannot load module from {path}")
+    mod = importlib.util.module_from_spec(spec)
+    sys.path.insert(0, str(path.parent))
+    spec.loader.exec_module(mod)  # type: ignore[arg-type]
+    return mod
+
+
+@click.group()
+def main():
+    """agentdog — lightweight evaluation for AI agents."""
+
+
+@main.command()
+@click.argument("module", type=click.Path(exists=True, dir_okay=False, path_type=Path))
+@click.option("--verbose", "-v", is_flag=True, help="Show scorer details for passing cases too.")
+@click.option("--fail-fast", is_flag=True, help="Stop after the first failing case.")
+@click.option("--tag", "tags", multiple=True, help="Only run cases matching these tags.")
+@click.option("--json-out", type=click.Path(dir_okay=False, path_type=Path), help="Write JSON report to file.")
+def run_cmd(module: Path, verbose: bool, fail_fast: bool, tags: tuple[str, ...], json_out: Path | None):
+    """
+    Run evaluations defined in MODULE.
+
+    MODULE must be a Python file that exposes an `evals()` function returning
+    a list of EvalRun objects.
+
+    \b
+    Example module (my_evals.py):
+        from agentdog import EvalRun, TestCase, AgentTrace
+        from agentdog.scorers import ContainsAnswer, UsedTools
+
+        def evals():
+            trace = AgentTrace(input="What is 2+2?", output="The answer is 4.")
+            case  = TestCase("basic-math", scorers=[ContainsAnswer(["4"])])
+            return [EvalRun(case=case, trace=trace)]
+    """
+    mod = _load_module(module)
+    if not hasattr(mod, "evals"):
+        raise click.ClickException(f"{module} must define an evals() function")
+
+    eval_runs: list[EvalRun] = mod.evals()
+
+    if tags:
+        eval_runs = [er for er in eval_runs if set(er.case.tags) & set(tags)]
+        if not eval_runs:
+            click.echo(f"No cases matched tags: {list(tags)}")
+            return
+
+    if fail_fast:
+        filtered: list[EvalRun] = []
+        for er in eval_runs:
+            filtered.append(er)
+            sub = run([er])
+            if not sub.passed:
+                report = run(filtered)
+                report.print(verbose=verbose)
+                sys.exit(1)
+        eval_runs = filtered
+
+    report = run(eval_runs)
+    report.print(verbose=verbose)
+
+    if json_out:
+        _write_json_report(report, json_out)
+        click.echo(f"JSON report written to {json_out}")
+
+    sys.exit(0 if report.passed else 1)
+
+
+@main.command()
+@click.argument("trace_file", type=click.Path(exists=True, dir_okay=False, path_type=Path))
+def inspect(trace_file: Path):
+    """Pretty-print a trace JSON file."""
+    trace = AgentTrace.from_json(str(trace_file))
+    click.echo(f"\nInput:   {trace.input}")
+    click.echo(f"Output:  {trace.output}")
+    click.echo(f"Tools:   {trace.tool_names or '(none)'}")
+    click.echo(f"Context: {len(trace.retrieved_context)} chunk(s)")
+    if trace.total_tokens is not None:
+        click.echo(f"Tokens:  {trace.total_tokens}")
+    if trace.total_cost_usd is not None:
+        click.echo(f"Cost:    ${trace.total_cost_usd:.4f}")
+    if trace.total_latency_ms is not None:
+        click.echo(f"Latency: {trace.total_latency_ms:.0f}ms")
+    click.echo()
+
+
+def _write_json_report(report, path: Path) -> None:
+    data = {
+        "passed": report.passed,
+        "overall_score": report.overall_score,
+        "elapsed_ms": report.elapsed_ms,
+        "num_passed": report.num_passed,
+        "num_failed": report.num_failed,
+        "cases": [
+            {
+                "name": cr.case_name,
+                "passed": cr.passed,
+                "score": cr.score,
+                "tags": cr.tags,
+                "scorers": [
+                    {
+                        "name": sr.scorer_name,
+                        "passed": sr.result.passed,
+                        "score": sr.result.score,
+                        "reason": sr.result.reason,
+                    }
+                    for sr in cr.scorer_results
+                ],
+            }
+            for cr in report.case_results
+        ],
+    }
+    path.write_text(json.dumps(data, indent=2))

agentdog/report.py

@@ -0,0 +1,104 @@
+from __future__ import annotations
+
+import time
+from dataclasses import dataclass, field
+
+import click
+
+from .scorers.base import ScoreResult
+
+
+@dataclass
+class ScorerResult:
+    scorer_name: str
+    result: ScoreResult
+
+
+@dataclass
+class CaseResult:
+    case_name: str
+    scorer_results: list[ScorerResult] = field(default_factory=list)
+    tags: list[str] = field(default_factory=list)
+
+    @property
+    def passed(self) -> bool:
+        return all(sr.result.passed for sr in self.scorer_results)
+
+    @property
+    def score(self) -> float:
+        if not self.scorer_results:
+            return 1.0
+        return sum(sr.result.score for sr in self.scorer_results) / len(self.scorer_results)
+
+    @property
+    def failures(self) -> list[ScorerResult]:
+        return [sr for sr in self.scorer_results if not sr.result.passed]
+
+
+@dataclass
+class Report:
+    case_results: list[CaseResult] = field(default_factory=list)
+    elapsed_ms: float = 0.0
+
+    @property
+    def passed(self) -> bool:
+        return all(cr.passed for cr in self.case_results)
+
+    @property
+    def num_passed(self) -> int:
+        return sum(1 for cr in self.case_results if cr.passed)
+
+    @property
+    def num_failed(self) -> int:
+        return len(self.case_results) - self.num_passed
+
+    @property
+    def overall_score(self) -> float:
+        if not self.case_results:
+            return 1.0
+        return sum(cr.score for cr in self.case_results) / len(self.case_results)
+
+    def print(self, verbose: bool = False) -> None:
+        _print_report(self, verbose=verbose)
+
+
+# ── Rendering ────────────────────────────────────────────────────────────────
+
+_PASS = click.style("PASS", fg="green", bold=True)
+_FAIL = click.style("FAIL", fg="red", bold=True)
+
+
+def _print_report(report: Report, verbose: bool = False) -> None:
+    click.echo()
+    click.echo(click.style("=" * 60, fg="bright_black"))
+    click.echo(click.style("  agentdog results", bold=True))
+    click.echo(click.style("=" * 60, fg="bright_black"))
+
+    for cr in report.case_results:
+        status = _PASS if cr.passed else _FAIL
+        tag_str = f"  [{', '.join(cr.tags)}]" if cr.tags else ""
+        click.echo(f"\n  {status}  {cr.case_name}{tag_str}  (score: {cr.score:.2f})")
+
+        if verbose or not cr.passed:
+            for sr in cr.scorer_results:
+                icon = click.style("ok", fg="green") if sr.result.passed else click.style("!!", fg="red")
+                line = f"       [{icon}] {sr.scorer_name}"
+                if sr.result.reason:
+                    line += f"  - {sr.result.reason}"
+                click.echo(line)
+
+    click.echo()
+    click.echo(click.style("-" * 60, fg="bright_black"))
+
+    summary_color = "green" if report.passed else "red"
+    click.echo(
+        click.style(
+            f"  {report.num_passed}/{len(report.case_results)} cases passed"
+            f"  |  overall score: {report.overall_score:.2f}"
+            f"  |  {report.elapsed_ms:.0f}ms",
+            fg=summary_color,
+            bold=True,
+        )
+    )
+    click.echo(click.style("=" * 60, fg="bright_black"))
+    click.echo()

agentdog/runner.py

@@ -0,0 +1,48 @@
+from __future__ import annotations
+
+import time
+from typing import Sequence
+
+from .case import EvalRun
+from .report import CaseResult, Report, ScorerResult
+
+
+def run(eval_runs: Sequence[EvalRun], verbose: bool = False) -> Report:
+    """
+    Execute all scorers for each EvalRun and return a Report.
+
+    Args:
+        eval_runs: Sequence of (TestCase, AgentTrace) pairs to evaluate.
+        verbose:   If True, print detailed scorer output for passing cases too.
+
+    Returns:
+        Report with per-case and aggregate results.
+    """
+    start = time.perf_counter()
+    case_results: list[CaseResult] = []
+
+    for er in eval_runs:
+        scorer_results: list[ScorerResult] = []
+        for scorer in er.case.scorers:
+            try:
+                result = scorer.score(er.trace)
+            except Exception as exc:
+                from .scorers.base import ScoreResult
+
+                result = ScoreResult(
+                    passed=False,
+                    score=0.0,
+                    reason=f"Scorer raised exception: {exc}",
+                )
+            scorer_results.append(ScorerResult(scorer_name=scorer.name, result=result))
+
+        case_results.append(
+            CaseResult(
+                case_name=er.case.name,
+                scorer_results=scorer_results,
+                tags=er.case.tags,
+            )
+        )
+
+    elapsed_ms = (time.perf_counter() - start) * 1000
+    return Report(case_results=case_results, elapsed_ms=elapsed_ms)

agentdog/scorers/__init__.py

@@ -0,0 +1,41 @@
+from .answer import AnswerNotEmpty, ContainsAnswer, ExactAnswer, ForbiddenContent, RegexAnswer
+from .base import ScoreResult, Scorer
+from .efficiency import MaxRetries, UnderCostLimit, UnderLatencyLimit, UnderTokenLimit
+from .grounding import CitedSource, GroundedInContext, NoContextHallucination
+from .judge import LLMJudge
+from .safety import NoRiskyActionTaken, NoSensitiveDataLeaked, PromptInjectionResisted
+from .tools import AvoidedTools, MaxToolCalls, ToolArgContains, ToolArgEquals, ToolCallOrder, UsedTools
+
+__all__ = [
+    # base
+    "Scorer",
+    "ScoreResult",
+    # answer
+    "ContainsAnswer",
+    "ExactAnswer",
+    "RegexAnswer",
+    "ForbiddenContent",
+    "AnswerNotEmpty",
+    # tools
+    "UsedTools",
+    "AvoidedTools",
+    "ToolCallOrder",
+    "MaxToolCalls",
+    "ToolArgContains",
+    "ToolArgEquals",
+    # grounding
+    "GroundedInContext",
+    "CitedSource",
+    "NoContextHallucination",
+    # safety
+    "NoSensitiveDataLeaked",
+    "NoRiskyActionTaken",
+    "PromptInjectionResisted",
+    # efficiency
+    "UnderTokenLimit",
+    "UnderCostLimit",
+    "UnderLatencyLimit",
+    "MaxRetries",
+    # llm judge
+    "LLMJudge",
+]

agentdog/scorers/answer.py

@@ -0,0 +1,105 @@
+from __future__ import annotations
+
+import re
+from typing import TYPE_CHECKING
+
+from .base import ScoreResult, Scorer
+
+if TYPE_CHECKING:
+    from agentdog.trace import AgentTrace
+
+
+class ContainsAnswer(Scorer):
+    """Pass if the output contains all required substrings."""
+
+    def __init__(self, required: list[str], case_sensitive: bool = False):
+        self.required = required
+        self.case_sensitive = case_sensitive
+
+    def score(self, trace: "AgentTrace") -> ScoreResult:
+        output = trace.output if self.case_sensitive else trace.output.lower()
+        missing = []
+        for phrase in self.required:
+            needle = phrase if self.case_sensitive else phrase.lower()
+            if needle not in output:
+                missing.append(phrase)
+        passed = len(missing) == 0
+        return ScoreResult(
+            passed=passed,
+            score=1.0 - len(missing) / len(self.required),
+            reason="" if passed else f"Missing from output: {missing}",
+        )
+
+
+class ExactAnswer(Scorer):
+    """Pass if the output exactly equals the expected string (after optional strip)."""
+
+    def __init__(self, expected: str, strip: bool = True):
+        self.expected = expected
+        self.strip = strip
+
+    def score(self, trace: "AgentTrace") -> ScoreResult:
+        actual = trace.output.strip() if self.strip else trace.output
+        expected = self.expected.strip() if self.strip else self.expected
+        passed = actual == expected
+        return ScoreResult(
+            passed=passed,
+            score=1.0 if passed else 0.0,
+            reason="" if passed else f"Expected {expected!r}, got {actual!r}",
+        )
+
+
+class RegexAnswer(Scorer):
+    """Pass if the output matches the given regex pattern."""
+
+    def __init__(self, pattern: str, flags: int = re.IGNORECASE):
+        self.pattern = pattern
+        self.flags = flags
+        self._re = re.compile(pattern, flags)
+
+    def score(self, trace: "AgentTrace") -> ScoreResult:
+        match = self._re.search(trace.output)
+        passed = match is not None
+        return ScoreResult(
+            passed=passed,
+            score=1.0 if passed else 0.0,
+            reason="" if passed else f"Pattern {self.pattern!r} not found in output",
+        )
+
+
+class ForbiddenContent(Scorer):
+    """Pass if the output contains none of the forbidden substrings."""
+
+    def __init__(self, forbidden: list[str], case_sensitive: bool = False):
+        self.forbidden = forbidden
+        self.case_sensitive = case_sensitive
+
+    def score(self, trace: "AgentTrace") -> ScoreResult:
+        output = trace.output if self.case_sensitive else trace.output.lower()
+        found = []
+        for phrase in self.forbidden:
+            needle = phrase if self.case_sensitive else phrase.lower()
+            if needle in output:
+                found.append(phrase)
+        passed = len(found) == 0
+        return ScoreResult(
+            passed=passed,
+            score=1.0 - len(found) / len(self.forbidden),
+            reason="" if passed else f"Forbidden content found: {found}",
+        )
+
+
+class AnswerNotEmpty(Scorer):
+    """Pass if the output is non-empty after stripping whitespace."""
+
+    def __init__(self, min_chars: int = 1):
+        self.min_chars = min_chars
+
+    def score(self, trace: "AgentTrace") -> ScoreResult:
+        length = len(trace.output.strip())
+        passed = length >= self.min_chars
+        return ScoreResult(
+            passed=passed,
+            score=1.0 if passed else 0.0,
+            reason="" if passed else f"Output too short: {length} chars (min {self.min_chars})",
+        )

agentdog/scorers/base.py

@@ -0,0 +1,31 @@
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from agentdog.trace import AgentTrace
+
+
+@dataclass
+class ScoreResult:
+    passed: bool
+    score: float  # 0.0 = complete failure, 1.0 = full pass
+    reason: str = ""
+    details: dict = field(default_factory=dict)
+
+    def __bool__(self) -> bool:
+        return self.passed
+
+
+class Scorer(ABC):
+    """Base class for all scorers."""
+
+    @property
+    def name(self) -> str:
+        return self.__class__.__name__
+
+    @abstractmethod
+    def score(self, trace: "AgentTrace") -> ScoreResult:
+        ...

agentdog/scorers/efficiency.py

@@ -0,0 +1,97 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from .base import ScoreResult, Scorer
+
+if TYPE_CHECKING:
+    from agentdog.trace import AgentTrace
+
+
+class UnderTokenLimit(Scorer):
+    """Pass if total token usage is at or below the limit."""
+
+    def __init__(self, max_tokens: int):
+        self.max_tokens = max_tokens
+
+    def score(self, trace: "AgentTrace") -> ScoreResult:
+        if trace.total_tokens is None:
+            return ScoreResult(
+                passed=True,
+                score=1.0,
+                reason="Token count not available in trace (skipped)",
+            )
+        passed = trace.total_tokens <= self.max_tokens
+        ratio = trace.total_tokens / self.max_tokens
+        return ScoreResult(
+            passed=passed,
+            score=max(0.0, 1.0 - (ratio - 1.0)) if not passed else 1.0,
+            reason="" if passed else f"Token usage {trace.total_tokens} exceeds limit {self.max_tokens}",
+            details={"actual": trace.total_tokens, "max": self.max_tokens},
+        )
+
+
+class UnderCostLimit(Scorer):
+    """Pass if total cost is at or below the USD limit."""
+
+    def __init__(self, max_cost_usd: float):
+        self.max_cost_usd = max_cost_usd
+
+    def score(self, trace: "AgentTrace") -> ScoreResult:
+        if trace.total_cost_usd is None:
+            return ScoreResult(
+                passed=True,
+                score=1.0,
+                reason="Cost not available in trace (skipped)",
+            )
+        passed = trace.total_cost_usd <= self.max_cost_usd
+        ratio = trace.total_cost_usd / self.max_cost_usd if self.max_cost_usd else float("inf")
+        return ScoreResult(
+            passed=passed,
+            score=max(0.0, 1.0 - (ratio - 1.0)) if not passed else 1.0,
+            reason="" if passed else (
+                f"Cost ${trace.total_cost_usd:.4f} exceeds limit ${self.max_cost_usd:.4f}"
+            ),
+            details={"actual_usd": trace.total_cost_usd, "max_usd": self.max_cost_usd},
+        )
+
+
+class UnderLatencyLimit(Scorer):
+    """Pass if total latency is at or below the limit in milliseconds."""
+
+    def __init__(self, max_latency_ms: float):
+        self.max_latency_ms = max_latency_ms
+
+    def score(self, trace: "AgentTrace") -> ScoreResult:
+        if trace.total_latency_ms is None:
+            return ScoreResult(
+                passed=True,
+                score=1.0,
+                reason="Latency not available in trace (skipped)",
+            )
+        passed = trace.total_latency_ms <= self.max_latency_ms
+        ratio = trace.total_latency_ms / self.max_latency_ms
+        return ScoreResult(
+            passed=passed,
+            score=max(0.0, 1.0 - (ratio - 1.0)) if not passed else 1.0,
+            reason="" if passed else (
+                f"Latency {trace.total_latency_ms:.0f}ms exceeds limit {self.max_latency_ms:.0f}ms"
+            ),
+            details={"actual_ms": trace.total_latency_ms, "max_ms": self.max_latency_ms},
+        )
+
+
+class MaxRetries(Scorer):
+    """Pass if the number of retries is at or below the limit."""
+
+    def __init__(self, max_retries: int):
+        self.max_retries = max_retries
+
+    def score(self, trace: "AgentTrace") -> ScoreResult:
+        passed = trace.num_retries <= self.max_retries
+        return ScoreResult(
+            passed=passed,
+            score=1.0 if passed else max(0.0, 1.0 - (trace.num_retries - self.max_retries) / (self.max_retries + 1)),
+            reason="" if passed else f"Retries {trace.num_retries} exceeds max {self.max_retries}",
+            details={"actual": trace.num_retries, "max": self.max_retries},
+        )