agentdog 0.1.0__tar.gz

agentdog-0.1.0/.gitignore

@@ -0,0 +1,29 @@
+# Virtual environments
+.venv/
+venv/
+env/
+
+# Build artifacts
+dist/
+build/
+*.egg-info/
+
+# Python cache
+__pycache__/
+*.py[cod]
+*.pyo
+
+# Test cache
+.pytest_cache/
+.coverage
+htmlcov/
+
+# Type checking
+.mypy_cache/
+.ruff_cache/
+
+# Reports
+*.json
+
+# Typical dev data
+dev-data/

agentdog-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Sai Teja Erukude
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

agentdog-0.1.0/PKG-INFO

@@ -0,0 +1,178 @@
+Metadata-Version: 2.4
+Name: agentdog
+Version: 0.1.0
+Summary: Lightweight evaluation toolkit for AI agents — test tool use, grounding, safety, and efficiency before production
+Project-URL: Homepage, https://github.com/SaiTeja-Erukude/agentdog
+Project-URL: Repository, https://github.com/SaiTeja-Erukude/agentdog
+Project-URL: Bug Tracker, https://github.com/SaiTeja-Erukude/agentdog/issues
+Project-URL: Changelog, https://github.com/SaiTeja-Erukude/agentdog/releases
+Author: Sai Teja Erukude
+License: MIT License
+        
+        Copyright (c) 2026 Sai Teja Erukude
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy
+        of this software and associated documentation files (the "Software"), to deal
+        in the Software without restriction, including without limitation the rights
+        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+        copies of the Software, and to permit persons to whom the Software is
+        furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all
+        copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+        SOFTWARE.
+License-File: LICENSE
+Keywords: agent,ai,evals,evaluation,llm,prompt-injection,rag,safety,testing,tool-use
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: Software Development :: Testing
+Requires-Python: >=3.10
+Requires-Dist: click>=8.0
+Provides-Extra: dev
+Requires-Dist: pytest-cov>=5.0; extra == 'dev'
+Requires-Dist: pytest>=8.0; extra == 'dev'
+Provides-Extra: llm-judge
+Requires-Dist: openai>=1.0; extra == 'llm-judge'
+Description-Content-Type: text/markdown
+
+# agentdog
+
+Lightweight evaluation toolkit for AI agents. `pytest` for agent behavior  —  test tool use, grounding, safety, and efficiency before production
+
+```
+pip install agentdog
+pip install "agentdog[llm-judge]"  # for LLMJudge scorer
+```
+
+---
+
+## Quickstart
+
+```python
+from agentdog import AgentTrace, ToolCall, TestCase, EvalRun, run
+from agentdog import ContainsAnswer, UsedTools, AvoidedTools, UnderTokenLimit
+
+trace = AgentTrace(
+    input="Summarize the Q3 report.",
+    output="Q3 revenue was $4.2M, up 12% YoY.",
+    tool_calls=[ToolCall(name="file_search", arguments={"query": "Q3 report"})],
+    retrieved_context=["Q3 revenue was $4.2M, growth 12% year over year."],
+    total_tokens=620,
+)
+
+case = TestCase(
+    name="q3-summary",
+    tags=["rag"],
+    scorers=[
+        ContainsAnswer(["4.2M", "12%"]),
+        UsedTools(["file_search"]),
+        AvoidedTools(["send_email"]),
+        UnderTokenLimit(max_tokens=1000),
+    ],
+)
+
+report = run([EvalRun(case=case, trace=trace)])
+report.print(verbose=True)
+```
+
+---
+
+## CLI
+
+Define an `evals()` function in any Python file that returns `list[EvalRun]`, then:
+
+```bash
+agentdog run my_evals.py             # run all cases
+agentdog run my_evals.py -v          # verbose: show scorer details for passing cases
+agentdog run my_evals.py --tag rag   # filter by tag
+agentdog run my_evals.py --json-out report.json  # machine-readable output
+agentdog inspect trace.json          # pretty-print a trace file
+```
+
+Exit code is `0` on full pass, `1` on any failure — CI-friendly by default.
+
+---
+
+## Scorers
+
+| Category | Scorers |
+|---|---|
+| **Answer** | `ContainsAnswer` `ExactAnswer` `RegexAnswer` `ForbiddenContent` `AnswerNotEmpty` |
+| **Tools** | `UsedTools` `AvoidedTools` `ToolCallOrder` `MaxToolCalls` `ToolArgContains` `ToolArgEquals` |
+| **Grounding** | `GroundedInContext` `CitedSource` `NoContextHallucination` |
+| **Safety** | `NoSensitiveDataLeaked` `NoRiskyActionTaken` `PromptInjectionResisted` |
+| **Efficiency** | `UnderTokenLimit` `UnderCostLimit` `UnderLatencyLimit` `MaxRetries` |
+| **LLM Judge** | `LLMJudge` — use only when deterministic checks aren't enough |
+
+---
+
+## Trace schema
+
+```python
+AgentTrace(
+    input: str,
+    output: str,
+    tool_calls: list[ToolCall],        # name, arguments, output, error, latency_ms
+    retrieved_context: list[str],
+    total_tokens: int | None,
+    total_cost_usd: float | None,
+    total_latency_ms: float | None,
+    num_retries: int,
+    metadata: dict,
+)
+```
+
+Load/save:
+
+```python
+trace = AgentTrace.from_json("trace.json")
+trace.to_json("trace.json")
+```
+
+---
+
+## Custom scorer
+
+```python
+from agentdog.scorers.base import Scorer, ScoreResult
+
+class AnswerStartsWith(Scorer):
+    def __init__(self, prefix: str):
+        self.prefix = prefix
+
+    def score(self, trace) -> ScoreResult:
+        passed = trace.output.startswith(self.prefix)
+        return ScoreResult(
+            passed=passed,
+            score=1.0 if passed else 0.0,
+            reason=f"Expected output to start with {self.prefix!r}",
+        )
+```
+
+---
+
+## Example
+
+See [`examples/sample_evals.py`](examples/sample_evals.py) for a complete working example covering RAG, safety, and prompt injection.
+
+---
+
+## Author
+
+**Sai Teja Erukude**  
+[GitHub](https://github.com/SaiTeja-Erukude) · [agentdog](https://github.com/SaiTeja-Erukude/agentdog)
+
+Licensed under the [MIT License](LICENSE).

agentdog-0.1.0/README.md

@@ -0,0 +1,128 @@
+# agentdog
+
+Lightweight evaluation toolkit for AI agents. `pytest` for agent behavior  —  test tool use, grounding, safety, and efficiency before production
+
+```
+pip install agentdog
+pip install "agentdog[llm-judge]"  # for LLMJudge scorer
+```
+
+---
+
+## Quickstart
+
+```python
+from agentdog import AgentTrace, ToolCall, TestCase, EvalRun, run
+from agentdog import ContainsAnswer, UsedTools, AvoidedTools, UnderTokenLimit
+
+trace = AgentTrace(
+    input="Summarize the Q3 report.",
+    output="Q3 revenue was $4.2M, up 12% YoY.",
+    tool_calls=[ToolCall(name="file_search", arguments={"query": "Q3 report"})],
+    retrieved_context=["Q3 revenue was $4.2M, growth 12% year over year."],
+    total_tokens=620,
+)
+
+case = TestCase(
+    name="q3-summary",
+    tags=["rag"],
+    scorers=[
+        ContainsAnswer(["4.2M", "12%"]),
+        UsedTools(["file_search"]),
+        AvoidedTools(["send_email"]),
+        UnderTokenLimit(max_tokens=1000),
+    ],
+)
+
+report = run([EvalRun(case=case, trace=trace)])
+report.print(verbose=True)
+```
+
+---
+
+## CLI
+
+Define an `evals()` function in any Python file that returns `list[EvalRun]`, then:
+
+```bash
+agentdog run my_evals.py             # run all cases
+agentdog run my_evals.py -v          # verbose: show scorer details for passing cases
+agentdog run my_evals.py --tag rag   # filter by tag
+agentdog run my_evals.py --json-out report.json  # machine-readable output
+agentdog inspect trace.json          # pretty-print a trace file
+```
+
+Exit code is `0` on full pass, `1` on any failure — CI-friendly by default.
+
+---
+
+## Scorers
+
+| Category | Scorers |
+|---|---|
+| **Answer** | `ContainsAnswer` `ExactAnswer` `RegexAnswer` `ForbiddenContent` `AnswerNotEmpty` |
+| **Tools** | `UsedTools` `AvoidedTools` `ToolCallOrder` `MaxToolCalls` `ToolArgContains` `ToolArgEquals` |
+| **Grounding** | `GroundedInContext` `CitedSource` `NoContextHallucination` |
+| **Safety** | `NoSensitiveDataLeaked` `NoRiskyActionTaken` `PromptInjectionResisted` |
+| **Efficiency** | `UnderTokenLimit` `UnderCostLimit` `UnderLatencyLimit` `MaxRetries` |
+| **LLM Judge** | `LLMJudge` — use only when deterministic checks aren't enough |
+
+---
+
+## Trace schema
+
+```python
+AgentTrace(
+    input: str,
+    output: str,
+    tool_calls: list[ToolCall],        # name, arguments, output, error, latency_ms
+    retrieved_context: list[str],
+    total_tokens: int | None,
+    total_cost_usd: float | None,
+    total_latency_ms: float | None,
+    num_retries: int,
+    metadata: dict,
+)
+```
+
+Load/save:
+
+```python
+trace = AgentTrace.from_json("trace.json")
+trace.to_json("trace.json")
+```
+
+---
+
+## Custom scorer
+
+```python
+from agentdog.scorers.base import Scorer, ScoreResult
+
+class AnswerStartsWith(Scorer):
+    def __init__(self, prefix: str):
+        self.prefix = prefix
+
+    def score(self, trace) -> ScoreResult:
+        passed = trace.output.startswith(self.prefix)
+        return ScoreResult(
+            passed=passed,
+            score=1.0 if passed else 0.0,
+            reason=f"Expected output to start with {self.prefix!r}",
+        )
+```
+
+---
+
+## Example
+
+See [`examples/sample_evals.py`](examples/sample_evals.py) for a complete working example covering RAG, safety, and prompt injection.
+
+---
+
+## Author
+
+**Sai Teja Erukude**  
+[GitHub](https://github.com/SaiTeja-Erukude) · [agentdog](https://github.com/SaiTeja-Erukude/agentdog)
+
+Licensed under the [MIT License](LICENSE).

agentdog-0.1.0/agentdog/__init__.py

@@ -0,0 +1,104 @@
+"""
+agentdog: lightweight evaluation toolkit for AI agents.
+"""
+
+from .case import EvalRun, TestCase
+from .report import CaseResult, Report, ScorerResult
+from .runner import run
+from .scorers import (
+    AnswerNotEmpty,
+    AvoidedTools,
+    CitedSource,
+    ContainsAnswer,
+    ExactAnswer,
+    ForbiddenContent,
+    GroundedInContext,
+    LLMJudge,
+    MaxRetries,
+    MaxToolCalls,
+    NoContextHallucination,
+    NoRiskyActionTaken,
+    NoSensitiveDataLeaked,
+    PromptInjectionResisted,
+    RegexAnswer,
+    ScoreResult,
+    Scorer,
+    ToolArgContains,
+    ToolArgEquals,
+    ToolCallOrder,
+    UnderCostLimit,
+    UnderLatencyLimit,
+    UnderTokenLimit,
+    UsedTools,
+)
+from .trace import AgentTrace, ToolCall
+
+__version__ = "0.1.0"
+
+__all__ = [
+    # trace
+    "AgentTrace",
+    "ToolCall",
+    # case
+    "TestCase",
+    "EvalRun",
+    # runner
+    "run",
+    # report
+    "Report",
+    "CaseResult",
+    "ScorerResult",
+    # scorers — base
+    "Scorer",
+    "ScoreResult",
+    # scorers — answer
+    "ContainsAnswer",
+    "ExactAnswer",
+    "RegexAnswer",
+    "ForbiddenContent",
+    "AnswerNotEmpty",
+    # scorers — tools
+    "UsedTools",
+    "AvoidedTools",
+    "ToolCallOrder",
+    "MaxToolCalls",
+    "ToolArgContains",
+    "ToolArgEquals",
+    # scorers — grounding
+    "GroundedInContext",
+    "CitedSource",
+    "NoContextHallucination",
+    # scorers — safety
+    "NoSensitiveDataLeaked",
+    "NoRiskyActionTaken",
+    "PromptInjectionResisted",
+    # scorers — efficiency
+    "UnderTokenLimit",
+    "UnderCostLimit",
+    "UnderLatencyLimit",
+    "MaxRetries",
+    # scorers — llm judge
+    "LLMJudge",
+]
+
+# ---------------------------------------------------------------------------
+# Aspirational features — tracked here until implemented
+# ---------------------------------------------------------------------------
+#
+# v2 candidates (medium effort, clear value):
+#   - human_in_the_loop_review: structured queue for human-reviewed cases
+#   - evaluation_packs: pre-built scorer bundles (rag_pack, security_pack, etc.)
+#   - html_reports: rich HTML report with drill-down per case
+#   - github_action: official GHA for agentdog run with PR comment integration
+#   - async_runner: parallel trace evaluation for large eval sets
+#
+# v3 candidates (higher effort or less certain):
+#   - framework_adapters: native converters for LangChain, LlamaIndex, OpenAI SDK traces
+#   - model_comparison: run same cases against multiple models, diff results
+#   - prompt_version_comparison: A/B eval across prompt variants
+#   - trace_replay: re-run a captured trace through a new model/prompt
+#   - synthetic_eval_generation: auto-generate eval cases from a prompt + schema
+#   - dataset_quality_checks: flag low-quality or duplicate eval cases
+#   - safety_governance_templates: pre-built packs for HIPAA, PCI, SOC2 patterns
+#   - rag_advanced: retrieval-specific metrics (NDCG, MRR, recall@k)
+#   - streaming_trace_support: capture and eval streaming agent runs

agentdog-0.1.0/agentdog/case.py

@@ -0,0 +1,26 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from agentdog.scorers.base import Scorer
+    from agentdog.trace import AgentTrace
+
+
+@dataclass
+class TestCase:
+    """Defines what to evaluate and how to evaluate it for a single agent input."""
+
+    name: str
+    scorers: list["Scorer"]
+    description: str = ""
+    tags: list[str] = field(default_factory=list)
+
+
+@dataclass
+class EvalRun:
+    """A test case paired with the agent trace to evaluate against."""
+
+    case: TestCase
+    trace: "AgentTrace"

agentdog-0.1.0/agentdog/cli.py

@@ -0,0 +1,130 @@
+from __future__ import annotations
+
+import importlib.util
+import json
+import sys
+from pathlib import Path
+
+import click
+
+from .case import EvalRun
+from .runner import run
+from .trace import AgentTrace
+
+
+def _load_module(path: Path):
+    spec = importlib.util.spec_from_file_location("_agentdog_module", path)
+    if spec is None or spec.loader is None:
+        raise click.ClickException(f"Cannot load module from {path}")
+    mod = importlib.util.module_from_spec(spec)
+    sys.path.insert(0, str(path.parent))
+    spec.loader.exec_module(mod)  # type: ignore[arg-type]
+    return mod
+
+
+@click.group()
+def main():
+    """agentdog — lightweight evaluation for AI agents."""
+
+
+@main.command()
+@click.argument("module", type=click.Path(exists=True, dir_okay=False, path_type=Path))
+@click.option("--verbose", "-v", is_flag=True, help="Show scorer details for passing cases too.")
+@click.option("--fail-fast", is_flag=True, help="Stop after the first failing case.")
+@click.option("--tag", "tags", multiple=True, help="Only run cases matching these tags.")
+@click.option("--json-out", type=click.Path(dir_okay=False, path_type=Path), help="Write JSON report to file.")
+def run_cmd(module: Path, verbose: bool, fail_fast: bool, tags: tuple[str, ...], json_out: Path | None):
+    """
+    Run evaluations defined in MODULE.
+
+    MODULE must be a Python file that exposes an `evals()` function returning
+    a list of EvalRun objects.
+
+    \b
+    Example module (my_evals.py):
+        from agentdog import EvalRun, TestCase, AgentTrace
+        from agentdog.scorers import ContainsAnswer, UsedTools
+
+        def evals():
+            trace = AgentTrace(input="What is 2+2?", output="The answer is 4.")
+            case  = TestCase("basic-math", scorers=[ContainsAnswer(["4"])])
+            return [EvalRun(case=case, trace=trace)]
+    """
+    mod = _load_module(module)
+    if not hasattr(mod, "evals"):
+        raise click.ClickException(f"{module} must define an evals() function")
+
+    eval_runs: list[EvalRun] = mod.evals()
+
+    if tags:
+        eval_runs = [er for er in eval_runs if set(er.case.tags) & set(tags)]
+        if not eval_runs:
+            click.echo(f"No cases matched tags: {list(tags)}")
+            return
+
+    if fail_fast:
+        filtered: list[EvalRun] = []
+        for er in eval_runs:
+            filtered.append(er)
+            sub = run([er])
+            if not sub.passed:
+                report = run(filtered)
+                report.print(verbose=verbose)
+                sys.exit(1)
+        eval_runs = filtered
+
+    report = run(eval_runs)
+    report.print(verbose=verbose)
+
+    if json_out:
+        _write_json_report(report, json_out)
+        click.echo(f"JSON report written to {json_out}")
+
+    sys.exit(0 if report.passed else 1)
+
+
+@main.command()
+@click.argument("trace_file", type=click.Path(exists=True, dir_okay=False, path_type=Path))
+def inspect(trace_file: Path):
+    """Pretty-print a trace JSON file."""
+    trace = AgentTrace.from_json(str(trace_file))
+    click.echo(f"\nInput:   {trace.input}")
+    click.echo(f"Output:  {trace.output}")
+    click.echo(f"Tools:   {trace.tool_names or '(none)'}")
+    click.echo(f"Context: {len(trace.retrieved_context)} chunk(s)")
+    if trace.total_tokens is not None:
+        click.echo(f"Tokens:  {trace.total_tokens}")
+    if trace.total_cost_usd is not None:
+        click.echo(f"Cost:    ${trace.total_cost_usd:.4f}")
+    if trace.total_latency_ms is not None:
+        click.echo(f"Latency: {trace.total_latency_ms:.0f}ms")
+    click.echo()
+
+
+def _write_json_report(report, path: Path) -> None:
+    data = {
+        "passed": report.passed,
+        "overall_score": report.overall_score,
+        "elapsed_ms": report.elapsed_ms,
+        "num_passed": report.num_passed,
+        "num_failed": report.num_failed,
+        "cases": [
+            {
+                "name": cr.case_name,
+                "passed": cr.passed,
+                "score": cr.score,
+                "tags": cr.tags,
+                "scorers": [
+                    {
+                        "name": sr.scorer_name,
+                        "passed": sr.result.passed,
+                        "score": sr.result.score,
+                        "reason": sr.result.reason,
+                    }
+                    for sr in cr.scorer_results
+                ],
+            }
+            for cr in report.case_results
+        ],
+    }
+    path.write_text(json.dumps(data, indent=2))