autobots-devtools-shared-lib 0.5.2__tar.gz → 0.6.0__tar.gz

{autobots_devtools_shared_lib-0.5.2 → autobots_devtools_shared_lib-0.6.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: autobots-devtools-shared-lib
-Version: 0.5.2
+Version: 0.6.0
 Summary: Shared library functions to be used for all autobots projects
 License: MIT
 Author: Pralhad
@@ -18,6 +18,7 @@ Requires-Dist: langchain (>=1.0.0)
 Requires-Dist: langchain-anthropic (>=1.4.0)
 Requires-Dist: langchain-google-genai (>=4.2.0)
 Requires-Dist: langfuse (>=3.12.1)
+Requires-Dist: openevals (>=0.1.0)
 Requires-Dist: opentelemetry-api (>=1.30.0,<2.0.0)
 Requires-Dist: opentelemetry-exporter-otlp-proto-http (>=1.30.0,<2.0.0)
 Requires-Dist: opentelemetry-instrumentation-fastapi (>=0.49b0)
@@ -26,6 +27,7 @@ Requires-Dist: pydantic-settings (>=2.10.1)
 Requires-Dist: python-dotenv (>=1.1.1)
 Requires-Dist: pyyaml (>=6.0.3)
 Requires-Dist: referencing (>=0.37.0)
+Requires-Dist: tiktoken (>=0.7.0)
 Requires-Dist: uvicorn[standard] (>=0.32.0)
 Description-Content-Type: text/markdown
 

{autobots_devtools_shared_lib-0.5.2 → autobots_devtools_shared_lib-0.6.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "autobots-devtools-shared-lib"
-version = "0.5.2"
+version = "0.6.0"
 description = "Shared library functions to be used for all autobots projects"
 readme = "README.md"
 authors = [
@@ -25,6 +25,8 @@ dependencies = [
     "opentelemetry-sdk>=1.30.0,<2.0.0",
     "opentelemetry-exporter-otlp-proto-http>=1.30.0,<2.0.0",
     "opentelemetry-instrumentation-fastapi>=0.49b0",
+    "openevals>=0.1.0",
+    "tiktoken>=0.7.0",
 ]
 dev = [
     "pre-commit>=4.5.1",
@@ -33,6 +35,7 @@ dev = [
     "pytest-asyncio>=1.3.0",
     "pytest-cov>=7.0.0",
     "ruff>=0.14.14",
+    "pytest-xdist>=3.0.0",
 ]
 
 [tool.poetry]
@@ -44,6 +47,9 @@ packages = [
 requires = ["poetry-core"]
 build-backend = "poetry.core.masonry.api"
 
+[project.entry-points."pytest11"]
+dynagent_eval = "autobots_devtools_shared_lib.eval.pytest_plugin.plugin"
+
 [tool.ruff]
 line-length = 100
 target-version = "py312"
@@ -106,7 +112,7 @@ markers = [
 pythonVersion = "3.12"
 typeCheckingMode = "basic"
 include = ["src", "tests"]
-exclude = ["**/__pycache__", ".venv"]
+exclude = ["**/__pycache__", ".venv", "**/eval/core/cost_tracker.py", "**/eval/assertions/llm_judge.py"]
 venvPath = ".."
 venv = ".venv"
 extraPaths = ["src"]

{autobots_devtools_shared_lib-0.5.2 → autobots_devtools_shared_lib-0.6.0}/src/autobots_devtools_shared_lib/dynagent/utils/schema_directive_resolver.py

@@ -91,9 +91,10 @@ def _retrieve_from_path(base_dir: Path):
 
     def _retrieve(uri: str) -> Resource:
         from urllib.parse import unquote, urlparse
+        from urllib.request import url2pathname
 
         parsed = urlparse(uri)
-        path = Path(unquote(parsed.path))
+        path = Path(url2pathname(unquote(parsed.path)))
         if not path.is_absolute():
             path = base_dir / path
         contents = json.loads(path.read_text())

autobots_devtools_shared_lib-0.6.0/src/autobots_devtools_shared_lib/eval/__init__.py

@@ -0,0 +1,33 @@
+# ABOUTME: Public API for the dynagent eval framework.
+# ABOUTME: Import from here for a stable surface — loader, models, and result types.
+
+from autobots_devtools_shared_lib.eval.assertions.registry import register_assertion
+from autobots_devtools_shared_lib.eval.core.loader import EvalConfigError, load_eval_cases
+from autobots_devtools_shared_lib.eval.models.eval_case import (
+    EvalCase,
+    SetupConfig,
+    WorkspaceFile,
+)
+from autobots_devtools_shared_lib.eval.models.result import (
+    AgentOutput,
+    AssertionResult,
+    CostDelta,
+    EvalCostSnapshot,
+    EvalResult,
+    TurnResult,
+)
+
+__all__ = [
+    "AgentOutput",
+    "AssertionResult",
+    "CostDelta",
+    "EvalCase",
+    "EvalConfigError",
+    "EvalCostSnapshot",
+    "EvalResult",
+    "SetupConfig",
+    "TurnResult",
+    "WorkspaceFile",
+    "load_eval_cases",
+    "register_assertion",
+]

autobots_devtools_shared_lib-0.6.0/src/autobots_devtools_shared_lib/eval/assertions/__init__.py

@@ -0,0 +1,3 @@
+# ABOUTME: Package init for the eval assertions module.
+# ABOUTME: Contains assertion registry and evaluator functions.
+from __future__ import annotations

autobots_devtools_shared_lib-0.6.0/src/autobots_devtools_shared_lib/eval/assertions/deterministic.py

@@ -0,0 +1,174 @@
+# ABOUTME: Deterministic assertion functions wrapping OpenEvals and built-in checks.
+# ABOUTME: Each function takes AgentOutput + config and returns AssertionResult.
+
+from __future__ import annotations
+
+import json
+import re
+from pathlib import Path
+from typing import Any
+
+import jsonschema as js
+
+from autobots_devtools_shared_lib.eval.models.result import AgentOutput, AssertionResult
+
+
+def _last_ai_content(agent_output: AgentOutput) -> str:
+    """Extract text content from the last AI message."""
+    for msg in reversed(agent_output.messages):
+        if hasattr(msg, "type") and msg.type == "ai" and msg.content:
+            return str(msg.content)
+    return ""
+
+
+def _all_tool_names(agent_output: AgentOutput) -> list[str]:
+    """Extract all tool names called across all messages."""
+    names: list[str] = []
+    for msg in agent_output.messages:
+        tool_calls = getattr(msg, "tool_calls", None)
+        if tool_calls:
+            for tc in tool_calls:
+                if isinstance(tc, dict):
+                    names.append(tc.get("name", ""))
+                elif hasattr(tc, "name"):
+                    names.append(tc.name)
+    return names
+
+
+def contains(agent_output: AgentOutput, config: Any) -> AssertionResult:
+    """Check if agent response contains a substring (case-insensitive)."""
+    text = _last_ai_content(agent_output).lower()
+    target = str(config).lower()
+    found = target in text
+    return AssertionResult(
+        passed=found,
+        name=f"contains:{config}",
+        detail=f"{'Found' if found else 'Not found'} in response",
+    )
+
+
+def regex(agent_output: AgentOutput, config: Any) -> AssertionResult:
+    """Check if agent response matches a regex pattern."""
+    text = _last_ai_content(agent_output)
+    pattern = str(config)
+    match = bool(re.search(pattern, text))
+    return AssertionResult(
+        passed=match,
+        name=f"regex:{pattern}",
+        detail=f"{'Matched' if match else 'No match'} for pattern",
+    )
+
+
+def exact_match(agent_output: AgentOutput, config: Any) -> AssertionResult:
+    """Check if agent response exactly matches expected string."""
+    text = _last_ai_content(agent_output)
+    expected = str(config)
+    passed = text.strip() == expected.strip()
+    return AssertionResult(
+        passed=passed,
+        name="exact_match",
+        detail=f"Expected: {expected[:100]}",
+    )
+
+
+def json_match(agent_output: AgentOutput, config: Any) -> AssertionResult:
+    """Check if agent response JSON matches expected JSON."""
+    text = _last_ai_content(agent_output)
+    try:
+        actual = json.loads(text)
+        expected = config if isinstance(config, dict) else json.loads(str(config))
+        passed = actual == expected
+        return AssertionResult(
+            passed=passed,
+            name="json_match",
+            detail="JSON matches" if passed else "JSON does not match",
+        )
+    except (json.JSONDecodeError, TypeError) as e:
+        return AssertionResult(passed=False, name="json_match", detail=f"Parse error: {e}")
+
+
+def schema_match(agent_output: AgentOutput, config: Any) -> AssertionResult:
+    """Validate agent response JSON against a JSON schema (dict or file path)."""
+    text = _last_ai_content(agent_output)
+    try:
+        # If config is already a dict, use it directly; otherwise treat as file path
+        if isinstance(config, dict):
+            schema = config
+        else:
+            schema_path = Path(str(config))
+            schema = json.loads(schema_path.read_text())
+        data = json.loads(text)
+        js.validate(instance=data, schema=schema)
+        return AssertionResult(passed=True, name="response_matches_schema", detail="Valid")
+    except js.ValidationError as e:
+        return AssertionResult(
+            passed=False,
+            name="response_matches_schema",
+            detail=f"Schema validation failed: {e.message}",
+        )
+    except (json.JSONDecodeError, FileNotFoundError, OSError) as e:
+        return AssertionResult(
+            passed=False,
+            name="response_matches_schema",
+            detail=f"Error: {e}",
+        )
+
+
+def tool_called(agent_output: AgentOutput, config: Any) -> AssertionResult:
+    """Check if a specific tool was called during the conversation."""
+    target = str(config)
+    called = _all_tool_names(agent_output)
+    found = target in called
+    return AssertionResult(
+        passed=found,
+        name=f"tool_called:{target}",
+        detail=f"Tools called: {called}" if not found else "Found",
+    )
+
+
+def tool_sequence(agent_output: AgentOutput, config: Any) -> AssertionResult:
+    """Check if tools were called in a specific order."""
+    if not isinstance(config, list):
+        return AssertionResult(passed=False, name="tool_sequence", detail="Config must be a list")
+
+    expected_names = [step["tool"] for step in config if isinstance(step, dict)]
+    called = _all_tool_names(agent_output)
+
+    # Check subsequence match (in order, not necessarily contiguous)
+    idx = 0
+    for name in called:
+        if idx < len(expected_names) and name == expected_names[idx]:
+            idx += 1
+    passed = idx == len(expected_names)
+
+    return AssertionResult(
+        passed=passed,
+        name="tool_sequence",
+        detail=f"Expected: {expected_names}, Called: {called}",
+    )
+
+
+def no_extra_tools(agent_output: AgentOutput, config: Any) -> AssertionResult:
+    """Check that no tools beyond the allowed set were called."""
+    allowed = set(config) if isinstance(config, list) else {str(config)}
+    called = set(_all_tool_names(agent_output))
+    extra = called - allowed
+    passed = len(extra) == 0
+    return AssertionResult(
+        passed=passed,
+        name="no_extra_tools",
+        detail=f"Extra tools: {extra}" if extra else "No extra tools",
+    )
+
+
+def tools_unordered(agent_output: AgentOutput, config: Any) -> AssertionResult:
+    """Check that all expected tools were called (order doesn't matter)."""
+    expected = set(config) if isinstance(config, list) else {str(config)}
+    called = set(_all_tool_names(agent_output))
+    missing = expected - called
+    passed = len(missing) == 0
+    return AssertionResult(
+        passed=passed,
+        name="tools_unordered",
+        detail=f"Missing: {missing}" if missing else "All tools called",
+    )

autobots_devtools_shared_lib-0.6.0/src/autobots_devtools_shared_lib/eval/assertions/golden.py

@@ -0,0 +1,161 @@
+# ABOUTME: Golden match assertion for comparing agent structured_response against reference files.
+# ABOUTME: Supports exact diff mode and structural-only comparison with field ignoring.
+"""Golden match assertion: compare agent output against reference files."""
+
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+from autobots_devtools_shared_lib.eval.models.result import AgentOutput, AssertionResult
+
+
+@dataclass
+class JsonDiff:
+    """Structured diff between reference and actual JSON."""
+
+    missing: list[str] = field(default_factory=list)
+    unexpected: list[str] = field(default_factory=list)
+    changed: list[str] = field(default_factory=list)
+
+    @property
+    def has_differences(self) -> bool:
+        return bool(self.missing or self.unexpected or self.changed)
+
+    def to_detail(self) -> str:
+        lines = [f"Missing from actual: {m}" for m in self.missing]
+        lines.extend(f"Unexpected in actual: {u}" for u in self.unexpected)
+        lines.extend(f"Changed: {c}" for c in self.changed)
+        return "\n".join(lines)
+
+
+def _diff_json(reference: Any, actual: Any, path: str = "") -> JsonDiff:
+    """Recursive deep diff between two JSON-like structures."""
+    diff = JsonDiff()
+
+    if isinstance(reference, dict) and isinstance(actual, dict):
+        for key in reference:
+            child_path = f"{path}.{key}" if path else key
+            if key not in actual:
+                diff.missing.append(f"{child_path}: {json.dumps(reference[key])}")
+            else:
+                child = _diff_json(reference[key], actual[key], child_path)
+                diff.missing.extend(child.missing)
+                diff.unexpected.extend(child.unexpected)
+                diff.changed.extend(child.changed)
+        for key in actual:
+            child_path = f"{path}.{key}" if path else key
+            if key not in reference:
+                diff.unexpected.append(f"{child_path}: {json.dumps(actual[key])}")
+
+    elif isinstance(reference, list) and isinstance(actual, list):
+        for i in range(max(len(reference), len(actual))):
+            child_path = f"{path}[{i}]"
+            if i >= len(actual):
+                diff.missing.append(f"{child_path}: {json.dumps(reference[i])}")
+            elif i >= len(reference):
+                diff.unexpected.append(f"{child_path}: {json.dumps(actual[i])}")
+            else:
+                child = _diff_json(reference[i], actual[i], child_path)
+                diff.missing.extend(child.missing)
+                diff.unexpected.extend(child.unexpected)
+                diff.changed.extend(child.changed)
+
+    elif reference != actual:
+        diff.changed.append(f"{path}: {json.dumps(reference)} → {json.dumps(actual)}")
+
+    return diff
+
+
+def _deep_structural_compare(
+    reference: Any,
+    actual: Any,
+    path: str = "",
+    ignore_fields: list[str] | None = None,
+) -> list[str]:
+    """Compare structure only: same keys, same types, same array lengths. Ignores string values."""
+    ignore = set(ignore_fields or [])
+    issues: list[str] = []
+
+    if isinstance(reference, dict) and isinstance(actual, dict):
+        for key in reference:
+            if key in ignore:
+                continue
+            child_path = f"{path}.{key}" if path else key
+            if key not in actual:
+                issues.append(f"Missing key: {child_path}")
+            else:
+                issues.extend(
+                    _deep_structural_compare(reference[key], actual[key], child_path, ignore_fields)
+                )
+        for key in actual:
+            if key in ignore:
+                continue
+            child_path = f"{path}.{key}" if path else key
+            if key not in reference:
+                issues.append(f"Unexpected key: {child_path}")
+
+    elif isinstance(reference, list) and isinstance(actual, list):
+        if len(reference) != len(actual):
+            issues.append(
+                f"Array length mismatch at {path or 'root'}: "
+                f"expected {len(reference)}, got {len(actual)}"
+            )
+        for i in range(min(len(reference), len(actual))):
+            child_path = f"{path}[{i}]"
+            issues.extend(
+                _deep_structural_compare(reference[i], actual[i], child_path, ignore_fields)
+            )
+
+    elif type(reference) is not type(actual):
+        issues.append(
+            f"Type mismatch at {path or 'root'}: "
+            f"expected {type(reference).__name__}, got {type(actual).__name__}"
+        )
+
+    return issues
+
+
+def golden_match(output: AgentOutput, config: Any) -> AssertionResult:
+    """Compare agent structured_response against a golden reference file."""
+    ref_path = Path(config["reference"])
+    mode: str = config.get("mode", "exact")
+    ignore_fields: list[str] = config.get("ignore_fields", [])
+
+    if not ref_path.exists():
+        return AssertionResult(
+            passed=False,
+            name="golden_match",
+            detail=f"Reference file not found: {ref_path}. Run with --update-golden to create.",
+        )
+
+    reference = json.loads(ref_path.read_text())
+    actual = output.structured_response
+
+    if mode == "exact":
+        diff = _diff_json(reference, actual)
+        if diff.has_differences:
+            return AssertionResult(
+                passed=False,
+                name="golden_match",
+                detail=f"Reference: {ref_path}\n\n{diff.to_detail()}",
+            )
+        return AssertionResult(passed=True, name="golden_match", detail="Exact match")
+
+    if mode == "structural":
+        issues = _deep_structural_compare(reference, actual, ignore_fields=ignore_fields)
+        if issues:
+            return AssertionResult(
+                passed=False,
+                name="golden_match",
+                detail="Structural mismatch:\n" + "\n".join(f"  {i}" for i in issues),
+            )
+        return AssertionResult(passed=True, name="golden_match", detail="Structural match")
+
+    return AssertionResult(
+        passed=False,
+        name="golden_match",
+        detail=f"Unknown mode: {mode}. Use 'exact' or 'structural'.",
+    )

autobots_devtools_shared_lib-0.6.0/src/autobots_devtools_shared_lib/eval/assertions/llm_judge.py

@@ -0,0 +1,196 @@
+# ABOUTME: LLM-as-judge assertion functions wrapping OpenEvals.
+# ABOUTME: Evaluates free-text agent responses against criteria using an LLM judge.
+
+from __future__ import annotations
+
+import logging
+from typing import Any
+
+from openevals.llm import create_llm_as_judge  # pyright: ignore[reportMissingImports]
+
+from autobots_devtools_shared_lib.eval.models.result import AgentOutput, AssertionResult
+
+logger = logging.getLogger(__name__)
+
+# Default judge model — cheap and fast for evaluation
+_DEFAULT_JUDGE_MODEL = "google_genai/gemini-2.0-flash"
+
+_LLM_JUDGE_PROMPT = """You are evaluating an AI agent's response.
+
+Criteria: {criteria}
+
+Agent response:
+{outputs}
+
+Rate how well the response meets the criteria on a scale from 0.0 to 1.0."""
+
+
+def _last_ai_content(agent_output: AgentOutput) -> str:
+    """Extract text content from the last AI message."""
+    for msg in reversed(agent_output.messages):
+        if hasattr(msg, "type") and msg.type == "ai" and msg.content:
+            return str(msg.content)
+    return ""
+
+
+def llm_judge(agent_output: AgentOutput, config: Any) -> AssertionResult:
+    """Evaluate agent response against criteria using an LLM judge.
+
+    Config can be:
+      - str: criteria string (threshold defaults to 0.5)
+      - dict: {"criteria": str, "threshold": float, "model": str (optional)}
+    """
+    if isinstance(config, str):
+        criteria = config
+        threshold = 0.5
+        model = _DEFAULT_JUDGE_MODEL
+    elif isinstance(config, dict):
+        criteria = config.get("criteria", "")
+        threshold = config.get("threshold", 0.5)
+        model = config.get("model", _DEFAULT_JUDGE_MODEL)
+    else:
+        return AssertionResult(
+            passed=False,
+            name="llm_judge",
+            detail=f"Invalid config type: {type(config).__name__}",
+        )
+
+    if not criteria:
+        return AssertionResult(
+            passed=False,
+            name="llm_judge",
+            detail="No criteria specified",
+        )
+
+    agent_text = _last_ai_content(agent_output)
+
+    try:
+        evaluator = create_llm_as_judge(
+            prompt=_LLM_JUDGE_PROMPT,
+            model=model,
+            continuous=True,
+            feedback_key="score",
+        )
+
+        result = evaluator(
+            outputs=agent_text,
+            criteria=criteria,
+        )
+
+        # result is a list[EvaluatorResult]; take the first entry
+        first = result[0] if isinstance(result, list) else result
+        score = float(first["score"]) if isinstance(first, dict) else float(first.score)
+        comment = first.get("comment") or "" if isinstance(first, dict) else (first.comment or "")
+        passed = score >= threshold
+
+        return AssertionResult(
+            passed=passed,
+            name="llm_judge",
+            detail=f"Score: {score:.2f} (threshold: {threshold}). {comment}",
+        )
+
+    except Exception as e:
+        logger.warning("LLM judge failed: %s", e)
+        return AssertionResult(
+            passed=False,
+            name="llm_judge",
+            detail=f"Judge error: {type(e).__name__}: {e}",
+        )
+
+
+_TRAJECTORY_PROMPT = """You are evaluating an AI agent's tool usage trajectory.
+
+Criteria: {criteria}
+
+Full conversation (messages and tool calls):
+{outputs}
+
+Rate how well the agent's tool usage meets the criteria on a scale from 0.0 to 1.0.
+Consider: Were tools used efficiently? Were there redundant calls? Was the sequence logical?"""
+
+
+def _format_trajectory(agent_output: AgentOutput) -> str:
+    """Format the full message history including tool calls for the judge."""
+    lines: list[str] = []
+    for msg in agent_output.messages:
+        msg_type = getattr(msg, "type", "unknown")
+        content = str(msg.content) if msg.content else ""
+
+        if msg_type == "human":
+            lines.append(f"[User]: {content}")
+        elif msg_type == "ai":
+            lines.append(f"[Agent]: {content}")
+            tool_calls = getattr(msg, "tool_calls", None)
+            if tool_calls:
+                for tc in tool_calls:
+                    if isinstance(tc, dict):
+                        lines.append(f"  -> Tool call: {tc.get('name', '?')}({tc.get('args', {})})")
+                    elif hasattr(tc, "name"):
+                        lines.append(f"  -> Tool call: {tc.name}({getattr(tc, 'args', {})})")
+        elif msg_type == "tool":
+            tool_name = getattr(msg, "name", "?")
+            lines.append(f"[Tool result ({tool_name})]: {content[:200]}...")
+    return "\n".join(lines)
+
+
+def trajectory_quality(agent_output: AgentOutput, config: Any) -> AssertionResult:
+    """Evaluate the quality of an agent's tool usage trajectory.
+
+    Config: {"criteria": str, "threshold": float (default 0.5), "model": str (optional)}
+    """
+    if isinstance(config, str):
+        criteria = config
+        threshold = 0.5
+        model = _DEFAULT_JUDGE_MODEL
+    elif isinstance(config, dict):
+        criteria = config.get("criteria", "")
+        threshold = config.get("threshold", 0.5)
+        model = config.get("model", _DEFAULT_JUDGE_MODEL)
+    else:
+        return AssertionResult(
+            passed=False,
+            name="trajectory_quality",
+            detail=f"Invalid config type: {type(config).__name__}",
+        )
+
+    if not criteria:
+        return AssertionResult(
+            passed=False,
+            name="trajectory_quality",
+            detail="No criteria specified",
+        )
+
+    trajectory_text = _format_trajectory(agent_output)
+
+    try:
+        evaluator = create_llm_as_judge(
+            prompt=_TRAJECTORY_PROMPT,
+            model=model,
+            continuous=True,
+            feedback_key="score",
+        )
+
+        result = evaluator(
+            outputs=trajectory_text,
+            criteria=criteria,
+        )
+
+        # result is a list[EvaluatorResult]; take the first entry
+        first = result[0] if isinstance(result, list) else result
+        score = float(first["score"]) if isinstance(first, dict) else float(first.score)
+        comment = first.get("comment") or "" if isinstance(first, dict) else (first.comment or "")
+        passed = score >= threshold
+
+        return AssertionResult(
+            passed=passed,
+            name="trajectory_quality",
+            detail=f"Score: {score:.2f} (threshold: {threshold}). {comment}",
+        )
+
+    except Exception as e:
+        logger.warning("Trajectory quality judge failed: %s", e)
+        return AssertionResult(
+            passed=False,
+            name="trajectory_quality",
+            detail=f"Judge error: {type(e).__name__}: {e}",
+        )