agentguard47 0.2.0__tar.gz → 0.3.0__tar.gz

{agentguard47-0.2.0 → agentguard47-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: agentguard47
-Version: 0.2.0
+Version: 0.3.0
 Summary: Lightweight observability and evaluation primitives for multi-agent systems
 Author: AgentGuard
 License-Expression: MIT
@@ -105,6 +105,45 @@ replayer = Replayer("runs.jsonl")
 resp = replayer.replay_call("llm", {"prompt": "hi"})
 ```
 
+## Evaluation as Code
+
+```python
+from agentguard import EvalSuite
+
+result = (
+    EvalSuite("traces.jsonl")
+    .assert_no_loops()
+    .assert_tool_called("search", min_times=1)
+    .assert_budget_under(tokens=50000)
+    .assert_completes_within(30.0)
+    .assert_no_errors()
+    .run()
+)
+print(result.summary)
+```
+
+## Auto-Instrumentation
+
+```python
+from agentguard import Tracer
+from agentguard.instrument import trace_agent, trace_tool
+
+tracer = Tracer()
+
+@trace_agent(tracer)
+def my_agent(query):
+    return search(query)
+
+@trace_tool(tracer)
+def search(q):
+    return f"results for {q}"
+
+# Monkey-patch OpenAI/Anthropic (safe if not installed)
+from agentguard.instrument import patch_openai, patch_anthropic
+patch_openai(tracer)
+patch_anthropic(tracer)
+```
+
 ## CLI
 
 ```bash
@@ -114,8 +153,11 @@ agentguard summarize traces.jsonl
 # Human-readable report
 agentguard report traces.jsonl
 
-# Open trace viewer in browser
+# Open Gantt trace viewer in browser
 agentguard view traces.jsonl
+
+# Run evaluation assertions
+agentguard eval traces.jsonl
 ```
 
 ## Trace Viewer
@@ -124,6 +166,8 @@ agentguard view traces.jsonl
 agentguard view traces.jsonl --port 8080
 ```
 
+Gantt-style timeline with color-coded spans (reasoning, tool, LLM, guard, error), click-to-expand detail panel, and aggregate stats.
+
 ## Integrations
 
 - LangChain: `agentguard.integrations.langchain`

{agentguard47-0.2.0 → agentguard47-0.3.0}/README.md

@@ -80,6 +80,45 @@ replayer = Replayer("runs.jsonl")
 resp = replayer.replay_call("llm", {"prompt": "hi"})
 ```
 
+## Evaluation as Code
+
+```python
+from agentguard import EvalSuite
+
+result = (
+    EvalSuite("traces.jsonl")
+    .assert_no_loops()
+    .assert_tool_called("search", min_times=1)
+    .assert_budget_under(tokens=50000)
+    .assert_completes_within(30.0)
+    .assert_no_errors()
+    .run()
+)
+print(result.summary)
+```
+
+## Auto-Instrumentation
+
+```python
+from agentguard import Tracer
+from agentguard.instrument import trace_agent, trace_tool
+
+tracer = Tracer()
+
+@trace_agent(tracer)
+def my_agent(query):
+    return search(query)
+
+@trace_tool(tracer)
+def search(q):
+    return f"results for {q}"
+
+# Monkey-patch OpenAI/Anthropic (safe if not installed)
+from agentguard.instrument import patch_openai, patch_anthropic
+patch_openai(tracer)
+patch_anthropic(tracer)
+```
+
 ## CLI
 
 ```bash
@@ -89,8 +128,11 @@ agentguard summarize traces.jsonl
 # Human-readable report
 agentguard report traces.jsonl
 
-# Open trace viewer in browser
+# Open Gantt trace viewer in browser
 agentguard view traces.jsonl
+
+# Run evaluation assertions
+agentguard eval traces.jsonl
 ```
 
 ## Trace Viewer
@@ -99,6 +141,8 @@ agentguard view traces.jsonl
 agentguard view traces.jsonl --port 8080
 ```
 
+Gantt-style timeline with color-coded spans (reasoning, tool, LLM, guard, error), click-to-expand detail panel, and aggregate stats.
+
 ## Integrations
 
 - LangChain: `agentguard.integrations.langchain`

{agentguard47-0.2.0 → agentguard47-0.3.0}/agentguard/__init__.py

@@ -9,6 +9,7 @@ from .guards import (
 )
 from .recording import Recorder, Replayer
 from .sinks import HttpSink
+from .evaluation import EvalSuite, EvalResult, AssertionResult
 
 __all__ = [
     "Tracer",
@@ -21,4 +22,7 @@ __all__ = [
     "Recorder",
     "Replayer",
     "HttpSink",
+    "EvalSuite",
+    "EvalResult",
+    "AssertionResult",
 ]

{agentguard47-0.2.0 → agentguard47-0.3.0}/agentguard/cli.py

@@ -81,6 +81,21 @@ def _report(path: str) -> None:
         print("  Loop guard triggered: 0")
 
 
+def _eval(path: str) -> None:
+    from agentguard.evaluation import EvalSuite
+
+    result = (
+        EvalSuite(path)
+        .assert_no_loops()
+        .assert_no_errors()
+        .assert_completes_within(30.0)
+        .run()
+    )
+    print(result.summary)
+    if not result.passed:
+        raise SystemExit(1)
+
+
 def main() -> None:
     parser = argparse.ArgumentParser(prog="agentguard")
     sub = parser.add_subparsers(dest="cmd")
@@ -96,6 +111,9 @@ def main() -> None:
     view.add_argument("--port", type=int, default=8080)
     view.add_argument("--no-open", action="store_true")
 
+    eval_cmd = sub.add_parser("eval", help="Run evaluation assertions on a trace")
+    eval_cmd.add_argument("path")
+
     args = parser.parse_args()
     if args.cmd == "summarize":
         _summarize(args.path)
@@ -105,6 +123,8 @@ def main() -> None:
         from agentguard.viewer import serve
 
         serve(args.path, port=args.port, open_browser=not args.no_open)
+    elif args.cmd == "eval":
+        _eval(args.path)
     else:
         parser.print_help()
 

agentguard47-0.3.0/agentguard/evaluation.py

@@ -0,0 +1,220 @@
+"""Evaluation as Code — assertion-based trace analysis."""
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Optional
+
+
+@dataclass
+class AssertionResult:
+    name: str
+    passed: bool
+    message: str
+
+
+@dataclass
+class EvalResult:
+    assertions: List[AssertionResult] = field(default_factory=list)
+
+    @property
+    def passed(self) -> bool:
+        return all(a.passed for a in self.assertions)
+
+    @property
+    def summary(self) -> str:
+        total = len(self.assertions)
+        passed = sum(1 for a in self.assertions if a.passed)
+        failed = total - passed
+        lines = [f"EvalResult: {passed}/{total} passed, {failed} failed"]
+        for a in self.assertions:
+            status = "PASS" if a.passed else "FAIL"
+            lines.append(f"  [{status}] {a.name}: {a.message}")
+        return "\n".join(lines)
+
+
+class EvalSuite:
+    """Load a trace from JSONL and run assertions against it."""
+
+    def __init__(self, path: str) -> None:
+        self._events = _load_events(path)
+        self._assertions: List[_Assertion] = []
+
+    @property
+    def events(self) -> List[Dict[str, Any]]:
+        return list(self._events)
+
+    def assert_no_loops(self) -> "EvalSuite":
+        """Assert that no loop guard events were recorded."""
+        self._assertions.append(_Assertion(
+            name="no_loops",
+            check=_check_no_loops,
+        ))
+        return self
+
+    def assert_tool_called(self, name: str, min_times: int = 1) -> "EvalSuite":
+        """Assert a tool was called at least min_times."""
+        self._assertions.append(_Assertion(
+            name=f"tool_called:{name}>={min_times}",
+            check=lambda events, n=name, m=min_times: _check_tool_called(events, n, m),
+        ))
+        return self
+
+    def assert_budget_under(self, tokens: Optional[int] = None, calls: Optional[int] = None) -> "EvalSuite":
+        """Assert total token/call usage is under a limit."""
+        label_parts = []
+        if tokens is not None:
+            label_parts.append(f"tokens<{tokens}")
+        if calls is not None:
+            label_parts.append(f"calls<{calls}")
+        self._assertions.append(_Assertion(
+            name=f"budget_under:{','.join(label_parts)}",
+            check=lambda events, t=tokens, c=calls: _check_budget_under(events, t, c),
+        ))
+        return self
+
+    def assert_completes_within(self, seconds: float) -> "EvalSuite":
+        """Assert the longest span completed within a time limit."""
+        self._assertions.append(_Assertion(
+            name=f"completes_within:{seconds}s",
+            check=lambda events, s=seconds: _check_completes_within(events, s),
+        ))
+        return self
+
+    def assert_event_exists(self, name: str) -> "EvalSuite":
+        """Assert that at least one event with the given name exists."""
+        self._assertions.append(_Assertion(
+            name=f"event_exists:{name}",
+            check=lambda events, n=name: _check_event_exists(events, n),
+        ))
+        return self
+
+    def assert_no_errors(self) -> "EvalSuite":
+        """Assert no events have error data."""
+        self._assertions.append(_Assertion(
+            name="no_errors",
+            check=_check_no_errors,
+        ))
+        return self
+
+    def run(self) -> EvalResult:
+        result = EvalResult()
+        for assertion in self._assertions:
+            ar = assertion.check(self._events)
+            result.assertions.append(ar)
+        return result
+
+
+@dataclass
+class _Assertion:
+    name: str
+    check: Any  # Callable[[List[Dict]], AssertionResult]
+
+
+# --- check functions ---
+
+
+def _check_no_loops(events: List[Dict[str, Any]]) -> AssertionResult:
+    loop_events = [e for e in events if e.get("name") == "guard.loop_detected"]
+    if loop_events:
+        return AssertionResult(
+            name="no_loops",
+            passed=False,
+            message=f"Found {len(loop_events)} loop detection event(s)",
+        )
+    return AssertionResult(name="no_loops", passed=True, message="No loops detected")
+
+
+def _check_tool_called(events: List[Dict[str, Any]], tool_name: str, min_times: int) -> AssertionResult:
+    name = f"tool_called:{tool_name}>={min_times}"
+    # Count tool.result events or span events with matching tool name
+    count = 0
+    for e in events:
+        ename = e.get("name", "")
+        if ename == "tool.result":
+            count += 1
+        elif ename.startswith(f"tool.{tool_name}"):
+            if e.get("phase") == "start" or e.get("kind") == "event":
+                count += 1
+    if count >= min_times:
+        return AssertionResult(name=name, passed=True, message=f"Tool called {count} time(s)")
+    return AssertionResult(name=name, passed=False, message=f"Tool called {count} time(s), expected >= {min_times}")
+
+
+def _check_budget_under(events: List[Dict[str, Any]], max_tokens: Optional[int], max_calls: Optional[int]) -> AssertionResult:
+    parts = []
+    if max_tokens is not None:
+        parts.append(f"tokens<{max_tokens}")
+    if max_calls is not None:
+        parts.append(f"calls<{max_calls}")
+    name = f"budget_under:{','.join(parts)}"
+
+    total_tokens = 0
+    total_calls = 0
+    for e in events:
+        data = e.get("data", {})
+        if isinstance(data, dict):
+            usage = data.get("token_usage") or data.get("usage") or {}
+            if isinstance(usage, dict):
+                total_tokens += usage.get("total_tokens", 0)
+        if e.get("name", "").startswith("tool.") and e.get("kind") == "span" and e.get("phase") == "start":
+            total_calls += 1
+
+    failures = []
+    if max_tokens is not None and total_tokens >= max_tokens:
+        failures.append(f"tokens={total_tokens} >= {max_tokens}")
+    if max_calls is not None and total_calls >= max_calls:
+        failures.append(f"calls={total_calls} >= {max_calls}")
+
+    if failures:
+        return AssertionResult(name=name, passed=False, message="; ".join(failures))
+    return AssertionResult(name=name, passed=True, message=f"tokens={total_tokens}, calls={total_calls}")
+
+
+def _check_completes_within(events: List[Dict[str, Any]], max_seconds: float) -> AssertionResult:
+    name = f"completes_within:{max_seconds}s"
+    max_ms = 0.0
+    for e in events:
+        dur = e.get("duration_ms")
+        if isinstance(dur, (int, float)) and dur > max_ms:
+            max_ms = dur
+    actual_seconds = max_ms / 1000.0
+    if actual_seconds <= max_seconds:
+        return AssertionResult(name=name, passed=True, message=f"Completed in {actual_seconds:.3f}s")
+    return AssertionResult(name=name, passed=False, message=f"Took {actual_seconds:.3f}s, limit is {max_seconds}s")
+
+
+def _check_event_exists(events: List[Dict[str, Any]], event_name: str) -> AssertionResult:
+    name = f"event_exists:{event_name}"
+    found = any(e.get("name") == event_name for e in events)
+    if found:
+        return AssertionResult(name=name, passed=True, message="Event found")
+    return AssertionResult(name=name, passed=False, message="Event not found")
+
+
+def _check_no_errors(events: List[Dict[str, Any]]) -> AssertionResult:
+    errors = [e for e in events if e.get("error") is not None]
+    if errors:
+        return AssertionResult(
+            name="no_errors",
+            passed=False,
+            message=f"Found {len(errors)} event(s) with errors",
+        )
+    return AssertionResult(name="no_errors", passed=True, message="No errors found")
+
+
+# --- loader ---
+
+
+def _load_events(path: str) -> List[Dict[str, Any]]:
+    events: List[Dict[str, Any]] = []
+    with open(path, "r", encoding="utf-8") as f:
+        for line in f:
+            line = line.strip()
+            if not line:
+                continue
+            try:
+                events.append(json.loads(line))
+            except json.JSONDecodeError:
+                continue
+    return events

agentguard47-0.3.0/agentguard/instrument.py

@@ -0,0 +1,183 @@
+"""Auto-instrumentation decorators and monkey-patches."""
+from __future__ import annotations
+
+import functools
+from typing import Any, Callable, Optional, TypeVar
+
+F = TypeVar("F", bound=Callable[..., Any])
+
+
+def trace_agent(tracer: Any, name: Optional[str] = None) -> Callable[[F], F]:
+    """Decorator that wraps a function in a top-level trace span.
+
+    Usage::
+
+        @trace_agent(tracer)
+        def my_agent(query: str) -> str:
+            ...
+    """
+
+    def decorator(fn: F) -> F:
+        span_name = name or f"agent.{fn.__name__}"
+
+        @functools.wraps(fn)
+        def wrapper(*args: Any, **kwargs: Any) -> Any:
+            with tracer.trace(span_name) as ctx:
+                kwargs["_trace_ctx"] = ctx
+                try:
+                    return fn(*args, **kwargs)
+                except Exception:
+                    raise
+                finally:
+                    kwargs.pop("_trace_ctx", None)
+
+        # If the function doesn't accept **kwargs, fall back to simple wrapping
+        @functools.wraps(fn)
+        def simple_wrapper(*args: Any, **kwargs: Any) -> Any:
+            with tracer.trace(span_name):
+                return fn(*args, **kwargs)
+
+        # Check if function can accept _trace_ctx kwarg
+        import inspect
+
+        sig = inspect.signature(fn)
+        has_var_keyword = any(
+            p.kind == inspect.Parameter.VAR_KEYWORD
+            for p in sig.parameters.values()
+        )
+        has_trace_ctx = "_trace_ctx" in sig.parameters
+
+        if has_var_keyword or has_trace_ctx:
+            return wrapper  # type: ignore[return-value]
+        return simple_wrapper  # type: ignore[return-value]
+
+    return decorator
+
+
+def trace_tool(tracer: Any, name: Optional[str] = None) -> Callable[[F], F]:
+    """Decorator that wraps a function in a tool span.
+
+    Usage::
+
+        @trace_tool(tracer)
+        def search(query: str) -> str:
+            ...
+    """
+
+    def decorator(fn: F) -> F:
+        span_name = name or f"tool.{fn.__name__}"
+
+        @functools.wraps(fn)
+        def wrapper(*args: Any, **kwargs: Any) -> Any:
+            with tracer.trace(span_name) as ctx:
+                result = fn(*args, **kwargs)
+                ctx.event("tool.result", data={"result": str(result)[:500]})
+                return result
+
+        return wrapper  # type: ignore[return-value]
+
+    return decorator
+
+
+def patch_openai(tracer: Any) -> None:
+    """Monkey-patch OpenAI's ChatCompletion.create to auto-trace calls.
+
+    Safe to call even if openai is not installed — silently returns.
+    """
+    try:
+        import openai  # noqa: F811
+    except ImportError:
+        return
+
+    _original = None
+
+    # Support openai >= 1.0 (client-based) and < 1.0 (module-based)
+    client_cls = getattr(openai, "OpenAI", None)
+    if client_cls is not None:
+        # openai >= 1.0: patch the completions create method on the class
+        chat_completions = getattr(
+            getattr(client_cls, "chat", None), "completions", None
+        )
+        if chat_completions is not None:
+            _original = getattr(chat_completions, "create", None)
+    else:
+        # openai < 1.0
+        chat = getattr(openai, "ChatCompletion", None)
+        if chat is not None:
+            _original = getattr(chat, "create", None)
+
+    if _original is None:
+        return
+
+    @functools.wraps(_original)
+    def traced_create(*args: Any, **kwargs: Any) -> Any:
+        model = kwargs.get("model", "unknown")
+        with tracer.trace(f"llm.openai.{model}") as ctx:
+            result = _original(*args, **kwargs)
+            # Try to extract usage
+            usage = getattr(result, "usage", None)
+            if usage is not None:
+                ctx.event(
+                    "llm.result",
+                    data={
+                        "model": model,
+                        "usage": {
+                            "prompt_tokens": getattr(usage, "prompt_tokens", 0),
+                            "completion_tokens": getattr(usage, "completion_tokens", 0),
+                            "total_tokens": getattr(usage, "total_tokens", 0),
+                        },
+                    },
+                )
+            return result
+
+    # Patch it back
+    if client_cls is not None and chat_completions is not None:
+        chat_completions.create = traced_create  # type: ignore[attr-defined]
+    else:
+        chat = getattr(openai, "ChatCompletion", None)
+        if chat is not None:
+            chat.create = traced_create  # type: ignore[attr-defined]
+
+
+def patch_anthropic(tracer: Any) -> None:
+    """Monkey-patch Anthropic's messages.create to auto-trace calls.
+
+    Safe to call even if anthropic is not installed — silently returns.
+    """
+    try:
+        import anthropic  # noqa: F811
+    except ImportError:
+        return
+
+    client_cls = getattr(anthropic, "Anthropic", None)
+    if client_cls is None:
+        return
+
+    messages = getattr(client_cls, "messages", None)
+    if messages is None:
+        return
+
+    _original = getattr(messages, "create", None)
+    if _original is None:
+        return
+
+    @functools.wraps(_original)
+    def traced_create(*args: Any, **kwargs: Any) -> Any:
+        model = kwargs.get("model", "unknown")
+        with tracer.trace(f"llm.anthropic.{model}") as ctx:
+            result = _original(*args, **kwargs)
+            usage = getattr(result, "usage", None)
+            if usage is not None:
+                ctx.event(
+                    "llm.result",
+                    data={
+                        "model": model,
+                        "usage": {
+                            "input_tokens": getattr(usage, "input_tokens", 0),
+                            "output_tokens": getattr(usage, "output_tokens", 0),
+                        },
+                    },
+                )
+            return result
+
+    messages.create = traced_create  # type: ignore[attr-defined]

{agentguard47-0.2.0 → agentguard47-0.3.0}/agentguard/integrations/langchain.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
 import uuid
-from typing import Any, Dict, List, Optional, Sequence
+from typing import Any, Dict, List, Optional
 
 from agentguard.guards import BudgetGuard, LoopGuard
 from agentguard.tracing import Tracer, TraceContext