agent-reliability-harness 0.2.1__py3-none-any.whl

agent_reliability_harness/__init__.py

@@ -0,0 +1,58 @@
+"""agent_reliability_harness
+
+A dependency-free, local-first reliability and regression-testing harness
+for tool-using AI agents. It validates agent execution traces against
+declarative policies (tool-call contracts, trajectory rules, budgets,
+unsafe-pattern detection, grounding), and compares candidate runs against a
+saved baseline to gate CI on real regressions.
+
+The core is deterministic: no model calls, no network, no clock reads.
+"""
+
+from agent_reliability_harness.models import (
+    SCHEMA_VERSION,
+    ArgSpec,
+    Budgets,
+    CompletionPolicy,
+    ErrorHandlingPolicy,
+    Finding,
+    GroundingPolicy,
+    Policy,
+    SequencePolicy,
+    Step,
+    ToolSchema,
+    Trace,
+    TraceReport,
+)
+from agent_reliability_harness.regression import (
+    ComparisonResult,
+    compare_reports,
+    evaluate_gate,
+)
+from agent_reliability_harness.rules import RULES, Rule
+from agent_reliability_harness.validator import validate_trace
+
+__version__ = "0.2.1"
+
+__all__ = [
+    "ArgSpec",
+    "Budgets",
+    "CompletionPolicy",
+    "ComparisonResult",
+    "ErrorHandlingPolicy",
+    "Finding",
+    "GroundingPolicy",
+    "Policy",
+    "RULES",
+    "Rule",
+    "SCHEMA_VERSION",
+    "SequencePolicy",
+    "Step",
+    "ToolSchema",
+    "Trace",
+    "TraceReport",
+    "compare_reports",
+    "evaluate_gate",
+    "validate_trace",
+    "__version__",
+]

agent_reliability_harness/__main__.py

@@ -0,0 +1,4 @@
+from agent_reliability_harness.cli import main
+
+if __name__ == "__main__":
+    raise SystemExit(main())

agent_reliability_harness/adapters/__init__.py

@@ -0,0 +1,103 @@
+"""Provider adapters: convert provider-native transcripts into canonical traces.
+
+Adapters are pure, deterministic functions from a provider-native JSON
+document to a canonical ARH trace ``dict`` (see TRACE-SPEC.md). They never
+call a network, never read a clock, and never drop information silently:
+anything an adapter cannot map is recorded under
+``trace["metadata"]["adapter"]`` so the loss is visible.
+
+Supported formats:
+
+- ``arh``: the canonical trace format (no conversion).
+- ``openai-chat``: an OpenAI Chat Completions message list with
+  ``tool_calls`` / ``role: "tool"`` messages.
+- ``anthropic-messages``: an Anthropic Messages API conversation with
+  ``tool_use`` / ``tool_result`` content blocks.
+
+Fields the source format does not carry (see docs/adapters.md for the full
+support matrix) are left unset, which automatically marks the dependent
+policy checks (e.g. latency budgets) as not applicable instead of silently
+passing them.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from typing import Any
+
+from agent_reliability_harness.adapters import anthropic_messages, openai_chat
+
+FORMAT_ARH = "arh"
+FORMAT_OPENAI_CHAT = "openai-chat"
+FORMAT_ANTHROPIC_MESSAGES = "anthropic-messages"
+FORMAT_AUTO = "auto"
+
+FORMATS = (FORMAT_ARH, FORMAT_OPENAI_CHAT, FORMAT_ANTHROPIC_MESSAGES)
+
+_ADAPTERS: dict[str, Callable[[Any, str], dict[str, Any]]] = {
+    FORMAT_OPENAI_CHAT: openai_chat.to_trace_dict,
+    FORMAT_ANTHROPIC_MESSAGES: anthropic_messages.to_trace_dict,
+}
+
+
+def detect_format(raw: Any) -> str:
+    """Deterministically detect the trace format of a parsed JSON document.
+
+    Detection rules, applied in order:
+
+    1. An object with a ``steps`` list is the canonical ``arh`` format.
+    2. An object with a ``messages`` list (or a bare message list) is
+       inspected message by message:
+       a. any ``role: "assistant"`` message with ``tool_calls`` or
+          ``function_call``, or any ``role: "tool"`` message -> ``openai-chat``;
+       b. any message whose ``content`` is a list containing ``tool_use`` /
+          ``tool_result`` typed blocks -> ``anthropic-messages``.
+    3. A plain-text-only message list defaults to ``openai-chat`` (both
+       vendors' text-only transcripts are structurally identical; the
+       resulting canonical trace is the same either way).
+
+    Raises ``ValueError`` for anything else.
+    """
+    if isinstance(raw, dict) and isinstance(raw.get("steps"), list):
+        return FORMAT_ARH
+    messages: Any = None
+    if isinstance(raw, dict) and isinstance(raw.get("messages"), list):
+        messages = raw["messages"]
+    elif isinstance(raw, list):
+        messages = raw
+    if messages is None:
+        raise ValueError(
+            "cannot detect trace format: expected an object with 'steps' (arh), "
+            "an object with 'messages', or a bare message list"
+        )
+    for message in messages:
+        if not isinstance(message, dict):
+            continue
+        if message.get("role") == "tool" or "tool_calls" in message or "function_call" in message:
+            return FORMAT_OPENAI_CHAT
+        content = message.get("content")
+        if isinstance(content, list) and any(
+            isinstance(block, dict) and block.get("type") in ("tool_use", "tool_result")
+            for block in content
+        ):
+            return FORMAT_ANTHROPIC_MESSAGES
+    return FORMAT_OPENAI_CHAT
+
+
+def normalize(raw: Any, fmt: str, fallback_trace_id: str) -> dict[str, Any]:
+    """Convert a provider-native document into a canonical trace dict.
+
+    ``fmt`` may be a concrete format name or ``"auto"`` for detection.
+    ``fallback_trace_id`` is used when the source document carries no trace
+    identifier (typically the source file stem).
+    """
+    if fmt == FORMAT_AUTO:
+        fmt = detect_format(raw)
+    if fmt == FORMAT_ARH:
+        if not isinstance(raw, dict):
+            raise ValueError("arh-format trace must be a JSON object")
+        return raw
+    adapter = _ADAPTERS.get(fmt)
+    if adapter is None:
+        raise ValueError(f"unknown trace format '{fmt}' (expected one of {FORMATS})")
+    return adapter(raw, fallback_trace_id)

agent_reliability_harness/adapters/anthropic_messages.py

@@ -0,0 +1,158 @@
+"""Adapter for Anthropic Messages API conversations.
+
+Maps an Anthropic-style conversation (list of messages whose ``content`` is
+a string or a list of typed blocks) into a canonical ARH trace:
+
+- assistant ``tool_use`` blocks become ``tool_call`` steps (``input`` maps
+  to ``arguments``);
+- user ``tool_result`` blocks attach their content as the ``output`` of the
+  matching tool_call step (via ``tool_use_id``); ``is_error: true`` sets the
+  step's ``status`` to ``"error"`` and records the result text as ``error``;
+- assistant ``text`` blocks become ``model_response`` steps; a block's
+  ``citations`` list (Anthropic citations feature) maps to step citations;
+- the top-level ``system`` prompt and user text content are skipped: they
+  are inputs to the agent, not agent behavior.
+
+Not carried by this format (documented, not guessed): latency, cost, and
+per-message token usage (transcripts do not embed ``usage``). Orphan tool
+results are recorded under ``metadata.adapter`` instead of being dropped.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+SOURCE = "anthropic-messages"
+
+
+def _result_text(content: Any) -> Any:
+    """Flatten a tool_result content payload to a comparable value."""
+    if isinstance(content, list):
+        texts = [
+            block.get("text", "")
+            for block in content
+            if isinstance(block, dict) and block.get("type") == "text"
+        ]
+        if texts:
+            return "\n".join(texts)
+    return content
+
+
+def to_trace_dict(raw: Any, fallback_trace_id: str) -> dict[str, Any]:
+    if isinstance(raw, list):
+        wrapper: dict[str, Any] = {"messages": raw}
+    elif isinstance(raw, dict):
+        wrapper = raw
+    else:
+        raise ValueError(
+            f"anthropic-messages input must be an object with 'messages' or a message "
+            f"list, got {type(raw).__name__}"
+        )
+    messages = wrapper.get("messages")
+    if not isinstance(messages, list):
+        raise ValueError("anthropic-messages input has no 'messages' list")
+
+    steps: list[dict[str, Any]] = []
+    step_by_use_id: dict[str, dict[str, Any]] = {}
+    adapter_notes: list[dict[str, Any]] = []
+
+    for index, message in enumerate(messages):
+        if not isinstance(message, dict):
+            adapter_notes.append({"issue": "non_object_message", "message_index": index})
+            continue
+        role = message.get("role")
+        content = message.get("content")
+        if role == "assistant":
+            if isinstance(content, str):
+                if content.strip():
+                    steps.append(
+                        {
+                            "step_id": f"m{index}-text0",
+                            "type": "model_response",
+                            "text": content,
+                        }
+                    )
+                continue
+            if not isinstance(content, list):
+                continue
+            for block_index, block in enumerate(content):
+                if not isinstance(block, dict):
+                    adapter_notes.append(
+                        {
+                            "issue": "non_object_block",
+                            "message_index": index,
+                            "block_index": block_index,
+                        }
+                    )
+                    continue
+                block_type = block.get("type")
+                if block_type == "tool_use":
+                    step_id = str(block.get("id") or f"m{index}-tool{block_index}")
+                    arguments = block.get("input")
+                    step: dict[str, Any] = {
+                        "step_id": step_id,
+                        "type": "tool_call",
+                        "tool_name": block.get("name"),
+                        "arguments": arguments if isinstance(arguments, dict) else {},
+                    }
+                    if not isinstance(arguments, dict) and arguments is not None:
+                        adapter_notes.append(
+                            {
+                                "step_id": step_id,
+                                "issue": "arguments_not_object",
+                                "raw_input": arguments,
+                            }
+                        )
+                    steps.append(step)
+                    step_by_use_id[step_id] = step
+                elif block_type == "text":
+                    text = block.get("text")
+                    if isinstance(text, str) and text.strip():
+                        step = {
+                            "step_id": f"m{index}-text{block_index}",
+                            "type": "model_response",
+                            "text": text,
+                        }
+                        citations = block.get("citations")
+                        if isinstance(citations, list) and citations:
+                            step["citations"] = [
+                                c for c in citations if isinstance(c, dict)
+                            ]
+                        steps.append(step)
+        elif role == "user":
+            if not isinstance(content, list):
+                continue  # plain user text: agent input, skipped
+            for block in content:
+                if not isinstance(block, dict) or block.get("type") != "tool_result":
+                    continue
+                use_id = block.get("tool_use_id")
+                target = step_by_use_id.get(str(use_id)) if use_id is not None else None
+                if target is None:
+                    adapter_notes.append(
+                        {
+                            "issue": "unmatched_tool_result",
+                            "message_index": index,
+                            "tool_use_id": use_id,
+                        }
+                    )
+                    continue
+                result = _result_text(block.get("content"))
+                target["output"] = result
+                if block.get("is_error") is True:
+                    target["status"] = "error"
+                    target["error"] = result if isinstance(result, str) else "tool error"
+
+    trace: dict[str, Any] = {
+        "schema_version": "1",
+        "trace_id": str(wrapper.get("trace_id") or fallback_trace_id),
+        "agent_name": str(wrapper.get("agent_name") or "anthropic-agent"),
+        "workflow": str(wrapper.get("workflow") or "anthropic-messages-import"),
+        "source": SOURCE,
+        "steps": steps,
+    }
+    metadata = dict(wrapper.get("metadata") or {})
+    if adapter_notes:
+        metadata.setdefault("adapter", {})["notes"] = adapter_notes
+    if metadata:
+        trace["metadata"] = metadata
+    return trace

agent_reliability_harness/adapters/openai_chat.py

@@ -0,0 +1,155 @@
+"""Adapter for OpenAI Chat Completions message lists.
+
+Maps an OpenAI-style conversation transcript into a canonical ARH trace:
+
+- each entry of an assistant message's ``tool_calls`` becomes a
+  ``tool_call`` step (arguments JSON-decoded from ``function.arguments``);
+- legacy single ``function_call`` fields are mapped the same way;
+- ``role: "tool"`` messages attach their ``content`` as the ``output`` of
+  the matching tool_call step (via ``tool_call_id``);
+- non-empty assistant text ``content`` becomes a ``model_response`` step;
+- ``system`` and ``user`` messages are skipped: they are inputs to the
+  agent, not agent behavior. (Policy safety checks scan agent-produced
+  content; if you need to scan user input, include it in the canonical
+  format directly.)
+
+Not carried by this format (documented, not guessed): latency, cost, token
+usage per message, citations, and step status (the Chat Completions
+transcript has no error channel for tool results, so ``status`` is always
+``"ok"``). Unparseable ``function.arguments`` and orphan tool results are
+recorded under ``metadata.adapter`` instead of being dropped.
+"""
+
+from __future__ import annotations
+
+import json
+from typing import Any
+
+SOURCE = "openai-chat"
+
+
+def _tool_call_step(
+    call: dict[str, Any], step_id: str, adapter_notes: list[dict[str, Any]]
+) -> dict[str, Any]:
+    function = call.get("function") or {}
+    name = function.get("name") or call.get("name")
+    raw_arguments = function.get("arguments", call.get("arguments"))
+    arguments: dict[str, Any] = {}
+    if isinstance(raw_arguments, dict):
+        arguments = raw_arguments
+    elif isinstance(raw_arguments, str) and raw_arguments.strip():
+        try:
+            parsed = json.loads(raw_arguments)
+            if isinstance(parsed, dict):
+                arguments = parsed
+            else:
+                adapter_notes.append(
+                    {
+                        "step_id": step_id,
+                        "issue": "arguments_not_object",
+                        "raw_arguments": raw_arguments,
+                    }
+                )
+        except json.JSONDecodeError as exc:
+            adapter_notes.append(
+                {
+                    "step_id": step_id,
+                    "issue": "argument_parse_error",
+                    "error": str(exc),
+                    "raw_arguments": raw_arguments,
+                }
+            )
+    step: dict[str, Any] = {
+        "step_id": step_id,
+        "type": "tool_call",
+        "tool_name": name,
+        "arguments": arguments,
+    }
+    return step
+
+
+def to_trace_dict(raw: Any, fallback_trace_id: str) -> dict[str, Any]:
+    if isinstance(raw, list):
+        wrapper: dict[str, Any] = {"messages": raw}
+    elif isinstance(raw, dict):
+        wrapper = raw
+    else:
+        raise ValueError(
+            f"openai-chat input must be an object with 'messages' or a message list, "
+            f"got {type(raw).__name__}"
+        )
+    messages = wrapper.get("messages")
+    if not isinstance(messages, list):
+        raise ValueError("openai-chat input has no 'messages' list")
+
+    steps: list[dict[str, Any]] = []
+    step_by_call_id: dict[str, dict[str, Any]] = {}
+    adapter_notes: list[dict[str, Any]] = []
+
+    for index, message in enumerate(messages):
+        if not isinstance(message, dict):
+            adapter_notes.append({"issue": "non_object_message", "message_index": index})
+            continue
+        role = message.get("role")
+        if role == "assistant":
+            tool_calls = message.get("tool_calls")
+            if isinstance(tool_calls, list):
+                for call_index, call in enumerate(tool_calls):
+                    if not isinstance(call, dict):
+                        adapter_notes.append(
+                            {
+                                "issue": "non_object_tool_call",
+                                "message_index": index,
+                                "tool_call_index": call_index,
+                            }
+                        )
+                        continue
+                    step_id = str(call.get("id") or f"m{index}-tool{call_index}")
+                    step = _tool_call_step(call, step_id, adapter_notes)
+                    steps.append(step)
+                    step_by_call_id[step_id] = step
+            function_call = message.get("function_call")
+            if isinstance(function_call, dict):
+                step_id = f"m{index}-function"
+                step = _tool_call_step(function_call, step_id, adapter_notes)
+                steps.append(step)
+                step_by_call_id[step_id] = step
+            content = message.get("content")
+            if isinstance(content, str) and content.strip():
+                steps.append(
+                    {
+                        "step_id": f"m{index}-response",
+                        "type": "model_response",
+                        "text": content,
+                    }
+                )
+        elif role == "tool":
+            call_id = message.get("tool_call_id")
+            target = step_by_call_id.get(str(call_id)) if call_id is not None else None
+            content = message.get("content")
+            if target is None:
+                adapter_notes.append(
+                    {
+                        "issue": "unmatched_tool_result",
+                        "message_index": index,
+                        "tool_call_id": call_id,
+                    }
+                )
+            else:
+                target["output"] = content
+        # system/user/developer messages: agent inputs, intentionally skipped.
+
+    trace: dict[str, Any] = {
+        "schema_version": "1",
+        "trace_id": str(wrapper.get("trace_id") or fallback_trace_id),
+        "agent_name": str(wrapper.get("agent_name") or "openai-chat-agent"),
+        "workflow": str(wrapper.get("workflow") or "openai-chat-import"),
+        "source": SOURCE,
+        "steps": steps,
+    }
+    metadata = dict(wrapper.get("metadata") or {})
+    if adapter_notes:
+        metadata.setdefault("adapter", {})["notes"] = adapter_notes
+    if metadata:
+        trace["metadata"] = metadata
+    return trace