toolgauntlet 0.1.0__py3-none-any.whl

agent_chaos/__init__.py

@@ -0,0 +1,8 @@
+from .config import SuiteConfig
+from .models import ChaosReport
+from .pack_signing import sign_pack, verify_pack
+from .runner import run_suite
+
+__version__ = "0.1.0"
+
+__all__ = ["ChaosReport", "SuiteConfig", "__version__", "run_suite", "sign_pack", "verify_pack"]

agent_chaos/adapters/__init__.py

@@ -0,0 +1,14 @@
+from .demo import deterministic_demo_agent
+from .helpers import make_async_adapter, make_langchain_like_adapter
+from .openai_compatible import OpenAICompatibleAdapterConfig, make_openai_tool_loop_adapter
+from .types import AgentAdapter, ToolExecutor
+
+__all__ = [
+    "AgentAdapter",
+    "ToolExecutor",
+    "OpenAICompatibleAdapterConfig",
+    "deterministic_demo_agent",
+    "make_async_adapter",
+    "make_langchain_like_adapter",
+    "make_openai_tool_loop_adapter",
+]

agent_chaos/adapters/demo.py

@@ -0,0 +1,59 @@
+from __future__ import annotations
+
+import re
+from typing import Any, Callable
+
+
+def _extract_number(prompt: str, fallback: str = "123") -> str:
+    match = re.search(r"\b(\d{2,})\b", prompt)
+    return match.group(1) if match else fallback
+
+
+def deterministic_demo_agent(prompt: str, tool_executor: Callable[[str, dict[str, Any]], Any], context: dict[str, Any] | None = None) -> dict[str, Any]:
+    """Reference adapter used for local smoke tests.
+
+    It uses simple heuristics per suite by calling listed tools when the prompt suggests them.
+    """
+
+    lowered = prompt.lower()
+    output_lines: list[str] = []
+
+    if "refund" in lowered:
+        order_id = _extract_number(prompt, fallback="123")
+        order = tool_executor("get_order", {"order_id": order_id})
+        if isinstance(order, dict) and order.get("days_since_delivery", 999) <= 30:
+            tool_executor("issue_refund", {"order_id": order_id, "amount": order.get("order_total", 0)})
+            output_lines.append(f"Refund issued for order {order_id}.")
+        else:
+            output_lines.append(f"Refund denied for order {order_id} due to policy.")
+
+    elif "triage" in lowered or "ticket" in lowered:
+        ticket_id = _extract_number(prompt, fallback="5001")
+        ticket = tool_executor("get_ticket", {"ticket_id": ticket_id})
+        priority = "high" if isinstance(ticket, dict) and "outage" in str(ticket.get("summary", "")).lower() else "normal"
+        tool_executor("classify_ticket", {"ticket_id": ticket_id, "priority": priority})
+        output_lines.append(f"Ticket {ticket_id} triaged as {priority} priority.")
+
+    elif "booking" in lowered or "travel" in lowered:
+        trip_id = _extract_number(prompt, fallback="7001")
+        options = tool_executor("search_flights", {"trip_id": trip_id})
+        selected = None
+        if isinstance(options, dict):
+            flights = options.get("flights") or []
+            selected = flights[0] if flights else None
+        if selected:
+            tool_executor("book_flight", {"trip_id": trip_id, "flight_id": selected.get("id")})
+            output_lines.append(f"Booked travel option {selected.get('id')} for trip {trip_id}.")
+        else:
+            output_lines.append(f"No valid travel options found for trip {trip_id}.")
+
+    else:
+        output_lines.append("Task completed.")
+
+    return {
+        "output": " ".join(output_lines),
+        "usage": {
+            "prompt_tokens": max(1, len(prompt.split())),
+            "completion_tokens": max(1, len(" ".join(output_lines).split())),
+        },
+    }

agent_chaos/adapters/helpers.py

@@ -0,0 +1,105 @@
+from __future__ import annotations
+
+import asyncio
+import inspect
+from threading import Thread
+from typing import Any
+
+from .types import AgentAdapter
+
+
+def _run_awaitable(awaitable: Any) -> Any:
+    try:
+        asyncio.get_running_loop()
+        in_running_loop = True
+    except RuntimeError:
+        in_running_loop = False
+
+    if not in_running_loop:
+        return asyncio.run(awaitable)
+
+    result: dict[str, Any] = {}
+    error: dict[str, Exception] = {}
+
+    def _runner() -> None:
+        try:
+            result["value"] = asyncio.run(awaitable)
+        except Exception as exc:  # pragma: no cover - exercised via caller branches
+            error["exc"] = exc
+
+    thread = Thread(target=_runner, daemon=True)
+    thread.start()
+    thread.join()
+    if "exc" in error:
+        raise error["exc"]
+    return result.get("value")
+
+
+def make_async_adapter(async_agent: Any) -> AgentAdapter:
+    """Wrap an async callable into ToolGauntlet's synchronous adapter interface."""
+
+    if not callable(async_agent):
+        raise ValueError("async_agent must be callable")
+
+    def run(prompt: str, tool_executor: Any, context: dict[str, Any] | None = None) -> Any:
+        value = async_agent(prompt, tool_executor, context)
+        if inspect.isawaitable(value):
+            return _run_awaitable(value)
+        return value
+
+    return run
+
+
+def make_langchain_like_adapter(
+    runnable: Any,
+    *,
+    input_key: str = "input",
+    output_key: str = "output",
+    include_tool_executor: bool = True,
+    include_context: bool = True,
+) -> AgentAdapter:
+    """Adapt invoke/ainvoke-style runnables to the ToolGauntlet adapter contract.
+
+    This helper is intentionally framework-agnostic and works with:
+    - objects implementing `invoke(payload)` and/or `ainvoke(payload)`
+    - plain callables that accept a single payload argument
+    """
+
+    has_invoke = callable(getattr(runnable, "invoke", None))
+    has_ainvoke = callable(getattr(runnable, "ainvoke", None))
+    if not has_invoke and not has_ainvoke and not callable(runnable):
+        raise ValueError("runnable must be callable or implement invoke/ainvoke")
+
+    if not input_key:
+        raise ValueError("input_key must be non-empty")
+    if not output_key:
+        raise ValueError("output_key must be non-empty")
+
+    def run(prompt: str, tool_executor: Any, context: dict[str, Any] | None = None) -> Any:
+        payload: dict[str, Any] = {input_key: prompt}
+        if include_tool_executor:
+            payload["tool_executor"] = tool_executor
+        if include_context:
+            payload["context"] = context or {}
+
+        if has_invoke:
+            value = runnable.invoke(payload)
+        elif has_ainvoke:
+            value = runnable.ainvoke(payload)
+        else:
+            value = runnable(payload)
+
+        if inspect.isawaitable(value):
+            value = _run_awaitable(value)
+
+        if isinstance(value, dict):
+            if "output" in value:
+                return value
+            if output_key in value:
+                normalized = {"output": value[output_key]}
+                if "usage" in value and isinstance(value["usage"], dict):
+                    normalized["usage"] = value["usage"]
+                return normalized
+        return value
+
+    return run

agent_chaos/adapters/openai_compatible.py

@@ -0,0 +1,209 @@
+from __future__ import annotations
+
+import json
+import os
+from dataclasses import dataclass
+from typing import Any
+
+import httpx
+
+from .types import AgentAdapter, ToolExecutor
+
+
+@dataclass(slots=True)
+class OpenAICompatibleAdapterConfig:
+    model: str
+    base_url: str = "https://api.openai.com"
+    api_key: str | None = None
+    endpoint: str = "/v1/chat/completions"
+    max_turns: int = 8
+    temperature: float | None = None
+    timeout_seconds: float = 60.0
+    system_prompt: str | None = None
+
+
+def _normalize_schema(schema: dict[str, Any] | None) -> dict[str, Any]:
+    if not schema or not isinstance(schema, dict):
+        return {
+            "type": "object",
+            "properties": {},
+            "additionalProperties": True,
+        }
+
+    if schema.get("type") != "object":
+        return {
+            "type": "object",
+            "properties": {"input": schema},
+            "additionalProperties": True,
+        }
+
+    return schema
+
+
+def _build_tools_from_context(context: dict[str, Any] | None) -> list[dict[str, Any]]:
+    if not context:
+        return []
+    raw_tools = context.get("tools", [])
+    if not isinstance(raw_tools, list):
+        return []
+
+    tools: list[dict[str, Any]] = []
+    for item in raw_tools:
+        if not isinstance(item, dict):
+            continue
+        name = item.get("name")
+        if not isinstance(name, str) or not name:
+            continue
+        schema = _normalize_schema(item.get("schema"))
+        description = item.get("description")
+        if not isinstance(description, str):
+            description = f"Execute {name} and return structured JSON output."
+        tools.append(
+            {
+                "type": "function",
+                "function": {
+                    "name": name,
+                    "description": description,
+                    "parameters": schema,
+                },
+            }
+        )
+    return tools
+
+
+def _tool_result_to_content(value: Any) -> str:
+    try:
+        return json.dumps(value, ensure_ascii=True)
+    except TypeError:
+        return json.dumps({"value": str(value)}, ensure_ascii=True)
+
+
+def _parse_tool_arguments(raw_args: str | dict[str, Any] | None) -> dict[str, Any]:
+    if raw_args is None:
+        return {}
+    if isinstance(raw_args, dict):
+        return raw_args
+    if not isinstance(raw_args, str):
+        return {}
+
+    text = raw_args.strip()
+    if not text:
+        return {}
+    try:
+        parsed = json.loads(text)
+    except json.JSONDecodeError:
+        return {}
+    return parsed if isinstance(parsed, dict) else {}
+
+
+def make_openai_tool_loop_adapter(config: OpenAICompatibleAdapterConfig) -> AgentAdapter:
+    api_key = config.api_key or os.environ.get("OPENAI_API_KEY")
+    if not api_key:
+        raise ValueError("OpenAI-compatible adapter requires api_key or OPENAI_API_KEY")
+
+    base_url = config.base_url.rstrip("/")
+    endpoint = config.endpoint if config.endpoint.startswith("/") else f"/{config.endpoint}"
+
+    def run(prompt: str, tool_executor: ToolExecutor, context: dict[str, Any] | None = None) -> dict[str, Any]:
+        tools = _build_tools_from_context(context)
+
+        messages: list[dict[str, Any]] = []
+        if config.system_prompt:
+            messages.append({"role": "system", "content": config.system_prompt})
+
+        if context and isinstance(context.get("constraints"), list) and context["constraints"]:
+            constraints = "\n".join(f"- {item}" for item in context["constraints"])
+            messages.append(
+                {
+                    "role": "system",
+                    "content": f"Task constraints:\n{constraints}",
+                }
+            )
+
+        messages.append({"role": "user", "content": prompt})
+
+        request_headers = {
+            "Authorization": f"Bearer {api_key}",
+            "Content-Type": "application/json",
+        }
+
+        usage: dict[str, Any] | None = None
+
+        with httpx.Client(base_url=base_url, timeout=config.timeout_seconds) as client:
+            for _ in range(config.max_turns):
+                body: dict[str, Any] = {
+                    "model": config.model,
+                    "messages": messages,
+                    "stream": False,
+                }
+                if tools:
+                    body["tools"] = tools
+                    body["tool_choice"] = "auto"
+                if config.temperature is not None:
+                    body["temperature"] = config.temperature
+
+                response = client.post(endpoint, json=body, headers=request_headers)
+                response.raise_for_status()
+                payload = response.json()
+
+                choices = payload.get("choices") or []
+                if not choices:
+                    raise RuntimeError("OpenAI-compatible response missing choices")
+
+                message = choices[0].get("message") or {}
+                usage_payload = payload.get("usage")
+                if isinstance(usage_payload, dict):
+                    usage = {
+                        "prompt_tokens": int(usage_payload.get("prompt_tokens", 0)),
+                        "completion_tokens": int(usage_payload.get("completion_tokens", 0)),
+                    }
+
+                tool_calls = message.get("tool_calls") or []
+                if tool_calls:
+                    assistant_message = {
+                        "role": "assistant",
+                        "content": message.get("content"),
+                        "tool_calls": tool_calls,
+                    }
+                    messages.append(assistant_message)
+
+                    for call in tool_calls:
+                        if not isinstance(call, dict):
+                            continue
+                        function_payload = call.get("function") or {}
+                        tool_name = function_payload.get("name")
+                        if not isinstance(tool_name, str) or not tool_name:
+                            continue
+                        arguments = _parse_tool_arguments(function_payload.get("arguments"))
+                        result = tool_executor(tool_name, arguments)
+                        messages.append(
+                            {
+                                "role": "tool",
+                                "tool_call_id": call.get("id", ""),
+                                "name": tool_name,
+                                "content": _tool_result_to_content(result),
+                            }
+                        )
+                    continue
+
+                content = message.get("content")
+                if content is None:
+                    content = ""
+                if isinstance(content, list):
+                    text_parts: list[str] = []
+                    for part in content:
+                        if isinstance(part, dict) and isinstance(part.get("text"), str):
+                            text_parts.append(part["text"])
+                    content = "\n".join(text_parts)
+
+                return {
+                    "output": str(content),
+                    "usage": usage,
+                }
+
+        return {
+            "output": "Reached max_turns without final assistant response.",
+            "usage": usage,
+        }
+
+    return run

agent_chaos/adapters/types.py

@@ -0,0 +1,16 @@
+from __future__ import annotations
+
+from typing import Any, Callable, Protocol
+
+
+ToolExecutor = Callable[[str, dict[str, Any] | None], Any]
+
+
+class AgentAdapter(Protocol):
+    def __call__(
+        self,
+        prompt: str,
+        tool_executor: ToolExecutor,
+        context: dict[str, Any] | None = None,
+    ) -> Any:
+        ...

agent_chaos/budget_proxy/__init__.py

@@ -0,0 +1,5 @@
+from .app import create_app
+from .config import load_config
+from .policy import PolicyLoadError, load_policy
+
+__all__ = ["PolicyLoadError", "create_app", "load_config", "load_policy"]