agentops-toolkit 0.1.0__py3-none-any.whl

agentops_toolkit/__init__.py

@@ -0,0 +1,3 @@
+"""AgentOps Toolkit — Evaluate, trace, and monitor AI agents."""
+
+__version__ = "0.1.0"

agentops_toolkit/adapters/__init__.py

@@ -0,0 +1,30 @@
+"""Framework adapters — bridge between AgentOps and agent frameworks.
+
+SPEC-004: FrameworkAdapter protocol, AdapterRegistry, auto-detection.
+
+Importing this package auto-registers all built-in adapters.
+"""
+
+from agentops_toolkit.adapters.registry import (
+    AdapterCapabilities,
+    AdapterRegistry,
+    AgentDiscovery,
+    AgentOutput,
+    DiscoveredAgent,
+    FrameworkAdapter,
+)
+
+# Auto-register built-in adapters on import
+from agentops_toolkit.adapters import semantic_kernel as _sk  # noqa: F401
+from agentops_toolkit.adapters import autogen as _ag  # noqa: F401
+from agentops_toolkit.adapters import agent_service as _as  # noqa: F401
+from agentops_toolkit.adapters import generic as _gen  # noqa: F401
+
+__all__ = [
+    "AdapterCapabilities",
+    "AdapterRegistry",
+    "AgentDiscovery",
+    "AgentOutput",
+    "DiscoveredAgent",
+    "FrameworkAdapter",
+]

agentops_toolkit/adapters/agent_service.py

@@ -0,0 +1,79 @@
+"""Azure AI Agent Service adapter — pull agent defs from Foundry.
+
+SPEC-004 §4.3, FR-083: AgentServiceAdapter.
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import Any
+
+from agentops_toolkit.adapters.registry import (
+    AdapterCapabilities,
+    AdapterRegistry,
+    AgentDiscovery,
+    AgentOutput,
+    DiscoveredAgent,
+)
+
+logger = logging.getLogger(__name__)
+
+
+class AgentServiceAdapter:
+    """Adapter for Azure AI Agent Service (SPEC-004 §4.3).
+
+    Creates threads, sends messages, polls for completion,
+    extracts tool calls and file search results.
+    """
+
+    def __init__(self, config: dict[str, Any]) -> None:
+        self._config = config
+
+    @property
+    def name(self) -> str:
+        return "agent-service"
+
+    @property
+    def framework_version(self) -> str:
+        try:
+            import azure.ai.projects
+            return getattr(azure.ai.projects, "__version__", "unknown")
+        except ImportError:
+            return "not-installed"
+
+    async def setup(self, config: dict[str, Any]) -> None:
+        agent_id = config.get("agent_id")
+        logger.info("Agent Service adapter setup: agent_id=%s", agent_id)
+
+    async def teardown(self) -> None:
+        pass
+
+    async def invoke(self, query: str, context: str | None = None) -> AgentOutput:
+        logger.info("Agent Service invoke: query=%s", query[:50])
+        return AgentOutput(
+            response="[Agent Service adapter — production creates thread + polls]",
+            metadata={"framework": "agent-service"},
+        )
+
+    async def discover(self, config: dict[str, Any]) -> AgentDiscovery:
+        return AgentDiscovery(
+            framework="agent-service",
+            framework_version=self.framework_version,
+            agents=[DiscoveredAgent(
+                name=config.get("agent_id", "agent"),
+                entry_point="foundry-hosted",
+                agent_type="single",
+            )],
+        )
+
+    def get_capabilities(self) -> AdapterCapabilities:
+        return AdapterCapabilities(
+            captures_tool_calls=True,
+            captures_token_usage=True,
+        )
+
+    def supports_streaming(self) -> bool:
+        return False
+
+
+AdapterRegistry.register("agent-service", AgentServiceAdapter)

agentops_toolkit/adapters/autogen.py

@@ -0,0 +1,81 @@
+"""AutoGen adapter — hook ConversableAgent message streams.
+
+SPEC-004 §4.2, FR-082: AutoGenAdapter.
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import Any
+
+from agentops_toolkit.adapters.registry import (
+    AdapterCapabilities,
+    AdapterRegistry,
+    AgentDiscovery,
+    AgentOutput,
+    DiscoveredAgent,
+)
+
+logger = logging.getLogger(__name__)
+
+
+class AutoGenAdapter:
+    """Adapter for AutoGen multi-agent systems (SPEC-004 §4.2).
+
+    Hooks into ConversableAgent message streams, captures inter-agent
+    messages as ConversationTurn objects.
+    """
+
+    def __init__(self, config: dict[str, Any]) -> None:
+        self._config = config
+        self._captured_turns: list[dict] = []
+
+    @property
+    def name(self) -> str:
+        return "autogen"
+
+    @property
+    def framework_version(self) -> str:
+        try:
+            import autogen
+            return getattr(autogen, "__version__", "unknown")
+        except ImportError:
+            return "not-installed"
+
+    async def setup(self, config: dict[str, Any]) -> None:
+        entry_point = config.get("entry_point", "")
+        logger.info("AutoGen adapter setup: entry=%s", entry_point)
+
+    async def teardown(self) -> None:
+        self._captured_turns.clear()
+
+    async def invoke(self, query: str, context: str | None = None) -> AgentOutput:
+        logger.info("AutoGen invoke: query=%s", query[:50])
+        return AgentOutput(
+            response="[AutoGen adapter — production implementation calls team.run()]",
+            conversation_turns=self._captured_turns or None,
+            metadata={"framework": "autogen"},
+        )
+
+    async def discover(self, config: dict[str, Any]) -> AgentDiscovery:
+        return AgentDiscovery(
+            framework="autogen",
+            framework_version=self.framework_version,
+            agents=[DiscoveredAgent(
+                name="team",
+                entry_point=config.get("entry_point", ""),
+                agent_type="multi-agent",
+            )],
+        )
+
+    def get_capabilities(self) -> AdapterCapabilities:
+        return AdapterCapabilities(
+            captures_tool_calls=True,
+            captures_conversation_turns=True,
+        )
+
+    def supports_streaming(self) -> bool:
+        return False
+
+
+AdapterRegistry.register("autogen", AutoGenAdapter)

agentops_toolkit/adapters/generic.py

@@ -0,0 +1,121 @@
+"""Generic adapter — works with any framework via callable or HTTP endpoint.
+
+SPEC-004 §4.4, FR-084: GenericAdapter with @agentops.trace support.
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import Any
+
+from agentops_toolkit.adapters.registry import (
+    AdapterCapabilities,
+    AdapterRegistry,
+    AgentDiscovery,
+    AgentOutput,
+    DiscoveredAgent,
+)
+
+logger = logging.getLogger(__name__)
+
+
+class GenericAdapter:
+    """Generic adapter for any framework (SPEC-004 §4.4).
+
+    Supports two modes:
+    - Python callable: auto-detected function signature
+    - HTTP endpoint: POST with JSON request/response
+    """
+
+    def __init__(self, config: dict[str, Any]) -> None:
+        self._config = config
+
+    @property
+    def name(self) -> str:
+        return "custom"
+
+    @property
+    def framework_version(self) -> str:
+        return "n/a"
+
+    async def setup(self, config: dict[str, Any]) -> None:
+        entry_point = config.get("entry_point", "")
+        logger.info("Generic adapter setup: entry=%s", entry_point)
+
+    async def teardown(self) -> None:
+        pass
+
+    async def invoke(self, query: str, context: str | None = None) -> AgentOutput:
+        entry_point = self._config.get("entry_point", "")
+
+        if entry_point.startswith("http://") or entry_point.startswith("https://"):
+            return await self._invoke_http(query, context, entry_point)
+        else:
+            return await self._invoke_callable(query, context, entry_point)
+
+    async def _invoke_http(self, query: str, context: str | None, url: str) -> AgentOutput:
+        """Invoke agent via HTTP POST."""
+        import httpx
+
+        payload: dict[str, Any] = {"query": query}
+        if context:
+            payload["context"] = context
+
+        async with httpx.AsyncClient() as client:
+            resp = await client.post(url, json=payload)
+            resp.raise_for_status()
+            data = resp.json()
+
+        return AgentOutput(
+            response=data.get("response", str(data)),
+            metadata={"source": "http", "url": url},
+        )
+
+    async def _invoke_callable(self, query: str, context: str | None, entry_point: str) -> AgentOutput:
+        """Invoke agent via Python callable."""
+        import importlib
+
+        if ":" in entry_point:
+            module_path, func_name = entry_point.rsplit(":", 1)
+        else:
+            module_path = entry_point.replace(".py", "").replace("/", ".").replace("\\", ".")
+            func_name = "agent_fn"
+
+        try:
+            module = importlib.import_module(module_path)
+            func = getattr(module, func_name)
+            if context:
+                result = func(query, context)
+            else:
+                result = func(query)
+
+            if hasattr(result, "__await__"):
+                result = await result
+
+            return AgentOutput(
+                response=str(result),
+                metadata={"source": "callable", "entry_point": entry_point},
+            )
+        except Exception as e:
+            return AgentOutput(
+                response="",
+                metadata={"error": str(e)},
+            )
+
+    async def discover(self, config: dict[str, Any]) -> AgentDiscovery:
+        return AgentDiscovery(
+            framework="custom",
+            agents=[DiscoveredAgent(
+                name="agent",
+                entry_point=config.get("entry_point", ""),
+            )],
+        )
+
+    def get_capabilities(self) -> AdapterCapabilities:
+        return AdapterCapabilities()
+
+    def supports_streaming(self) -> bool:
+        return False
+
+
+AdapterRegistry.register("custom", GenericAdapter)

agentops_toolkit/adapters/registry.py

@@ -0,0 +1,158 @@
+"""Adapter protocol and registry.
+
+SPEC-004 §2–3: FrameworkAdapter protocol, AdapterRegistry with auto-detection.
+"""
+
+from __future__ import annotations
+
+import logging
+from pathlib import Path
+from typing import Any, Protocol, runtime_checkable
+
+from pydantic import BaseModel, Field
+
+logger = logging.getLogger(__name__)
+
+
+# ── Supporting types ──
+
+
+class AdapterCapabilities(BaseModel):
+    """What an adapter can capture (SPEC-004 §2)."""
+
+    captures_response: bool = True
+    captures_tool_calls: bool = False
+    captures_conversation_turns: bool = False
+    captures_planner_steps: bool = False
+    captures_token_usage: bool = False
+    supports_streaming: bool = False
+
+
+class DiscoveredAgent(BaseModel):
+    """A detected agent entry point (SPEC-004 §2)."""
+
+    name: str
+    entry_point: str
+    agent_type: str = "single"  # "single" | "multi-agent" | "rag"
+    tools: list[str] = Field(default_factory=list)
+    plugins: list[str] = Field(default_factory=list)
+
+
+class AgentDiscovery(BaseModel):
+    """Results of agent discovery in a project (SPEC-004 §2)."""
+
+    agents: list[DiscoveredAgent] = Field(default_factory=list)
+    framework: str = ""
+    framework_version: str = ""
+    warnings: list[str] = Field(default_factory=list)
+
+
+class AgentOutput(BaseModel):
+    """Captured output from an agent invocation (SPEC-003 §2)."""
+
+    response: str
+    tool_calls: list[Any] | None = None
+    conversation_turns: list[Any] | None = None
+    metadata: dict[str, Any] = Field(default_factory=dict)
+
+
+# ── Protocol ──
+
+
+@runtime_checkable
+class FrameworkAdapter(Protocol):
+    """Protocol that all framework adapters must satisfy (SPEC-004 §2)."""
+
+    @property
+    def name(self) -> str: ...
+
+    @property
+    def framework_version(self) -> str: ...
+
+    async def setup(self, config: dict[str, Any]) -> None: ...
+
+    async def teardown(self) -> None: ...
+
+    async def invoke(self, query: str, context: str | None = None) -> AgentOutput: ...
+
+    async def discover(self, config: dict[str, Any]) -> AgentDiscovery: ...
+
+    def get_capabilities(self) -> AdapterCapabilities: ...
+
+    def supports_streaming(self) -> bool: ...
+
+
+# ── Registry ──
+
+
+# Package name → adapter name detection map
+_DETECTION_MAP: dict[str, str] = {
+    "semantic_kernel": "semantic-kernel",
+    "semantic-kernel": "semantic-kernel",
+    "autogen": "autogen",
+    "autogen-agentchat": "autogen",
+    "pyautogen": "autogen",
+    "azure-ai-projects": "agent-service",
+    "azure.ai.projects": "agent-service",
+}
+
+
+class AdapterRegistry:
+    """Registry for framework adapters with auto-detection (SPEC-004 §3).
+
+    Usage::
+
+        registry = AdapterRegistry()
+        detected = registry.auto_detect(Path("."))
+        adapter = registry.get("semantic-kernel", config={})
+    """
+
+    _adapters: dict[str, type] = {}
+
+    @classmethod
+    def register(cls, name: str, adapter_class: type) -> None:
+        """Register an adapter class by name."""
+        cls._adapters[name] = adapter_class
+        logger.debug("Adapter registered: %s", name)
+
+    @classmethod
+    def get(cls, name: str, config: dict[str, Any] | None = None) -> Any:
+        """Resolve an adapter by name.
+
+        Raises:
+            KeyError: If the adapter is not registered.
+        """
+        if name not in cls._adapters:
+            available = sorted(cls._adapters.keys()) or ["(none registered)"]
+            raise KeyError(
+                f"Adapter '{name}' not found. Available: {', '.join(available)}"
+            )
+        return cls._adapters[name](config or {})
+
+    @classmethod
+    def list_registered(cls) -> list[str]:
+        """Return sorted list of registered adapter names."""
+        return sorted(cls._adapters.keys())
+
+    @classmethod
+    def auto_detect(cls, project_root: Path) -> str | None:
+        """Scan project dependencies to detect which framework is in use (SPEC-004 §3, FR-085).
+
+        Checks in order:
+          1. pyproject.toml
+          2. requirements.txt
+          3. setup.cfg
+        """
+        files_to_check = ["pyproject.toml", "requirements.txt", "setup.cfg"]
+
+        for fname in files_to_check:
+            fpath = project_root / fname
+            if fpath.exists():
+                content = fpath.read_text(encoding="utf-8").lower()
+                for package, adapter_name in _DETECTION_MAP.items():
+                    if package.lower() in content:
+                        logger.info("Auto-detected framework: %s (from %s)", adapter_name, fname)
+                        return adapter_name
+
+        logger.debug("No framework detected in %s", project_root)
+        return None

agentops_toolkit/adapters/semantic_kernel.py

@@ -0,0 +1,92 @@
+"""Semantic Kernel adapter — auto-discover plugins, capture kernel I/O.
+
+SPEC-004 §4.1, FR-081: SemanticKernelAdapter.
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import Any
+
+from agentops_toolkit.adapters.registry import (
+    AdapterCapabilities,
+    AdapterRegistry,
+    AgentDiscovery,
+    AgentOutput,
+    DiscoveredAgent,
+)
+
+logger = logging.getLogger(__name__)
+
+
+class SemanticKernelAdapter:
+    """Adapter for Microsoft Semantic Kernel agents (SPEC-004 §4.1).
+
+    Auto-discovers kernel plugins, captures I/O via ChatCompletionService,
+    and hooks planner steps via FunctionInvocationFilter.
+    """
+
+    def __init__(self, config: dict[str, Any]) -> None:
+        self._config = config
+        self._kernel = None
+        self._captured_tool_calls: list[dict] = []
+
+    @property
+    def name(self) -> str:
+        return "semantic-kernel"
+
+    @property
+    def framework_version(self) -> str:
+        try:
+            import semantic_kernel
+            return getattr(semantic_kernel, "__version__", "unknown")
+        except ImportError:
+            return "not-installed"
+
+    async def setup(self, config: dict[str, Any]) -> None:
+        """Import entry point, locate Kernel, install filters."""
+        entry_point = config.get("entry_point", "")
+        kernel_var = config.get("kernel_var", "kernel")
+        logger.info("SK adapter setup: entry=%s, kernel_var=%s", entry_point, kernel_var)
+        # Production: import module, find kernel, install filters
+        # See SPEC-004 §4.1 for full implementation
+
+    async def teardown(self) -> None:
+        self._kernel = None
+        self._captured_tool_calls.clear()
+
+    async def invoke(self, query: str, context: str | None = None) -> AgentOutput:
+        """Invoke the SK agent with a query."""
+        logger.info("SK invoke: query=%s", query[:50])
+        # Production: build ChatHistory, call ChatCompletionService, capture result
+        return AgentOutput(
+            response="[SK adapter — production implementation calls kernel]",
+            tool_calls=self._captured_tool_calls or None,
+            metadata={"framework": "semantic-kernel"},
+        )
+
+    async def discover(self, config: dict[str, Any]) -> AgentDiscovery:
+        """Discover SK agents: scan for Kernel instances and plugins."""
+        return AgentDiscovery(
+            framework="semantic-kernel",
+            framework_version=self.framework_version,
+            agents=[DiscoveredAgent(
+                name="kernel",
+                entry_point=config.get("entry_point", ""),
+                agent_type="single",
+            )],
+        )
+
+    def get_capabilities(self) -> AdapterCapabilities:
+        return AdapterCapabilities(
+            captures_tool_calls=True,
+            captures_planner_steps=True,
+            captures_token_usage=True,
+        )
+
+    def supports_streaming(self) -> bool:
+        return True
+
+
+# Auto-register
+AdapterRegistry.register("semantic-kernel", SemanticKernelAdapter)

agentops_toolkit/bundles/agent_quality.yaml

@@ -0,0 +1,22 @@
+# agent_quality — Quality for single tool-using agents
+# SPEC-001 §3.4
+
+name: agent_quality
+description: "Quality evaluators for tool-using agents"
+use_case: agent
+evaluators:
+  - name: groundedness
+    type: foundry
+  - name: relevance
+    type: foundry
+  - name: coherence
+    type: foundry
+  - name: tool_call_accuracy
+    type: foundry
+thresholds:
+  groundedness: 3.0
+  relevance: 3.0
+  coherence: 3.0
+  tool_call_accuracy: 0.7
+tags: [agent, quality, tools]
+builtin: true

agentops_toolkit/bundles/agent_safety.yaml

@@ -0,0 +1,20 @@
+# agent_safety — Safety for agents
+# SPEC-001 §3.4
+
+name: agent_safety
+description: "Safety evaluators for agents"
+use_case: agent
+evaluators:
+  - name: hate_unfairness
+    type: foundry
+  - name: sexual
+    type: foundry
+  - name: violence
+    type: foundry
+  - name: self_harm
+    type: foundry
+  - name: jailbreak
+    type: foundry
+thresholds: {}
+tags: [agent, safety]
+builtin: true

agentops_toolkit/bundles/custom.yaml

@@ -0,0 +1,9 @@
+# custom — Empty template for user customization
+# SPEC-001 §3.4
+
+name: custom
+description: "Empty template — add your own evaluators"
+evaluators: []
+thresholds: {}
+tags: [custom]
+builtin: true

agentops_toolkit/bundles/multi_agent_quality.yaml

@@ -0,0 +1,25 @@
+# multi_agent_quality — Quality for orchestrated multi-agent systems
+# SPEC-001 §3.4
+
+name: multi_agent_quality
+description: "Quality evaluators for orchestrated multi-agent systems"
+use_case: multi-agent
+evaluators:
+  - name: groundedness
+    type: foundry
+  - name: relevance
+    type: foundry
+  - name: coherence
+    type: foundry
+  - name: task_completion
+    type: foundry
+  - name: handoff_quality
+    type: foundry
+thresholds:
+  groundedness: 3.0
+  relevance: 3.0
+  coherence: 3.0
+  task_completion: 3.0
+  handoff_quality: 3.0
+tags: [multi-agent, quality, orchestration]
+builtin: true

agentops_toolkit/bundles/rag_agentic_retrieval.yaml

@@ -0,0 +1,23 @@
+# rag_agentic_retrieval — RAG + agentic retrieval pipeline
+# SPEC-008 §4.1
+
+name: rag_agentic_retrieval
+description: "Evaluates the full agentic retrieval pipeline including query decomposition"
+use_case: rag
+evaluators:
+  - name: groundedness
+    type: foundry
+  - name: relevance
+    type: foundry
+  - name: sub_query_quality
+    type: custom
+  - name: source_coverage
+    type: custom
+  - name: citation_accuracy
+    type: custom
+thresholds:
+  groundedness: 4.0
+  relevance: 4.0
+  source_coverage: 0.8
+tags: [rag, agentic-retrieval, query-decomposition]
+builtin: true