k-eval 0.3.0__py3-none-any.whl

k_eval/agent/domain/agent.py

@@ -0,0 +1,14 @@
+"""Agent Protocol — structural interface for all agent implementations."""
+
+from typing import Protocol
+
+from k_eval.agent.domain.result import AgentResult
+
+
+class Agent(Protocol):
+    """Structural interface satisfied by any agent implementation.
+
+    Each instance is constructed once per (condition, sample) evaluation.
+    """
+
+    async def ask(self, question: str) -> AgentResult: ...

k_eval/agent/domain/factory.py

@@ -0,0 +1,18 @@
+"""AgentFactory Protocol — structural interface for constructing Agent instances."""
+
+from typing import Protocol
+
+from k_eval.agent.domain.agent import Agent
+from k_eval.config.domain.condition_mcp_server import ConditionMcpServer
+
+
+class AgentFactory(Protocol):
+    """Constructs a new Agent instance for a given (condition, sample) pair."""
+
+    def create(
+        self,
+        condition: str,
+        sample_idx: str,
+        system_prompt: str,
+        mcp_servers: list[ConditionMcpServer],
+    ) -> Agent: ...

k_eval/agent/domain/observer.py

@@ -0,0 +1,27 @@
+"""AgentObserver port — domain events emitted during agent invocations."""
+
+from typing import Protocol
+
+
+class AgentObserver(Protocol):
+    """Observer port for agent domain events.
+
+    Implementations may log to structlog, record for tests, or emit metrics.
+    """
+
+    def agent_invocation_started(
+        self, condition: str, sample_idx: str, model: str
+    ) -> None: ...
+
+    def agent_invocation_completed(
+        self,
+        condition: str,
+        sample_idx: str,
+        duration_ms: int,
+        num_turns: int,
+        cost_usd: float | None,
+    ) -> None: ...
+
+    def agent_invocation_failed(
+        self, condition: str, sample_idx: str, reason: str
+    ) -> None: ...

k_eval/agent/domain/result.py

@@ -0,0 +1,18 @@
+"""AgentResult value object — the outcome of a single agent invocation."""
+
+from pydantic import BaseModel, ConfigDict
+
+from k_eval.agent.domain.usage import UsageMetrics
+
+
+class AgentResult(BaseModel, frozen=True):
+    """Immutable value object capturing all outcome data from one agent invocation."""
+
+    model_config = ConfigDict(frozen=True)
+
+    response: str
+    cost_usd: float | None
+    duration_ms: int
+    duration_api_ms: int
+    num_turns: int
+    usage: UsageMetrics | None

k_eval/agent/domain/usage.py

@@ -0,0 +1,12 @@
+"""UsageMetrics value object — token usage from an agent invocation."""
+
+from pydantic import BaseModel, ConfigDict
+
+
+class UsageMetrics(BaseModel, frozen=True):
+    """Immutable value object capturing token usage from a single agent invocation."""
+
+    model_config = ConfigDict(frozen=True)
+
+    input_tokens: int | None
+    output_tokens: int | None

k_eval/agent/infrastructure/claude_sdk.py

@@ -0,0 +1,222 @@
+"""ClaudeAgentSDKAgent — agent implementation using the Claude Agent SDK."""
+
+from typing import Any
+
+from claude_agent_sdk import query
+from claude_agent_sdk._errors import ClaudeSDKError
+from claude_agent_sdk.types import (
+    ClaudeAgentOptions,
+    McpHttpServerConfig,
+    McpSdkServerConfig,
+    McpSSEServerConfig,
+    McpStdioServerConfig,
+    ResultMessage,
+)
+
+from k_eval.agent.domain.observer import AgentObserver
+from k_eval.agent.domain.result import AgentResult
+from k_eval.agent.domain.usage import UsageMetrics
+from k_eval.agent.infrastructure.errors import AgentInvocationError
+from k_eval.config.domain.agent import AgentConfig
+from k_eval.config.domain.condition_mcp_server import ConditionMcpServer
+from k_eval.config.domain.mcp_server import HttpMcpServer, SseMcpServer, StdioMcpServer
+
+type McpServerConfigMap = dict[
+    str,
+    McpStdioServerConfig
+    | McpSSEServerConfig
+    | McpHttpServerConfig
+    | McpSdkServerConfig,
+]
+
+
+class ClaudeAgentSDKAgent:
+    """Agent implementation that delegates to the Claude Agent SDK.
+
+    One instance is constructed per (condition, sample) evaluation run.
+    The condition and sample_idx are injected at construction time so that
+    observer events carry full context without polluting the ask() signature.
+    """
+
+    def __init__(
+        self,
+        config: AgentConfig,
+        condition: str,
+        sample_idx: str,
+        system_prompt: str,
+        mcp_servers: list[ConditionMcpServer],
+        observer: AgentObserver,
+    ) -> None:
+        self._config = config
+        self._condition = condition
+        self._sample_idx = sample_idx
+        self._system_prompt = system_prompt
+        self._mcp_servers = mcp_servers
+        self._observer = observer
+
+    async def ask(self, question: str) -> AgentResult:
+        """Invoke the agent with a question and return the structured result.
+
+        Opens a new SDK session per call — correct for independent eval samples.
+
+        Raises:
+            AgentInvocationError: if the SDK raises, the agent returns an error,
+                or no ResultMessage is present in the response stream.
+        """
+        self._observer.agent_invocation_started(
+            condition=self._condition,
+            sample_idx=self._sample_idx,
+            model=self._config.model,
+        )
+
+        try:
+            options = ClaudeAgentOptions(
+                model=self._config.model,
+                system_prompt=self._system_prompt,
+                mcp_servers=self._build_mcp_servers(),
+                disallowed_tools=self._build_disallowed_tools(),
+                permission_mode="bypassPermissions",
+                setting_sources=[],
+            )
+
+            result_message = await self._collect_result(
+                prompt=question, options=options
+            )
+        except AgentInvocationError as exc:
+            reason = str(exc).removeprefix("Failed to invoke agent: ")
+            self._observer.agent_invocation_failed(
+                condition=self._condition,
+                sample_idx=self._sample_idx,
+                reason=reason,
+            )
+            raise
+
+        self._observer.agent_invocation_completed(
+            condition=self._condition,
+            sample_idx=self._sample_idx,
+            duration_ms=result_message.duration_ms,
+            num_turns=result_message.num_turns,
+            cost_usd=result_message.total_cost_usd,
+        )
+
+        assert result_message.result is not None  # guaranteed by _collect_result
+        return AgentResult(
+            response=result_message.result,
+            cost_usd=result_message.total_cost_usd,
+            duration_ms=result_message.duration_ms,
+            duration_api_ms=result_message.duration_api_ms,
+            num_turns=result_message.num_turns,
+            usage=self._map_usage(raw=result_message.usage),
+        )
+
+    async def _collect_result(
+        self, prompt: str, options: ClaudeAgentOptions
+    ) -> ResultMessage:
+        """Run the SDK query and extract the single ResultMessage.
+
+        Raises:
+            AgentInvocationError: on SDK errors or missing/error ResultMessage.
+        """
+        result_message: ResultMessage | None = None
+
+        try:
+            async for message in query(prompt=prompt, options=options):
+                if isinstance(message, ResultMessage):
+                    result_message = message
+        except ClaudeSDKError as exc:
+            raise AgentInvocationError(reason=str(exc), retriable=True) from exc
+        except Exception as exc:
+            # The SDK internally raises a bare Exception (not ClaudeSDKError) when
+            # its message reader encounters a fatal error (e.g. subprocess exit).
+            raise AgentInvocationError(reason=str(exc), retriable=True) from exc
+
+        if result_message is None:
+            raise AgentInvocationError(reason="no ResultMessage in response stream")
+
+        if result_message.is_error:
+            raise AgentInvocationError(
+                reason=f"agent returned error response: {result_message.result}"
+            )
+
+        if result_message.result is None:
+            raise AgentInvocationError(reason="ResultMessage has no result text")
+
+        return result_message
+
+    def _build_mcp_servers(self) -> McpServerConfigMap:
+        """Convert ConditionMcpServer list to the SDK's TypedDict format."""
+        servers: McpServerConfigMap = {}
+
+        for server in self._mcp_servers:
+            config = server.config
+
+            if isinstance(config, StdioMcpServer):
+                servers[server.name] = self._build_stdio_server(config=config)
+            elif isinstance(config, SseMcpServer):
+                servers[server.name] = self._build_sse_server(config=config)
+            elif isinstance(config, HttpMcpServer):
+                servers[server.name] = self._build_http_server(config=config)
+            else:
+                raise AgentInvocationError(
+                    reason=f"unsupported MCP server type for server '{server.name}'"
+                )
+
+        return servers
+
+    def _build_stdio_server(self, config: StdioMcpServer) -> McpStdioServerConfig:
+        """Build a McpStdioServerConfig TypedDict from a StdioMcpServer model."""
+        server: McpStdioServerConfig = McpStdioServerConfig(command=config.command)
+        if config.args:
+            server["args"] = list(config.args)
+        if config.env:
+            server["env"] = dict(config.env)
+        return server
+
+    def _build_sse_server(self, config: SseMcpServer) -> McpSSEServerConfig:
+        """Build a McpSSEServerConfig TypedDict from a SseMcpServer model."""
+        server: McpSSEServerConfig = McpSSEServerConfig(type="sse", url=config.url)
+        if config.headers:
+            server["headers"] = dict(config.headers)
+        return server
+
+    def _build_http_server(self, config: HttpMcpServer) -> McpHttpServerConfig:
+        """Build a McpHttpServerConfig TypedDict from an HttpMcpServer model."""
+        server: McpHttpServerConfig = McpHttpServerConfig(type="http", url=config.url)
+        if config.headers:
+            server["headers"] = dict(config.headers)
+        return server
+
+    def _build_disallowed_tools(self) -> list[str]:
+        """Build the disallowed tools list — all Claude built-in tools.
+
+        allowed_tools alone does not remove built-in tools from the agent's
+        context; it only controls approval requirements. Explicitly disallowing
+        all built-in tools ensures the agent cannot use web search, file I/O,
+        or any other built-in capability regardless of permission_mode.
+        """
+        return [
+            "Bash",
+            "Edit",
+            "Glob",
+            "Grep",
+            "LS",
+            "MultiEdit",
+            "NotebookEdit",
+            "NotebookRead",
+            "Read",
+            "Task",
+            "TodoRead",
+            "TodoWrite",
+            "WebFetch",
+            "WebSearch",
+            "Write",
+        ]
+
+    def _map_usage(self, raw: dict[str, Any] | None) -> UsageMetrics | None:
+        """Map the SDK's raw usage dict to a typed UsageMetrics value object."""
+        if raw is None:
+            return None
+        return UsageMetrics(
+            input_tokens=raw.get("input_tokens"),
+            output_tokens=raw.get("output_tokens"),
+        )

k_eval/agent/infrastructure/errors.py

@@ -0,0 +1,19 @@
+"""Error types raised by agent infrastructure."""
+
+from k_eval.core.errors import KEvalError
+
+
+class AgentInvocationError(KEvalError):
+    """Raised when the agent cannot be invoked or returns an error response."""
+
+    def __init__(self, reason: str, retriable: bool = False) -> None:
+        super().__init__(f"Failed to invoke agent: {reason}", retriable=retriable)
+
+
+class AgentTypeNotSupportedError(KEvalError):
+    """Raised when the agent type specified in config is not a known agent type."""
+
+    def __init__(self, agent_type: str) -> None:
+        super().__init__(
+            f"Failed to create agent factory: unsupported agent type '{agent_type}'"
+        )

k_eval/agent/infrastructure/factory.py

@@ -0,0 +1,32 @@
+"""ClaudeAgentSDKAgentFactory — constructs ClaudeAgentSDKAgent instances."""
+
+from k_eval.agent.domain.agent import Agent
+from k_eval.agent.domain.observer import AgentObserver
+from k_eval.agent.infrastructure.claude_sdk import ClaudeAgentSDKAgent
+from k_eval.config.domain.agent import AgentConfig
+from k_eval.config.domain.condition_mcp_server import ConditionMcpServer
+
+
+class ClaudeAgentSDKAgentFactory:
+    """Creates ClaudeAgentSDKAgent instances configured for a given condition and sample."""
+
+    def __init__(self, config: AgentConfig, observer: AgentObserver) -> None:
+        self._config = config
+        self._observer = observer
+
+    def create(
+        self,
+        condition: str,
+        sample_idx: str,
+        system_prompt: str,
+        mcp_servers: list[ConditionMcpServer],
+    ) -> Agent:
+        """Construct a new ClaudeAgentSDKAgent for the given condition and sample."""
+        return ClaudeAgentSDKAgent(
+            config=self._config,
+            condition=condition,
+            sample_idx=sample_idx,
+            system_prompt=system_prompt,
+            mcp_servers=mcp_servers,
+            observer=self._observer,
+        )

k_eval/agent/infrastructure/observer.py

@@ -0,0 +1,50 @@
+"""Structlog implementation of the AgentObserver port."""
+
+import structlog
+
+
+class StructlogAgentObserver:
+    """Delegates agent domain events to structlog.
+
+    Satisfies the AgentObserver protocol structurally.
+    """
+
+    def __init__(self) -> None:
+        self._log = structlog.get_logger()
+
+    def agent_invocation_started(
+        self, condition: str, sample_idx: str, model: str
+    ) -> None:
+        self._log.info(
+            "agent.invocation_started",
+            condition=condition,
+            sample_idx=sample_idx,
+            model=model,
+        )
+
+    def agent_invocation_completed(
+        self,
+        condition: str,
+        sample_idx: str,
+        duration_ms: int,
+        num_turns: int,
+        cost_usd: float | None,
+    ) -> None:
+        self._log.info(
+            "agent.invocation_completed",
+            condition=condition,
+            sample_idx=sample_idx,
+            duration_ms=duration_ms,
+            num_turns=num_turns,
+            cost_usd=cost_usd,
+        )
+
+    def agent_invocation_failed(
+        self, condition: str, sample_idx: str, reason: str
+    ) -> None:
+        self._log.error(
+            "agent.invocation_failed",
+            condition=condition,
+            sample_idx=sample_idx,
+            reason=reason,
+        )

k_eval/agent/infrastructure/registry.py

@@ -0,0 +1,21 @@
+"""AgentFactoryRegistry — maps AgentConfig.type to the correct AgentFactory."""
+
+from k_eval.agent.domain.factory import AgentFactory
+from k_eval.agent.domain.observer import AgentObserver
+from k_eval.agent.infrastructure.errors import AgentTypeNotSupportedError
+from k_eval.agent.infrastructure.factory import ClaudeAgentSDKAgentFactory
+from k_eval.config.domain.agent import AgentConfig
+
+_SUPPORTED_TYPE = "claude_code_sdk"
+
+
+def create_agent_factory(config: AgentConfig, observer: AgentObserver) -> AgentFactory:
+    """Return the appropriate AgentFactory for the given AgentConfig.
+
+    Raises:
+        AgentTypeNotSupportedError: if config.type is not a known agent type.
+    """
+    if config.type == _SUPPORTED_TYPE:
+        return ClaudeAgentSDKAgentFactory(config=config, observer=observer)
+
+    raise AgentTypeNotSupportedError(agent_type=config.type)