skillware-bench 0.3.2__py3-none-any.whl

skillware_bench/core/adapters/anthropic_adapter.py

@@ -0,0 +1,166 @@
+# Copyright (c) 2026 ARPA Hellenic Logical Systems
+# SPDX-License-Identifier: MIT
+
+"""Anthropic model adapter (Claude 3.7 Sonnet, Claude 3.5 Haiku)."""
+
+import os
+import time
+from typing import Any, Dict, List, Optional
+
+from skillware_bench.core.adapters.base import (
+    BaseModelAdapter,
+    ModelResponse,
+    TokenUsage,
+    ToolCallRequest,
+    execute_with_retry,
+)
+
+try:
+    from skillware.core.loader import SkillLoader
+except ImportError:
+    SkillLoader = None  # type: ignore
+
+
+class AnthropicAdapter(BaseModelAdapter):
+    """Adapter for Anthropic Claude models supporting structured tool use."""
+
+    def __init__(
+        self,
+        model_name: str = "claude-3-7-sonnet-20250219",
+        api_key: Optional[str] = None,
+        base_url: Optional[str] = None,
+        **kwargs: Any,
+    ):
+        super().__init__(model_name=model_name, api_key=api_key, **kwargs)
+        self.api_key = api_key or os.environ.get("ANTHROPIC_API_KEY")
+        self.base_url = base_url or os.environ.get("ANTHROPIC_BASE_URL")
+        self._client = None
+
+    def _get_client(self):
+        if self._client is None:
+            try:
+                import anthropic
+
+                self._client = anthropic.Anthropic(
+                    api_key=self.api_key,
+                    base_url=self.base_url,
+                    timeout=self.timeout_seconds,
+                )
+            except ImportError:
+                raise ImportError(
+                    "anthropic package is required to use AnthropicAdapter. Run: pip install anthropic"
+                )
+        return self._client
+
+    def format_tools(self, tool_definitions: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+        formatted: List[Dict[str, Any]] = []
+        for tool in tool_definitions:
+            if "name" in tool and "input_schema" in tool:
+                formatted.append(tool)
+            elif "manifest" in tool and SkillLoader is not None:
+                formatted.append(SkillLoader.to_claude_tool(tool))
+            else:
+                name = tool.get("name", "unknown_tool")
+                desc = tool.get("description", "")
+                params = tool.get("parameters") or tool.get("input_schema") or {}
+                formatted.append(
+                    {
+                        "name": name,
+                        "description": desc,
+                        "input_schema": params,
+                    }
+                )
+        return formatted
+
+    def chat(
+        self,
+        messages: List[Dict[str, Any]],
+        tools: Optional[List[Dict[str, Any]]] = None,
+        system_prompt: Optional[str] = None,
+        **kwargs: Any,
+    ) -> ModelResponse:
+        client = self._get_client()
+
+        anthropic_messages: List[Dict[str, Any]] = []
+        for m in messages:
+            role = m.get("role", "user")
+            if role == "system":
+                if not system_prompt:
+                    system_prompt = m.get("content", "")
+                continue
+            anthropic_messages.append({"role": role, "content": m.get("content", "")})
+
+        payload: Dict[str, Any] = {
+            "model": self.model_name,
+            "messages": anthropic_messages,
+            "max_tokens": self.max_tokens or 2048,
+        }
+
+        if self.temperature is not None:
+            payload["temperature"] = self.temperature
+
+        if system_prompt:
+            payload["system"] = system_prompt
+
+        if tools:
+            payload["tools"] = self.format_tools(tools)
+
+        t0 = time.perf_counter()
+
+        def _call_api():
+            return client.messages.create(**payload)
+
+        try:
+            import anthropic
+
+            retry_types = (
+                anthropic.RateLimitError,
+                anthropic.APIConnectionError,
+                anthropic.InternalServerError,
+            )
+        except Exception:
+            retry_types = (Exception,)
+
+        response = execute_with_retry(_call_api, retryable_exceptions=retry_types)
+        latency_ms = (time.perf_counter() - t0) * 1000.0
+
+        content_parts = []
+        tool_calls: List[ToolCallRequest] = []
+
+        for block in response.content:
+            if getattr(block, "type", "") == "text":
+                content_parts.append(block.text)
+            elif getattr(block, "type", "") == "tool_use":
+                tool_calls.append(
+                    ToolCallRequest(
+                        name=block.name,
+                        arguments=block.input if isinstance(block.input, dict) else {},
+                        id=block.id,
+                    )
+                )
+
+        content = "\n".join(content_parts)
+
+        cached_tokens = 0
+        if hasattr(response.usage, "cache_read_input_tokens"):
+            cached_tokens += getattr(response.usage, "cache_read_input_tokens", 0) or 0
+        if hasattr(response.usage, "cache_creation_input_tokens"):
+            cached_tokens += getattr(response.usage, "cache_creation_input_tokens", 0) or 0
+
+        usage = TokenUsage(
+            prompt_tokens=response.usage.input_tokens,
+            completion_tokens=response.usage.output_tokens,
+            cached_tokens=cached_tokens,
+            total_tokens=response.usage.input_tokens + response.usage.output_tokens,
+        )
+        self.record_usage(usage)
+
+        return ModelResponse(
+            content=content,
+            tool_calls=tool_calls,
+            usage=usage,
+            latency_ms=latency_ms,
+            ttft_ms=latency_ms * 0.40,
+            model=self.model_name,
+            raw=response,
+        )

skillware_bench/core/adapters/base.py

@@ -0,0 +1,182 @@
+# Copyright (c) 2026 ARPA Hellenic Logical Systems
+# SPDX-License-Identifier: MIT
+
+"""Base model adapter interface, precision token accounting, pricing, and retry engine."""
+
+from abc import ABC, abstractmethod
+from dataclasses import dataclass, field
+import random
+import time
+from typing import Any, Callable, Dict, List, Optional, Tuple, Type
+
+
+@dataclass
+class ToolCallRequest:
+    """Represents a structured tool/function call emitted by an LLM."""
+
+    name: str
+    arguments: Dict[str, Any] = field(default_factory=dict)
+    id: str = ""
+
+
+@dataclass
+class TokenUsage:
+    """Precision token telemetry including prompt, completion, and prefix caching."""
+
+    prompt_tokens: int = 0
+    completion_tokens: int = 0
+    cached_tokens: int = 0
+    total_tokens: int = 0
+    cost_usd: float = 0.0
+
+    def __post_init__(self):
+        if self.total_tokens == 0:
+            self.total_tokens = self.prompt_tokens + self.completion_tokens
+
+
+@dataclass
+class ModelResponse:
+    """Standardized response across all LLM provider adapters."""
+
+    content: str
+    tool_calls: List[ToolCallRequest] = field(default_factory=list)
+    usage: TokenUsage = field(default_factory=TokenUsage)
+    latency_ms: float = 0.0
+    ttft_ms: float = 0.0
+    model: str = ""
+    raw: Any = None
+
+
+# Pricing matrix (USD per million tokens): (prompt, completion, cached_prompt)
+MODEL_PRICING_TABLE: Dict[str, Tuple[float, float, float]] = {
+    # OpenAI
+    "gpt-5": (5.00, 20.00, 2.50),
+    "gpt-4.5": (10.00, 40.00, 5.00),
+    "gpt-4o": (2.50, 10.00, 1.25),
+    "gpt-4o-mini": (0.15, 0.60, 0.075),
+    "o3-mini": (1.10, 4.40, 0.55),
+    "o1": (15.00, 60.00, 7.50),
+    # Anthropic
+    "claude-sonnet-5": (3.00, 15.00, 0.30),
+    "sonnet-5": (3.00, 15.00, 0.30),
+    "claude-5": (3.00, 15.00, 0.30),
+    "claude-3-7-sonnet": (3.00, 15.00, 0.30),
+    "claude-3-5-sonnet": (3.00, 15.00, 0.30),
+    "claude-3-5-haiku": (0.80, 4.00, 0.08),
+    # Google Gemini
+    "gemini-3.8-pro": (1.25, 5.00, 0.3125),
+    "gemini-3.8-flash": (0.075, 0.30, 0.01875),
+    "gemini-3.8": (0.075, 0.30, 0.01875),
+    "gemini-3-flash": (0.075, 0.30, 0.01875),
+    "gemini-2.5-pro": (1.25, 5.00, 0.3125),
+    "gemini-2.5-flash": (0.075, 0.30, 0.01875),
+    "gemini-1.5-pro": (1.25, 5.00, 0.3125),
+    "gemini-1.5-flash": (0.075, 0.30, 0.01875),
+    # Mistral
+    "mistral-large-3": (2.00, 6.00, 0.0),
+    "mistral-large": (2.00, 6.00, 0.0),
+    "codestral": (0.30, 0.90, 0.0),
+    # Local Sovereign (Zero API Cost)
+    "llama3.3": (0.0, 0.0, 0.0),
+    "llama3.2": (0.0, 0.0, 0.0),
+    "llama3": (0.0, 0.0, 0.0),
+    "qwen2.5": (0.0, 0.0, 0.0),
+}
+
+
+def calculate_cost(model_name: str, usage: TokenUsage) -> float:
+    """Compute exact USD cost based on token telemetry and model pricing tier."""
+    name_lower = model_name.lower()
+    rates: Optional[Tuple[float, float, float]] = None
+
+    for key, val in MODEL_PRICING_TABLE.items():
+        if key in name_lower:
+            rates = val
+            break
+
+    if rates is None:
+        rates = (1.00, 3.00, 0.0)
+
+    prompt_rate, completion_rate, cached_rate = rates
+    uncached_prompt = max(0, usage.prompt_tokens - usage.cached_tokens)
+
+    cost = (
+        (uncached_prompt * prompt_rate / 1_000_000.0)
+        + (usage.cached_tokens * cached_rate / 1_000_000.0)
+        + (usage.completion_tokens * completion_rate / 1_000_000.0)
+    )
+    return round(cost, 6)
+
+
+def execute_with_retry(
+    fn: Callable[[], Any],
+    max_retries: int = 4,
+    initial_delay: float = 1.0,
+    backoff_factor: float = 2.0,
+    jitter: bool = True,
+    retryable_exceptions: Tuple[Type[Exception], ...] = (Exception,),
+) -> Any:
+    """Execute callable with exponential backoff and jitter for resilient API calling."""
+    delay = initial_delay
+    last_err: Optional[Exception] = None
+
+    for attempt in range(max_retries + 1):
+        try:
+            return fn()
+        except retryable_exceptions as e:
+            last_err = e
+            if attempt == max_retries:
+                raise e
+            sleep_time = delay * (1.0 + (random.uniform(0, 0.25) if jitter else 0.0))
+            time.sleep(sleep_time)
+            delay *= backoff_factor
+
+    if last_err is not None:
+        raise last_err
+
+
+class BaseModelAdapter(ABC):
+    """Abstract base adapter for frontier and sovereign LLM providers."""
+
+    def __init__(
+        self,
+        model_name: str,
+        api_key: Optional[str] = None,
+        temperature: float = 0.0,
+        max_tokens: Optional[int] = None,
+        timeout_seconds: float = 60.0,
+        **kwargs: Any,
+    ):
+        self.model_name = model_name
+        self.api_key = api_key
+        self.temperature = temperature
+        self.max_tokens = max_tokens
+        self.timeout_seconds = timeout_seconds
+        self.extra_config = kwargs
+
+        self.cumulative_usage = TokenUsage()
+        self.cumulative_cost_usd: float = 0.0
+
+    @abstractmethod
+    def chat(
+        self,
+        messages: List[Dict[str, Any]],
+        tools: Optional[List[Dict[str, Any]]] = None,
+        system_prompt: Optional[str] = None,
+        **kwargs: Any,
+    ) -> ModelResponse:
+        """Send chat messages and optional tool definitions to the LLM."""
+        pass
+
+    def format_tools(self, tool_definitions: List[Dict[str, Any]]) -> List[Any]:
+        """Convert generic tool definitions into provider-native format."""
+        return tool_definitions
+
+    def record_usage(self, usage: TokenUsage) -> None:
+        """Accumulate token accounting and compute incremental cost."""
+        usage.cost_usd = calculate_cost(self.model_name, usage)
+        self.cumulative_usage.prompt_tokens += usage.prompt_tokens
+        self.cumulative_usage.completion_tokens += usage.completion_tokens
+        self.cumulative_usage.cached_tokens += usage.cached_tokens
+        self.cumulative_usage.total_tokens += usage.total_tokens
+        self.cumulative_cost_usd += usage.cost_usd

skillware_bench/core/adapters/factory.py

@@ -0,0 +1,140 @@
+# Copyright (c) 2026 ARPA Hellenic Logical Systems
+# SPDX-License-Identifier: MIT
+
+"""Adapter factory and Bring-Your-Own-Model (BYOM) registry."""
+
+from typing import Any, Dict, Optional, Type
+
+from skillware_bench.core.adapters.anthropic_adapter import AnthropicAdapter
+from skillware_bench.core.adapters.base import BaseModelAdapter
+from skillware_bench.core.adapters.gemini_adapter import GeminiAdapter
+from skillware_bench.core.adapters.mistral_adapter import MistralAdapter
+from skillware_bench.core.adapters.mock_adapter import MockModelAdapter
+from skillware_bench.core.adapters.ollama_adapter import OllamaAdapter
+from skillware_bench.core.adapters.openai_adapter import OpenAIAdapter
+from skillware_bench.core.adapters.openrouter_adapter import OpenRouterAdapter
+
+
+class AdapterRegistry:
+    """Registry allowing users to plug in custom model adapters for BYOM evaluation."""
+
+    _custom_adapters: Dict[str, Type[BaseModelAdapter]] = {}
+
+    @classmethod
+    def register(cls, model_prefix: str, adapter_cls: Type[BaseModelAdapter]) -> None:
+        """Register a custom adapter class for a model identifier prefix."""
+        cls._custom_adapters[model_prefix.lower()] = adapter_cls
+
+    @classmethod
+    def get_custom(cls, model_name: str) -> Optional[Type[BaseModelAdapter]]:
+        name_lower = model_name.lower()
+        for prefix, adapter_cls in cls._custom_adapters.items():
+            if name_lower.startswith(prefix):
+                return adapter_cls
+        return None
+
+
+def get_adapter(
+    model_name: str,
+    mock: bool = False,
+    adapter_path: Optional[str] = None,
+    **kwargs: Any,
+) -> BaseModelAdapter:
+    """Resolve and instantiate the appropriate model adapter."""
+    if adapter_path:
+        return load_adapter_from_file(adapter_path, model_name=model_name, **kwargs)
+
+    if mock:
+        return MockModelAdapter(model_name=model_name, **kwargs)
+
+    name_lower = model_name.lower()
+
+    # Check custom BYOM registry first
+    custom_cls = AdapterRegistry.get_custom(name_lower)
+    if custom_cls is not None:
+        return custom_cls(model_name=model_name, **kwargs)
+
+    # Mock identifiers
+    if name_lower in ("mock", "mock-agent", "golden-target", "synthetic"):
+        return MockModelAdapter(model_name=model_name, **kwargs)
+
+    # OpenRouter routing (e.g. "nvidia/...", "deepseek/...", "openrouter:model", or provider="openrouter")
+    provider = kwargs.pop("provider", None)
+    if (
+        provider == "openrouter"
+        or name_lower.startswith("openrouter:")
+        or ("/" in model_name and not name_lower.startswith("ollama:"))
+    ):
+        clean_name = model_name.split("openrouter:", 1)[-1]
+        return OpenRouterAdapter(model_name=clean_name, **kwargs)
+
+    # OpenAI family
+    if (
+        name_lower.startswith("gpt-")
+        or name_lower.startswith("gpt")
+        or name_lower.startswith("o1")
+        or name_lower.startswith("o3")
+        or name_lower.startswith("o4")
+        or name_lower.startswith("chatgpt-")
+    ):
+        return OpenAIAdapter(model_name=model_name, **kwargs)
+
+    # Anthropic Claude family
+    if (
+        name_lower.startswith("claude-")
+        or name_lower.startswith("claude")
+        or name_lower.startswith("sonnet")
+        or name_lower.startswith("haiku")
+        or name_lower.startswith("opus")
+    ):
+        return AnthropicAdapter(model_name=model_name, **kwargs)
+
+    # Google Gemini family
+    if name_lower.startswith("gemini-") or name_lower.startswith("gemini"):
+        return GeminiAdapter(model_name=model_name, **kwargs)
+
+    # Mistral AI family
+    if name_lower.startswith("mistral") or name_lower.startswith("codestral"):
+        return MistralAdapter(model_name=model_name, **kwargs)
+
+    # Ollama / Local Sovereign models
+    if (
+        name_lower.startswith("ollama:")
+        or name_lower.startswith("llama")
+        or name_lower.startswith("qwen")
+    ):
+        clean_name = model_name.split(":", 1)[-1]
+        return OllamaAdapter(model_name=clean_name, **kwargs)
+
+    # Fallback to Mock if unrecognized, or raise if mock is not desired
+    return MockModelAdapter(model_name=model_name, **kwargs)
+
+
+def load_adapter_from_file(file_path: str, **kwargs: Any) -> BaseModelAdapter:
+    """Dynamically load and instantiate a BaseModelAdapter subclass from a Python file."""
+    import importlib.util
+    import inspect
+    from pathlib import Path
+
+    path = Path(file_path).resolve()
+    if not path.exists():
+        raise FileNotFoundError(f"Adapter file not found: {file_path}")
+
+    spec = importlib.util.spec_from_file_location("custom_adapter_module", path)
+    if spec is None or spec.loader is None:
+        raise ImportError(f"Cannot load module from: {file_path}")
+
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+
+    for attr_name in dir(module):
+        attr = getattr(module, attr_name)
+        if (
+            inspect.isclass(attr)
+            and issubclass(attr, BaseModelAdapter)
+            and attr is not BaseModelAdapter
+        ):
+            model_name = kwargs.pop("model_name", "custom-byom-model")
+            return attr(model_name=model_name, **kwargs)
+
+    raise ValueError(f"No BaseModelAdapter subclass found in {file_path}")

skillware_bench/core/adapters/gemini_adapter.py

@@ -0,0 +1,179 @@
+# Copyright (c) 2026 ARPA Hellenic Logical Systems
+# SPDX-License-Identifier: MIT
+
+"""Google Gemini model adapter (Gemini 2.5 Pro, Gemini 2.5 Flash)."""
+
+import os
+import time
+from typing import Any, Dict, List, Optional
+
+from skillware_bench.core.adapters.base import (
+    BaseModelAdapter,
+    ModelResponse,
+    TokenUsage,
+    ToolCallRequest,
+    execute_with_retry,
+)
+
+try:
+    from skillware.core.loader import SkillLoader
+except ImportError:
+    SkillLoader = None  # type: ignore
+
+
+class GeminiAdapter(BaseModelAdapter):
+    """Adapter for Google Gemini models using the modern google-genai SDK."""
+
+    def __init__(
+        self,
+        model_name: str = "gemini-2.5-flash",
+        api_key: Optional[str] = None,
+        **kwargs: Any,
+    ):
+        super().__init__(model_name=model_name, api_key=api_key, **kwargs)
+        self.api_key = (
+            api_key or os.environ.get("GEMINI_API_KEY") or os.environ.get("GOOGLE_API_KEY")
+        )
+        self._client = None
+
+    def _get_client(self):
+        if self._client is None:
+            try:
+                from google import genai
+
+                self._client = genai.Client(api_key=self.api_key)
+            except ImportError:
+                raise ImportError(
+                    "google-genai package is required for GeminiAdapter. Run: pip install google-genai"
+                )
+        return self._client
+
+    def format_tools(self, tool_definitions: List[Dict[str, Any]]) -> List[Any]:
+        from google.genai import types
+
+        declarations = []
+        for tool in tool_definitions:
+            if isinstance(tool, types.Tool):
+                return [tool]
+            if "manifest" in tool and SkillLoader is not None:
+                gemini_tool = SkillLoader.to_gemini_tool(tool)
+                if isinstance(gemini_tool, types.Tool):
+                    declarations.extend(gemini_tool.function_declarations or [])
+                else:
+                    declarations.append(gemini_tool)
+            else:
+                name = tool.get("name", "unknown_tool")
+                desc = tool.get("description", "")
+                params = tool.get("parameters", {})
+                declarations.append(
+                    {
+                        "name": name,
+                        "description": desc,
+                        "parameters": params,
+                    }
+                )
+
+        if declarations:
+            return [types.Tool(function_declarations=declarations)]
+        return []
+
+    def chat(
+        self,
+        messages: List[Dict[str, Any]],
+        tools: Optional[List[Dict[str, Any]]] = None,
+        system_prompt: Optional[str] = None,
+        **kwargs: Any,
+    ) -> ModelResponse:
+        client = self._get_client()
+        from google.genai import types
+
+        # Build contents from message list
+        contents = []
+        for m in messages:
+            role = m.get("role", "user")
+            content = m.get("content", "")
+            if role == "system":
+                if not system_prompt:
+                    system_prompt = content
+                continue
+            gemini_role = "model" if role == "assistant" else "user"
+            contents.append(
+                types.Content(
+                    role=gemini_role,
+                    parts=[types.Part.from_text(text=str(content))],
+                )
+            )
+
+        config_params: Dict[str, Any] = {}
+        if system_prompt:
+            config_params["system_instruction"] = system_prompt
+        if self.temperature is not None:
+            config_params["temperature"] = self.temperature
+        if self.max_tokens:
+            config_params["max_output_tokens"] = self.max_tokens
+        if tools:
+            config_params["tools"] = self.format_tools(tools)
+
+        config = types.GenerateContentConfig(**config_params) if config_params else None
+
+        t0 = time.perf_counter()
+
+        def _call_api():
+            return client.models.generate_content(
+                model=self.model_name,
+                contents=contents,
+                config=config,
+            )
+
+        response = execute_with_retry(_call_api)
+        latency_ms = (time.perf_counter() - t0) * 1000.0
+
+        content = ""
+        tool_calls: List[ToolCallRequest] = []
+
+        if hasattr(response, "text") and response.text:
+            content = response.text
+
+        if hasattr(response, "function_calls") and response.function_calls:
+            for fc in response.function_calls:
+                args = fc.args if isinstance(fc.args, dict) else dict(fc.args or {})
+                tool_calls.append(
+                    ToolCallRequest(
+                        name=fc.name,
+                        arguments=args,
+                        id=fc.name,
+                    )
+                )
+
+        # Extract tokens from usage_metadata
+        prompt_tokens = 0
+        completion_tokens = 0
+        cached_tokens = 0
+        total_tokens = 0
+
+        if hasattr(response, "usage_metadata") and response.usage_metadata:
+            meta = response.usage_metadata
+            prompt_tokens = getattr(meta, "prompt_token_count", 0) or 0
+            completion_tokens = getattr(meta, "candidates_token_count", 0) or 0
+            cached_tokens = getattr(meta, "cached_content_token_count", 0) or 0
+            total_tokens = getattr(meta, "total_token_count", 0) or (
+                prompt_tokens + completion_tokens
+            )
+
+        usage = TokenUsage(
+            prompt_tokens=prompt_tokens,
+            completion_tokens=completion_tokens,
+            cached_tokens=cached_tokens,
+            total_tokens=total_tokens,
+        )
+        self.record_usage(usage)
+
+        return ModelResponse(
+            content=content,
+            tool_calls=tool_calls,
+            usage=usage,
+            latency_ms=latency_ms,
+            ttft_ms=latency_ms * 0.35,
+            model=self.model_name,
+            raw=response,
+        )