skillware-bench 0.3.2__py3-none-any.whl

skillware_bench/core/adapters/mistral_adapter.py

@@ -0,0 +1,136 @@
+# Copyright (c) 2026 ARPA Hellenic Logical Systems
+# SPDX-License-Identifier: MIT
+
+"""Mistral AI model adapter (Mistral Large, Codestral)."""
+
+import json
+import os
+import time
+from typing import Any, Dict, List, Optional
+
+from skillware_bench.core.adapters.base import (
+    BaseModelAdapter,
+    ModelResponse,
+    TokenUsage,
+    ToolCallRequest,
+    execute_with_retry,
+)
+
+try:
+    from skillware.core.loader import SkillLoader
+except ImportError:
+    SkillLoader = None  # type: ignore
+
+
+class MistralAdapter(BaseModelAdapter):
+    """Adapter for Mistral AI frontier models using the official mistralai SDK."""
+
+    def __init__(
+        self,
+        model_name: str = "mistral-large-latest",
+        api_key: Optional[str] = None,
+        **kwargs: Any,
+    ):
+        super().__init__(model_name=model_name, api_key=api_key, **kwargs)
+        self.api_key = api_key or os.environ.get("MISTRAL_API_KEY")
+        self._client = None
+
+    def _get_client(self):
+        if self._client is None:
+            try:
+                from mistralai import Mistral
+
+                self._client = Mistral(api_key=self.api_key)
+            except ImportError:
+                raise ImportError(
+                    "mistralai package is required for MistralAdapter. Run: pip install mistralai"
+                )
+        return self._client
+
+    def format_tools(self, tool_definitions: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+        formatted: List[Dict[str, Any]] = []
+        for tool in tool_definitions:
+            if "function" in tool and tool.get("type") == "function":
+                formatted.append(tool)
+            elif "manifest" in tool and SkillLoader is not None:
+                formatted.append(SkillLoader.to_openai_tool(tool))
+            else:
+                name = tool.get("name", "unknown_tool")
+                desc = tool.get("description", "")
+                params = tool.get("parameters", {})
+                formatted.append(
+                    {
+                        "type": "function",
+                        "function": {
+                            "name": name,
+                            "description": desc,
+                            "parameters": params,
+                        },
+                    }
+                )
+        return formatted
+
+    def chat(
+        self,
+        messages: List[Dict[str, Any]],
+        tools: Optional[List[Dict[str, Any]]] = None,
+        system_prompt: Optional[str] = None,
+        **kwargs: Any,
+    ) -> ModelResponse:
+        client = self._get_client()
+
+        chat_messages = list(messages)
+        if system_prompt and not any(m.get("role") == "system" for m in chat_messages):
+            chat_messages.insert(0, {"role": "system", "content": system_prompt})
+
+        payload: Dict[str, Any] = {
+            "model": self.model_name,
+            "messages": chat_messages,
+        }
+
+        if self.temperature is not None:
+            payload["temperature"] = self.temperature
+        if self.max_tokens:
+            payload["max_tokens"] = self.max_tokens
+        if tools:
+            payload["tools"] = self.format_tools(tools)
+
+        t0 = time.perf_counter()
+
+        def _call_api():
+            return client.chat.complete(**payload)
+
+        response = execute_with_retry(_call_api)
+        latency_ms = (time.perf_counter() - t0) * 1000.0
+
+        choice = response.choices[0]
+        content = choice.message.content or ""
+
+        tool_calls: List[ToolCallRequest] = []
+        if getattr(choice.message, "tool_calls", None):
+            for tc in choice.message.tool_calls:
+                fn_name = tc.function.name
+                try:
+                    args = json.loads(tc.function.arguments) if tc.function.arguments else {}
+                except (json.JSONDecodeError, TypeError):
+                    args = tc.function.arguments if isinstance(tc.function.arguments, dict) else {}
+                tool_calls.append(
+                    ToolCallRequest(name=fn_name, arguments=args, id=tc.id or fn_name)
+                )
+
+        usage = TokenUsage(
+            prompt_tokens=response.usage.prompt_tokens,
+            completion_tokens=response.usage.completion_tokens,
+            total_tokens=response.usage.total_tokens,
+        )
+        self.record_usage(usage)
+
+        return ModelResponse(
+            content=content,
+            tool_calls=tool_calls,
+            usage=usage,
+            latency_ms=latency_ms,
+            ttft_ms=latency_ms * 0.40,
+            model=self.model_name,
+            raw=response,
+        )

skillware_bench/core/adapters/mock_adapter.py

@@ -0,0 +1,100 @@
+# Copyright (c) 2026 ARPA Hellenic Logical Systems
+# SPDX-License-Identifier: MIT
+
+"""Mock model adapter for deterministic offline CI, unit tests, and fault simulation."""
+
+from typing import Any, Dict, List, Optional
+
+from skillware_bench.core.adapters.base import (
+    BaseModelAdapter,
+    ModelResponse,
+    TokenUsage,
+    ToolCallRequest,
+)
+
+
+class SimulatedRateLimitError(Exception):
+    """Simulated 429 Too Many Requests error for retry testing."""
+
+
+class SimulatedServerError(Exception):
+    """Simulated 503 Service Unavailable error for retry testing."""
+
+
+class MockModelAdapter(BaseModelAdapter):
+    """Deterministic scriptable adapter for hermetic testing and error simulation."""
+
+    def __init__(
+        self,
+        model_name: str = "golden-target",
+        scripted_responses: Optional[List[ModelResponse]] = None,
+        simulated_failures_before_success: int = 0,
+        failure_exception_type: type = SimulatedRateLimitError,
+        **kwargs: Any,
+    ):
+        super().__init__(model_name=model_name, **kwargs)
+        self.scripted_responses = list(scripted_responses) if scripted_responses else []
+        self.simulated_failures_remaining = simulated_failures_before_success
+        self.failure_exception_type = failure_exception_type
+        self.call_history: List[Dict[str, Any]] = []
+        self.step_counter = 0
+
+    def add_response(self, response: ModelResponse) -> None:
+        self.scripted_responses.append(response)
+
+    def chat(
+        self,
+        messages: List[Dict[str, Any]],
+        tools: Optional[List[Dict[str, Any]]] = None,
+        system_prompt: Optional[str] = None,
+        **kwargs: Any,
+    ) -> ModelResponse:
+        self.call_history.append(
+            {
+                "messages": messages,
+                "tools": tools,
+                "system_prompt": system_prompt,
+                "kwargs": kwargs,
+            }
+        )
+
+        if self.simulated_failures_remaining > 0:
+            self.simulated_failures_remaining -= 1
+            raise self.failure_exception_type("Simulated provider rate limit (429) backoff trigger")
+
+        if self.scripted_responses and self.step_counter < len(self.scripted_responses):
+            resp = self.scripted_responses[self.step_counter]
+            self.step_counter += 1
+            self.record_usage(resp.usage)
+            return resp
+
+        # Default fallback response
+        self.step_counter += 1
+        default_usage = TokenUsage(prompt_tokens=150, completion_tokens=30, total_tokens=180)
+        self.record_usage(default_usage)
+
+        tool_calls: List[ToolCallRequest] = []
+        if tools and len(tools) > 0:
+            first_tool = tools[0]
+            name = (
+                first_tool.get("function", {}).get("name")
+                or first_tool.get("name")
+                or first_tool.get("manifest", {}).get("name")
+                or "execute_action"
+            )
+            tool_calls.append(
+                ToolCallRequest(
+                    name=name,
+                    arguments={"status": "mock_arg"},
+                    id=f"call_{self.step_counter}",
+                )
+            )
+
+        return ModelResponse(
+            content=f"Mock model response turn {self.step_counter}",
+            tool_calls=tool_calls,
+            usage=default_usage,
+            latency_ms=120.0,
+            ttft_ms=50.0,
+            model=self.model_name,
+        )

skillware_bench/core/adapters/ollama_adapter.py

@@ -0,0 +1,137 @@
+# Copyright (c) 2026 ARPA Hellenic Logical Systems
+# SPDX-License-Identifier: MIT
+
+"""Ollama local sovereign model adapter (Llama 3.2, Qwen 2.5)."""
+
+import json
+import os
+import time
+from typing import Any, Dict, List, Optional
+import requests
+
+from skillware_bench.core.adapters.base import (
+    BaseModelAdapter,
+    ModelResponse,
+    TokenUsage,
+    ToolCallRequest,
+    execute_with_retry,
+)
+
+try:
+    from skillware.core.loader import SkillLoader
+except ImportError:
+    SkillLoader = None  # type: ignore
+
+
+class OllamaAdapter(BaseModelAdapter):
+    """Adapter for local sovereign models running via Ollama API."""
+
+    def __init__(
+        self,
+        model_name: str = "llama3.2",
+        host: Optional[str] = None,
+        **kwargs: Any,
+    ):
+        super().__init__(model_name=model_name, **kwargs)
+        self.host = (host or os.environ.get("OLLAMA_HOST") or "http://localhost:11434").rstrip("/")
+
+    def format_tools(self, tool_definitions: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+        formatted: List[Dict[str, Any]] = []
+        for tool in tool_definitions:
+            if "function" in tool and tool.get("type") == "function":
+                formatted.append(tool)
+            elif "manifest" in tool and SkillLoader is not None:
+                formatted.append(SkillLoader.to_openai_tool(tool))
+            else:
+                name = tool.get("name", "unknown_tool")
+                desc = tool.get("description", "")
+                params = tool.get("parameters", {})
+                formatted.append(
+                    {
+                        "type": "function",
+                        "function": {
+                            "name": name,
+                            "description": desc,
+                            "parameters": params,
+                        },
+                    }
+                )
+        return formatted
+
+    def chat(
+        self,
+        messages: List[Dict[str, Any]],
+        tools: Optional[List[Dict[str, Any]]] = None,
+        system_prompt: Optional[str] = None,
+        **kwargs: Any,
+    ) -> ModelResponse:
+        url = f"{self.host}/api/chat"
+
+        chat_messages = list(messages)
+        if system_prompt and not any(m.get("role") == "system" for m in chat_messages):
+            chat_messages.insert(0, {"role": "system", "content": system_prompt})
+
+        payload: Dict[str, Any] = {
+            "model": self.model_name,
+            "messages": chat_messages,
+            "stream": False,
+        }
+
+        options: Dict[str, Any] = {}
+        if self.temperature is not None:
+            options["temperature"] = self.temperature
+        if self.max_tokens:
+            options["num_predict"] = self.max_tokens
+        if options:
+            payload["options"] = options
+
+        if tools:
+            payload["tools"] = self.format_tools(tools)
+
+        t0 = time.perf_counter()
+
+        def _call_api():
+            resp = requests.post(url, json=payload, timeout=self.timeout_seconds)
+            resp.raise_for_status()
+            return resp.json()
+
+        data = execute_with_retry(
+            _call_api,
+            retryable_exceptions=(requests.RequestException,),
+        )
+        latency_ms = (time.perf_counter() - t0) * 1000.0
+
+        message_data = data.get("message", {})
+        content = message_data.get("content", "")
+
+        tool_calls: List[ToolCallRequest] = []
+        raw_tools = message_data.get("tool_calls", [])
+        for tc in raw_tools:
+            fn = tc.get("function", {})
+            fn_name = fn.get("name", "unknown")
+            args = fn.get("arguments", {})
+            if isinstance(args, str):
+                try:
+                    args = json.loads(args)
+                except json.JSONDecodeError:
+                    args = {"_raw": args}
+            tool_calls.append(ToolCallRequest(name=fn_name, arguments=args, id=fn_name))
+
+        prompt_tokens = data.get("prompt_eval_count", 0) or 0
+        completion_tokens = data.get("eval_count", 0) or 0
+        usage = TokenUsage(
+            prompt_tokens=prompt_tokens,
+            completion_tokens=completion_tokens,
+            total_tokens=prompt_tokens + completion_tokens,
+        )
+        self.record_usage(usage)
+
+        return ModelResponse(
+            content=content,
+            tool_calls=tool_calls,
+            usage=usage,
+            latency_ms=latency_ms,
+            ttft_ms=latency_ms * 0.50,
+            model=self.model_name,
+            raw=data,
+        )

skillware_bench/core/adapters/openai_adapter.py

@@ -0,0 +1,164 @@
+# Copyright (c) 2026 ARPA Hellenic Logical Systems
+# SPDX-License-Identifier: MIT
+
+"""OpenAI model adapter (GPT-4o, o3-mini, o1)."""
+
+import json
+import os
+import time
+from typing import Any, Dict, List, Optional
+
+from skillware_bench.core.adapters.base import (
+    BaseModelAdapter,
+    ModelResponse,
+    TokenUsage,
+    ToolCallRequest,
+    execute_with_retry,
+)
+
+try:
+    from skillware.core.loader import SkillLoader
+except ImportError:
+    SkillLoader = None  # type: ignore
+
+
+class OpenAIAdapter(BaseModelAdapter):
+    """Adapter for OpenAI models supporting chat completions and function calling."""
+
+    def __init__(
+        self,
+        model_name: str = "gpt-4o",
+        api_key: Optional[str] = None,
+        base_url: Optional[str] = None,
+        **kwargs: Any,
+    ):
+        super().__init__(model_name=model_name, api_key=api_key, **kwargs)
+        self.api_key = api_key or os.environ.get("OPENAI_API_KEY")
+        self.base_url = base_url or os.environ.get("OPENAI_BASE_URL")
+        self._client = None
+
+    def _get_client(self):
+        if self._client is None:
+            try:
+                import openai
+
+                self._client = openai.OpenAI(
+                    api_key=self.api_key,
+                    base_url=self.base_url,
+                    timeout=self.timeout_seconds,
+                )
+            except ImportError:
+                raise ImportError(
+                    "openai package is required to use OpenAIAdapter. Run: pip install openai"
+                )
+        return self._client
+
+    def format_tools(self, tool_definitions: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+        formatted: List[Dict[str, Any]] = []
+        for tool in tool_definitions:
+            if "function" in tool and tool.get("type") == "function":
+                formatted.append(tool)
+            elif "manifest" in tool and SkillLoader is not None:
+                formatted.append(SkillLoader.to_openai_tool(tool))
+            else:
+                name = tool.get("name", "unknown_tool")
+                desc = tool.get("description", "")
+                params = tool.get("parameters", {})
+                formatted.append(
+                    {
+                        "type": "function",
+                        "function": {
+                            "name": name,
+                            "description": desc,
+                            "parameters": params,
+                        },
+                    }
+                )
+        return formatted
+
+    def chat(
+        self,
+        messages: List[Dict[str, Any]],
+        tools: Optional[List[Dict[str, Any]]] = None,
+        system_prompt: Optional[str] = None,
+        **kwargs: Any,
+    ) -> ModelResponse:
+        client = self._get_client()
+
+        chat_messages = list(messages)
+        if system_prompt and not any(m.get("role") == "system" for m in chat_messages):
+            role = "developer" if ("o1" in self.model_name or "o3" in self.model_name) else "system"
+            chat_messages.insert(0, {"role": role, "content": system_prompt})
+
+        payload: Dict[str, Any] = {
+            "model": self.model_name,
+            "messages": chat_messages,
+        }
+
+        # Temperature is not supported on o1/o3-mini in certain API versions
+        if not ("o1" in self.model_name or "o3" in self.model_name):
+            payload["temperature"] = self.temperature
+
+        if self.max_tokens:
+            payload["max_tokens"] = self.max_tokens
+
+        if tools:
+            payload["tools"] = self.format_tools(tools)
+            payload["tool_choice"] = kwargs.get("tool_choice", "auto")
+
+        t0 = time.perf_counter()
+
+        def _call_api():
+            return client.chat.completions.create(**payload)
+
+        try:
+            import openai
+
+            retry_types = (
+                openai.RateLimitError,
+                openai.APIConnectionError,
+                openai.InternalServerError,
+            )
+        except Exception:
+            retry_types = (Exception,)
+
+        response = execute_with_retry(_call_api, retryable_exceptions=retry_types)
+        latency_ms = (time.perf_counter() - t0) * 1000.0
+
+        choice = response.choices[0]
+        content = choice.message.content or ""
+
+        tool_calls: List[ToolCallRequest] = []
+        if getattr(choice.message, "tool_calls", None):
+            for tc in choice.message.tool_calls:
+                fn_name = tc.function.name
+                try:
+                    args = json.loads(tc.function.arguments) if tc.function.arguments else {}
+                except json.JSONDecodeError:
+                    args = {"_raw": tc.function.arguments}
+                tool_calls.append(ToolCallRequest(name=fn_name, arguments=args, id=tc.id))
+
+        cached_tokens = 0
+        if (
+            hasattr(response.usage, "prompt_tokens_details")
+            and response.usage.prompt_tokens_details
+        ):
+            cached_tokens = getattr(response.usage.prompt_tokens_details, "cached_tokens", 0) or 0
+
+        usage = TokenUsage(
+            prompt_tokens=response.usage.prompt_tokens,
+            completion_tokens=response.usage.completion_tokens,
+            cached_tokens=cached_tokens,
+            total_tokens=response.usage.total_tokens,
+        )
+        self.record_usage(usage)
+
+        return ModelResponse(
+            content=content,
+            tool_calls=tool_calls,
+            usage=usage,
+            latency_ms=latency_ms,
+            ttft_ms=latency_ms * 0.45,
+            model=self.model_name,
+            raw=response,
+        )