inspect-ai 0.3.88__py3-none-any.whl → 0.3.90__py3-none-any.whl

inspect_ai/model/_providers/openai.py

@@ -1,14 +1,8 @@
 import os
-import socket
 from logging import getLogger
-from typing import Any
+from typing import Any, Literal
 
-import httpx
 from openai import (
-    DEFAULT_CONNECTION_LIMITS,
-    DEFAULT_TIMEOUT,
-    APIStatusError,
-    APITimeoutError,
     AsyncAzureOpenAI,
     AsyncOpenAI,
     BadRequestError,
@@ -20,7 +14,6 @@ from openai.types.chat import ChatCompletion
 from typing_extensions import override
 
 from inspect_ai._util.error import PrerequisiteError
-from inspect_ai._util.http import is_retryable_http_status
 from inspect_ai._util.logger import warn_once
 from inspect_ai.model._openai import chat_choices_from_openai
 from inspect_ai.model._providers.openai_responses import generate_responses
@@ -31,20 +24,23 @@ from .._chat_message import ChatMessage
 from .._generate_config import GenerateConfig
 from .._model import ModelAPI
 from .._model_call import ModelCall
-from .._model_output import ChatCompletionChoice, ModelOutput, ModelUsage
+from .._model_output import ModelOutput
 from .._openai import (
-    OpenAIResponseError,
+    OpenAIAsyncHttpxClient,
     is_computer_use_preview,
     is_gpt,
     is_o1_mini,
     is_o1_preview,
     is_o1_pro,
     is_o_series,
+    model_output_from_openai,
     openai_chat_messages,
     openai_chat_tool_choice,
     openai_chat_tools,
+    openai_completion_params,
     openai_handle_bad_request,
     openai_media_filter,
+    openai_should_retry,
 )
 from .openai_o1 import generate_o1
 from .util import environment_prerequisite_error, model_base_url
@@ -55,6 +51,8 @@ OPENAI_API_KEY = "OPENAI_API_KEY"
 AZURE_OPENAI_API_KEY = "AZURE_OPENAI_API_KEY"
 AZUREAI_OPENAI_API_KEY = "AZUREAI_OPENAI_API_KEY"
 
+# NOTE: If you are creating a new provider that is OpenAI compatible you should inherit from OpenAICompatibleAPI rather than OpenAPAPI.
+
 
 class OpenAIAPI(ModelAPI):
     def __init__(
@@ -72,7 +70,6 @@ class OpenAIAPI(ModelAPI):
         parts = model_name.split("/")
         if parts[0] == "azure" and len(parts) > 1:
             self.service: str | None = parts[0]
-            model_name = "/".join(parts[1:])
         else:
             self.service = None
 
@@ -135,7 +132,7 @@ class OpenAIAPI(ModelAPI):
             else:
                 api_version = os.environ.get(
                     "AZUREAI_OPENAI_API_VERSION",
-                    os.environ.get("OPENAI_API_VERSION", "2025-02-01-preview"),
+                    os.environ.get("OPENAI_API_VERSION", "2025-03-01-preview"),
                 )
 
             self.client: AsyncAzureOpenAI | AsyncOpenAI = AsyncAzureOpenAI(
@@ -160,22 +157,22 @@ class OpenAIAPI(ModelAPI):
         return self.service == "azure"
 
     def is_o_series(self) -> bool:
-        return is_o_series(self.model_name)
+        return is_o_series(self.service_model_name())
 
     def is_o1_pro(self) -> bool:
-        return is_o1_pro(self.model_name)
+        return is_o1_pro(self.service_model_name())
 
     def is_o1_mini(self) -> bool:
-        return is_o1_mini(self.model_name)
+        return is_o1_mini(self.service_model_name())
 
     def is_o1_preview(self) -> bool:
-        return is_o1_preview(self.model_name)
+        return is_o1_preview(self.service_model_name())
 
     def is_computer_use_preview(self) -> bool:
-        return is_computer_use_preview(self.model_name)
+        return is_computer_use_preview(self.service_model_name())
 
     def is_gpt(self) -> bool:
-        return is_gpt(self.model_name)
+        return is_gpt(self.service_model_name())
 
     @override
     async def aclose(self) -> None:
@@ -217,7 +214,7 @@ class OpenAIAPI(ModelAPI):
             return await generate_responses(
                 client=self.client,
                 http_hooks=self._http_hooks,
-                model_name=self.model_name,
+                model_name=self.service_model_name(),
                 input=input,
                 tools=tools,
                 tool_choice=tool_choice,
@@ -242,15 +239,27 @@ class OpenAIAPI(ModelAPI):
         # unlike text models, vision models require a max_tokens (and set it to a very low
         # default, see https://community.openai.com/t/gpt-4-vision-preview-finish-details/475911/10)
         OPENAI_IMAGE_DEFAULT_TOKENS = 4096
-        if "vision" in self.model_name:
+        if "vision" in self.service_model_name():
             if isinstance(config.max_tokens, int):
                 config.max_tokens = max(config.max_tokens, OPENAI_IMAGE_DEFAULT_TOKENS)
             else:
                 config.max_tokens = OPENAI_IMAGE_DEFAULT_TOKENS
 
+        # determine system role
+        # o1-mini does not support developer or system messages
+        # (see Dec 17, 2024 changelog: https://platform.openai.com/docs/changelog)
+        if self.is_o1_mini():
+            system_role: Literal["user", "system", "developer"] = "user"
+        # other o-series models use 'developer' rather than 'system' messages
+        # https://platform.openai.com/docs/guides/reasoning#advice-on-prompting
+        elif self.is_o_series():
+            system_role = "developer"
+        else:
+            system_role = "system"
+
         # prepare request (we do this so we can log the ModelCall)
         request = dict(
-            messages=await openai_chat_messages(input, self.model_name),
+            messages=await openai_chat_messages(input, system_role),
             tools=openai_chat_tools(tools) if len(tools) > 0 else NOT_GIVEN,
             tool_choice=openai_chat_tool_choice(tool_choice)
             if len(tools) > 0
@@ -267,49 +276,16 @@ class OpenAIAPI(ModelAPI):
 
             # save response for model_call
             response = completion.model_dump()
-            self.on_response(response)
-
-            # parse out choices
-            choices = self._chat_choices_from_response(completion, tools)
 
             # return output and call
-            return ModelOutput(
-                model=completion.model,
-                choices=choices,
-                usage=(
-                    ModelUsage(
-                        input_tokens=completion.usage.prompt_tokens,
-                        output_tokens=completion.usage.completion_tokens,
-                        input_tokens_cache_read=(
-                            completion.usage.prompt_tokens_details.cached_tokens
-                            if completion.usage.prompt_tokens_details is not None
-                            else None  # openai only have cache read stats/pricing.
-                        ),
-                        reasoning_tokens=(
-                            completion.usage.completion_tokens_details.reasoning_tokens
-                            if completion.usage.completion_tokens_details is not None
-                            else None
-                        ),
-                        total_tokens=completion.usage.total_tokens,
-                    )
-                    if completion.usage
-                    else None
-                ),
-            ), model_call()
+            choices = chat_choices_from_openai(completion, tools)
+            return model_output_from_openai(completion, choices), model_call()
         except (BadRequestError, UnprocessableEntityError) as e:
-            return self.handle_bad_request(e), model_call()
+            return openai_handle_bad_request(self.service_model_name(), e), model_call()
 
-    def on_response(self, response: dict[str, Any]) -> None:
-        pass
-
-    def handle_bad_request(self, ex: APIStatusError) -> ModelOutput | Exception:
-        return openai_handle_bad_request(self.model_name, ex)
-
-    def _chat_choices_from_response(
-        self, response: ChatCompletion, tools: list[ToolInfo]
-    ) -> list[ChatCompletionChoice]:
-        # adding this as a method so we can override from other classes (e.g together)
-        return chat_choices_from_openai(response, tools)
+    def service_model_name(self) -> str:
+        """Model name without any service prefix."""
+        return self.model_name.replace(f"{self.service}/", "", 1)
 
     @override
     def should_retry(self, ex: Exception) -> bool:
@@ -321,14 +297,8 @@ class OpenAIAPI(ModelAPI):
                 return False
             else:
                 return True
-        elif isinstance(ex, APIStatusError):
-            return is_retryable_http_status(ex.status_code)
-        elif isinstance(ex, OpenAIResponseError):
-            return ex.code in ["rate_limit_exceeded", "server_error"]
-        elif isinstance(ex, APITimeoutError):
-            return True
         else:
-            return False
+            return openai_should_retry(ex)
 
     @override
     def connection_key(self) -> str:
@@ -336,105 +306,31 @@ class OpenAIAPI(ModelAPI):
         return str(self.api_key)
 
     def completion_params(self, config: GenerateConfig, tools: bool) -> dict[str, Any]:
-        params: dict[str, Any] = dict(
-            model=self.model_name,
-        )
+        # first call the default processing
+        params = openai_completion_params(self.service_model_name(), config, tools)
+
+        # now tailor to current model
         if config.max_tokens is not None:
             if self.is_o_series():
                 params["max_completion_tokens"] = config.max_tokens
-            else:
-                params["max_tokens"] = config.max_tokens
-        if config.frequency_penalty is not None:
-            params["frequency_penalty"] = config.frequency_penalty
-        if config.stop_seqs is not None:
-            params["stop"] = config.stop_seqs
-        if config.presence_penalty is not None:
-            params["presence_penalty"] = config.presence_penalty
-        if config.logit_bias is not None:
-            params["logit_bias"] = config.logit_bias
-        if config.seed is not None:
-            params["seed"] = config.seed
+                del params["max_tokens"]
+
         if config.temperature is not None:
             if self.is_o_series():
                 warn_once(
                     logger,
                     "o series models do not support the 'temperature' parameter (temperature is always 1).",
                 )
-            else:
-                params["temperature"] = config.temperature
-        # TogetherAPI requires temperature w/ num_choices
-        elif config.num_choices is not None:
-            params["temperature"] = 1
-        if config.top_p is not None:
-            params["top_p"] = config.top_p
-        if config.num_choices is not None:
-            params["n"] = config.num_choices
-        params = self.set_logprobs_params(params, config)
-        if tools and config.parallel_tool_calls is not None and not self.is_o_series():
-            params["parallel_tool_calls"] = config.parallel_tool_calls
-        if (
-            config.reasoning_effort is not None
-            and not self.is_gpt()
-            and not self.is_o1_mini()
-            and not self.is_o1_preview()
-        ):
-            params["reasoning_effort"] = config.reasoning_effort
-        if config.response_schema is not None:
-            params["response_format"] = dict(
-                type="json_schema",
-                json_schema=dict(
-                    name=config.response_schema.name,
-                    schema=config.response_schema.json_schema.model_dump(
-                        exclude_none=True
-                    ),
-                    description=config.response_schema.description,
-                    strict=config.response_schema.strict,
-                ),
-            )
+                del params["temperature"]
 
-        return params
-
-    def set_logprobs_params(
-        self, params: dict[str, Any], config: GenerateConfig
-    ) -> dict[str, Any]:
-        if config.logprobs is not None:
-            params["logprobs"] = config.logprobs
-        if config.top_logprobs is not None:
-            params["top_logprobs"] = config.top_logprobs
-        return params
-
-
-class OpenAIAsyncHttpxClient(httpx.AsyncClient):
-    """Custom async client that deals better with long running Async requests.
-
-    Based on Anthropic DefaultAsyncHttpClient implementation that they
-    released along with Claude 3.7 as well as the OpenAI DefaultAsyncHttpxClient
-
-    """
+        # remove parallel_tool_calls if not supported
+        if "parallel_tool_calls" in params.keys() and self.is_o_series():
+            del params["parallel_tool_calls"]
 
-    def __init__(self, **kwargs: Any) -> None:
-        # This is based on the openai DefaultAsyncHttpxClient:
-        # https://github.com/openai/openai-python/commit/347363ed67a6a1611346427bb9ebe4becce53f7e
-        kwargs.setdefault("timeout", DEFAULT_TIMEOUT)
-        kwargs.setdefault("limits", DEFAULT_CONNECTION_LIMITS)
-        kwargs.setdefault("follow_redirects", True)
-
-        # This is based on the anthrpopic changes for claude 3.7:
-        # https://github.com/anthropics/anthropic-sdk-python/commit/c5387e69e799f14e44006ea4e54fdf32f2f74393#diff-3acba71f89118b06b03f2ba9f782c49ceed5bb9f68d62727d929f1841b61d12bR1387-R1403
-
-        # set socket options to deal with long running reasoning requests
-        socket_options = [
-            (socket.SOL_SOCKET, socket.SO_KEEPALIVE, True),
-            (socket.IPPROTO_TCP, socket.TCP_KEEPINTVL, 60),
-            (socket.IPPROTO_TCP, socket.TCP_KEEPCNT, 5),
-        ]
-        TCP_KEEPIDLE = getattr(socket, "TCP_KEEPIDLE", None)
-        if TCP_KEEPIDLE is not None:
-            socket_options.append((socket.IPPROTO_TCP, TCP_KEEPIDLE, 60))
-
-        kwargs["transport"] = httpx.AsyncHTTPTransport(
-            limits=DEFAULT_CONNECTION_LIMITS,
-            socket_options=socket_options,
-        )
+        # remove reasoning_effort if not supported
+        if "reasoning_effort" in params.keys() and (
+            self.is_gpt() or self.is_o1_mini() or self.is_o1_preview()
+        ):
+            del params["reasoning_effort"]
 
-        super().__init__(**kwargs)
+        return params

inspect_ai/model/_providers/openai_compatible.py

@@ -0,0 +1,195 @@
+import os
+from logging import getLogger
+from typing import Any
+
+from openai import (
+    APIStatusError,
+    AsyncOpenAI,
+    BadRequestError,
+    PermissionDeniedError,
+    UnprocessableEntityError,
+)
+from openai._types import NOT_GIVEN
+from openai.types.chat import ChatCompletion
+from typing_extensions import override
+
+from inspect_ai.model._openai import chat_choices_from_openai
+from inspect_ai.model._providers.util.hooks import HttpxHooks
+from inspect_ai.tool import ToolChoice, ToolInfo
+
+from .._chat_message import ChatMessage
+from .._generate_config import GenerateConfig
+from .._model import ModelAPI
+from .._model_call import ModelCall
+from .._model_output import ChatCompletionChoice, ModelOutput
+from .._openai import (
+    OpenAIAsyncHttpxClient,
+    model_output_from_openai,
+    openai_chat_messages,
+    openai_chat_tool_choice,
+    openai_chat_tools,
+    openai_completion_params,
+    openai_handle_bad_request,
+    openai_media_filter,
+    openai_should_retry,
+)
+from .util import environment_prerequisite_error, model_base_url
+
+logger = getLogger(__name__)
+
+
+class OpenAICompatibleAPI(ModelAPI):
+    def __init__(
+        self,
+        model_name: str,
+        base_url: str | None = None,
+        api_key: str | None = None,
+        config: GenerateConfig = GenerateConfig(),
+        service: str | None = None,
+        service_base_url: str | None = None,
+        **model_args: Any,
+    ) -> None:
+        # extract service prefix from model name if not specified
+        if service is None:
+            parts = model_name.split("/")
+            if len(parts) == 1:
+                raise ValueError(
+                    "openai-api model names must include a service prefix (e.g. 'openai-api/service/model')"
+                )
+            self.service = parts[0]
+        else:
+            self.service = service
+
+        # compute api key
+        api_key_var = f"{self.service.upper()}_API_KEY"
+
+        super().__init__(
+            model_name=model_name,
+            base_url=base_url,
+            api_key=api_key,
+            api_key_vars=[api_key_var],
+            config=config,
+        )
+
+        # use service prefix to lookup api_key
+        if not self.api_key:
+            self.api_key = os.environ.get(api_key_var, None)
+            if not self.api_key:
+                raise environment_prerequisite_error(
+                    self.service,
+                    [api_key_var],
+                )
+
+        # use service prefix to lookup base_url
+        if not self.base_url:
+            base_url_var = f"{self.service.upper()}_BASE_URL"
+            self.base_url = model_base_url(base_url, [base_url_var]) or service_base_url
+            if not self.base_url:
+                raise environment_prerequisite_error(
+                    self.service,
+                    [base_url_var],
+                )
+
+        # create async http client
+        http_client = OpenAIAsyncHttpxClient()
+        self.client = AsyncOpenAI(
+            api_key=self.api_key,
+            base_url=self.base_url,
+            http_client=http_client,
+            **model_args,
+        )
+
+        # create time tracker
+        self._http_hooks = HttpxHooks(self.client._client)
+
+    @override
+    async def aclose(self) -> None:
+        await self.client.close()
+
+    async def generate(
+        self,
+        input: list[ChatMessage],
+        tools: list[ToolInfo],
+        tool_choice: ToolChoice,
+        config: GenerateConfig,
+    ) -> ModelOutput | tuple[ModelOutput | Exception, ModelCall]:
+        # allocate request_id (so we can see it from ModelCall)
+        request_id = self._http_hooks.start_request()
+
+        # setup request and response for ModelCall
+        request: dict[str, Any] = {}
+        response: dict[str, Any] = {}
+
+        def model_call() -> ModelCall:
+            return ModelCall.create(
+                request=request,
+                response=response,
+                filter=openai_media_filter,
+                time=self._http_hooks.end_request(request_id),
+            )
+
+        # get completion params (slice off service from model name)
+        completion_params = self.completion_params(
+            config=config,
+            tools=len(tools) > 0,
+        )
+
+        # prepare request (we do this so we can log the ModelCall)
+        request = dict(
+            messages=await openai_chat_messages(input),
+            tools=openai_chat_tools(tools) if len(tools) > 0 else NOT_GIVEN,
+            tool_choice=openai_chat_tool_choice(tool_choice)
+            if len(tools) > 0
+            else NOT_GIVEN,
+            extra_headers={HttpxHooks.REQUEST_ID_HEADER: request_id},
+            **completion_params,
+        )
+
+        try:
+            # generate completion and save response for model call
+            completion: ChatCompletion = await self.client.chat.completions.create(
+                **request
+            )
+            response = completion.model_dump()
+            self.on_response(response)
+
+            # return output and call
+            choices = self.chat_choices_from_completion(completion, tools)
+            return model_output_from_openai(completion, choices), model_call()
+
+        except (BadRequestError, UnprocessableEntityError, PermissionDeniedError) as ex:
+            return self.handle_bad_request(ex), model_call()
+
+    def service_model_name(self) -> str:
+        """Model name without any service prefix."""
+        return self.model_name.replace(f"{self.service}/", "", 1)
+
+    @override
+    def should_retry(self, ex: Exception) -> bool:
+        return openai_should_retry(ex)
+
+    @override
+    def connection_key(self) -> str:
+        """Scope for enforcing max_connections (could also use endpoint)."""
+        return str(self.api_key)
+
+    def completion_params(self, config: GenerateConfig, tools: bool) -> dict[str, Any]:
+        return openai_completion_params(
+            model=self.service_model_name(),
+            config=config,
+            tools=tools,
+        )
+
+    def on_response(self, response: dict[str, Any]) -> None:
+        """Hook for subclasses to do custom response handling."""
+        pass
+
+    def chat_choices_from_completion(
+        self, completion: ChatCompletion, tools: list[ToolInfo]
+    ) -> list[ChatCompletionChoice]:
+        """Hook for subclasses to do custom chat choice processing."""
+        return chat_choices_from_openai(completion, tools)
+
+    def handle_bad_request(self, ex: APIStatusError) -> ModelOutput | Exception:
+        """Hook for subclasses to do bad request handling"""
+        return openai_handle_bad_request(self.service_model_name(), ex)

inspect_ai/model/_providers/openrouter.py

@@ -1,16 +1,13 @@
 import json
-import os
 from typing import Any, TypedDict
 
 from typing_extensions import NotRequired, override
 
 from inspect_ai._util.error import PrerequisiteError
 from inspect_ai.model._openai import OpenAIResponseError
-from inspect_ai.model._providers.util import model_base_url
-from inspect_ai.model._providers.util.util import environment_prerequisite_error
 
 from .._generate_config import GenerateConfig
-from .openai import OpenAIAPI
+from .openai_compatible import OpenAICompatibleAPI
 
 OPENROUTER_API_KEY = "OPENROUTER_API_KEY"
 
@@ -37,7 +34,7 @@ class OpenRouterError(Exception):
         )
 
 
-class OpenRouterAPI(OpenAIAPI):
+class OpenRouterAPI(OpenAICompatibleAPI):
     def __init__(
         self,
         model_name: str,
@@ -46,16 +43,6 @@ class OpenRouterAPI(OpenAIAPI):
         config: GenerateConfig = GenerateConfig(),
         **model_args: Any,
     ) -> None:
-        # api_key
-        if not api_key:
-            api_key = os.environ.get(OPENROUTER_API_KEY, None)
-            if not api_key:
-                raise environment_prerequisite_error("OpenRouter", OPENROUTER_API_KEY)
-
-        # base_url
-        base_url = model_base_url(base_url, "OPENROUTER_BASE_URL")
-        base_url = base_url if base_url else "https://openrouter.ai/api/v1"
-
         # collect known model args that we forward to generate
         def collect_model_arg(name: str) -> Any | None:
             nonlocal model_args
@@ -88,6 +75,8 @@ class OpenRouterAPI(OpenAIAPI):
             base_url=base_url,
             api_key=api_key,
             config=config,
+            service="OpenRouter",
+            service_base_url="https://openrouter.ai/api/v1",
             **model_args,
         )
 

inspect_ai/model/_providers/providers.py

@@ -44,6 +44,17 @@ def openai() -> type[ModelAPI]:
     return OpenAIAPI
 
 
+@modelapi(name="openai-api")
+def openai_api() -> type[ModelAPI]:
+    # validate
+    validate_openai_client("OpenAI Compatible API")
+
+    # in the clear
+    from .openai_compatible import OpenAICompatibleAPI
+
+    return OpenAICompatibleAPI
+
+
 @modelapi(name="anthropic")
 def anthropic() -> type[ModelAPI]:
     FEATURE = "Anthropic API"