inspect-ai 0.3.56__py3-none-any.whl → 0.3.58__py3-none-any.whl

inspect_ai/model/_providers/openai.py

@@ -17,6 +17,7 @@ from openai.types.chat import (
     ChatCompletion,
     ChatCompletionAssistantMessageParam,
     ChatCompletionContentPartImageParam,
+    ChatCompletionContentPartInputAudioParam,
     ChatCompletionContentPartParam,
     ChatCompletionContentPartTextParam,
     ChatCompletionDeveloperMessageParam,
@@ -36,9 +37,9 @@ from typing_extensions import override
 from inspect_ai._util.constants import DEFAULT_MAX_RETRIES
 from inspect_ai._util.content import Content
 from inspect_ai._util.error import PrerequisiteError
-from inspect_ai._util.images import image_as_data_uri
+from inspect_ai._util.images import file_as_data_uri
 from inspect_ai._util.logger import warn_once
-from inspect_ai._util.url import is_data_uri, is_http_url
+from inspect_ai._util.url import is_http_url
 from inspect_ai.tool import ToolCall, ToolChoice, ToolFunction, ToolInfo
 
 from .._chat_message import ChatMessage, ChatMessageAssistant
@@ -51,6 +52,7 @@ from .._model_output import (
     Logprobs,
     ModelOutput,
     ModelUsage,
+    StopReason,
 )
 from .openai_o1 import generate_o1
 from .util import (
@@ -262,7 +264,10 @@ class OpenAIAPI(ModelAPI):
             model=self.model_name,
         )
         if config.max_tokens is not None:
-            params["max_tokens"] = config.max_tokens
+            if self.is_o1():
+                params["max_completion_tokens"] = config.max_tokens
+            else:
+                params["max_tokens"] = config.max_tokens
         if config.frequency_penalty is not None:
             params["frequency_penalty"] = config.frequency_penalty
         if config.stop_seqs is not None:
@@ -303,13 +308,23 @@ class OpenAIAPI(ModelAPI):
 
     # convert some well known bad request errors into ModelOutput
     def handle_bad_request(self, e: BadRequestError) -> ModelOutput:
-        if e.status_code == 400 and e.code == "context_length_exceeded":
+        if e.status_code == 400:
+            # extract message
             if isinstance(e.body, dict) and "message" in e.body.keys():
                 content = str(e.body.get("message"))
             else:
                 content = e.message
+
+            # narrow stop_reason
+            if e.code == "context_length_exceeded":
+                stop_reason: StopReason = "model_length"
+            elif e.code == "invalid_prompt":
+                stop_reason = "content_filter"
+            else:
+                stop_reason = "unknown"
+
             return ModelOutput.from_content(
-                model=self.model_name, content=content, stop_reason="model_length"
+                model=self.model_name, content=content, stop_reason=stop_reason
             )
         else:
             raise e
@@ -449,16 +464,27 @@ async def as_chat_completion_part(
 ) -> ChatCompletionContentPartParam:
     if content.type == "text":
         return ChatCompletionContentPartTextParam(type="text", text=content.text)
-    else:
+    elif content.type == "image":
         # API takes URL or base64 encoded file. If it's a remote file or
         # data URL leave it alone, otherwise encode it
         image_url = content.image
         detail = content.detail
 
-        if not is_http_url(image_url) and not is_data_uri(image_url):
-            image_url = await image_as_data_uri(image_url)
+        if not is_http_url(image_url):
+            image_url = await file_as_data_uri(image_url)
 
         return ChatCompletionContentPartImageParam(
             type="image_url",
             image_url=dict(url=image_url, detail=detail),
         )
+    elif content.type == "audio":
+        audio_data = await file_as_data_uri(content.audio)
+
+        return ChatCompletionContentPartInputAudioParam(
+            type="input_audio", input_audio=dict(data=audio_data, format=content.format)
+        )
+
+    else:
+        raise RuntimeError(
+            "Video content is not currently supported by Open AI chat models."
+        )

inspect_ai/model/_providers/openai_o1.py

@@ -25,7 +25,7 @@ from inspect_ai.model import (
 from inspect_ai.tool import ToolCall, ToolInfo
 
 from .._model_call import ModelCall
-from .._model_output import ModelUsage
+from .._model_output import ModelUsage, StopReason
 from .._providers.util import (
     ChatAPIHandler,
     ChatAPIMessage,
@@ -48,12 +48,6 @@ async def generate_o1(
     # create chatapi handler
     handler = O1PreviewChatAPIHandler()
 
-    # map max_tokens => max_completion_tokens
-    max_tokens = params.get("max_tokens", None)
-    if max_tokens:
-        params["max_completion_tokens"] = max_tokens
-        del params["max_tokens"]
-
     # call model
     request = dict(
         model=model,
@@ -89,12 +83,16 @@ async def generate_o1(
 
 
 def handle_bad_request(model: str, ex: BadRequestError) -> ModelOutput:
-    if ex.code == "invalid_prompt":
-        return ModelOutput.from_content(
-            model=model, content=str(ex), stop_reason="content_filter"
-        )
+    if ex.code == "context_length_exceeded":
+        stop_reason: StopReason = "model_length"
+    elif ex.code == "invalid_prompt":
+        stop_reason = "content_filter"
     else:
-        raise ex
+        stop_reason = "unknown"
+
+    return ModelOutput.from_content(
+        model=model, content=str(ex), stop_reason=stop_reason
+    )
 
 
 def chat_messages(

inspect_ai/model/_providers/vertex.py

@@ -24,8 +24,14 @@ from vertexai.generative_models import (  # type: ignore
 from vertexai.generative_models import Content as VertexContent
 
 from inspect_ai._util.constants import BASE_64_DATA_REMOVED
-from inspect_ai._util.content import Content, ContentText
-from inspect_ai._util.images import image_as_data
+from inspect_ai._util.content import (
+    Content,
+    ContentAudio,
+    ContentImage,
+    ContentText,
+    ContentVideo,
+)
+from inspect_ai._util.images import file_as_data
 from inspect_ai.tool import ToolCall, ToolChoice, ToolInfo
 
 from .._chat_message import (
@@ -308,9 +314,16 @@ async def content_part(content: Content | str) -> Part:
         return Part.from_text(content or NO_CONTENT)
     elif isinstance(content, ContentText):
         return Part.from_text(content.text or NO_CONTENT)
-    else:
-        image_bytes, mime_type = await image_as_data(content.image)
+    elif isinstance(content, ContentImage):
+        image_bytes, mime_type = await file_as_data(content.image)
         return Part.from_image(image=Image.from_bytes(data=image_bytes))
+    else:
+        if isinstance(content, ContentAudio):
+            file = content.audio
+        elif isinstance(content, ContentVideo):
+            file = content.video
+        file_bytes, mime_type = await file_as_data(file)
+        return Part.from_data(file_bytes, mime_type)
 
 
 def prepend_system_messages(

inspect_ai/scorer/__init__.py

@@ -1,3 +1,5 @@
+from inspect_ai._util.deprecation import relocated_module_attribute
+
 from ._answer import AnswerPattern, answer
 from ._choice import choice
 from ._classification import exact, f1
@@ -16,7 +18,7 @@ from ._metric import (
 )
 from ._metrics.accuracy import accuracy
 from ._metrics.mean import mean
-from ._metrics.std import bootstrap_std, std, stderr
+from ._metrics.std import bootstrap_stderr, std, stderr
 from ._model import model_graded_fact, model_graded_qa
 from ._multi import multi_scorer
 from ._pattern import pattern
@@ -50,7 +52,7 @@ __all__ = [
     "Target",
     "scorer",
     "accuracy",
-    "bootstrap_std",
+    "bootstrap_stderr",
     "std",
     "stderr",
     "mean",
@@ -76,3 +78,12 @@ __all__ = [
     "at_least",
     "pass_at",
 ]
+_BOOTSTRAP_RENAME_VERSION = "0.3.58"
+_REMOVED_IN = "0.4"
+
+relocated_module_attribute(
+    "bootstrap_std",
+    "inspect_ai.scorer.bootstrap_stderr",
+    _BOOTSTRAP_RENAME_VERSION,
+    _REMOVED_IN,
+)

inspect_ai/scorer/_metrics/__init__.py

@@ -1,12 +1,12 @@
 from .accuracy import accuracy
 from .mean import mean, var
-from .std import bootstrap_std, std, stderr
+from .std import bootstrap_stderr, std, stderr
 
 __all__ = [
     "accuracy",
     "mean",
     "var",
-    "bootstrap_std",
+    "bootstrap_stderr",
     "std",
     "stderr",
 ]

inspect_ai/scorer/_metrics/std.py

@@ -15,10 +15,10 @@ logger = getLogger(__name__)
 
 
 @metric
-def bootstrap_std(
+def bootstrap_stderr(
     num_samples: int = 1000, to_float: ValueToFloat = value_to_float()
 ) -> Metric:
-    """Standard deviation of a bootstrapped estimate of the mean.
+    """Standard error of the mean using bootstrap.
 
     Args:
        num_samples (int): Number of bootstrap samples to take.
@@ -31,7 +31,7 @@ def bootstrap_std(
          0 if the Value is a complex object (list or dict).
 
     Returns:
-       bootstrap_std metric
+       bootstrap_stderr metric
     """
 
     def metric(scores: list[Score]) -> float:

inspect_ai/tool/__init__.py

@@ -1,4 +1,10 @@
-from inspect_ai._util.content import Content, ContentImage, ContentText
+from inspect_ai._util.content import (
+    Content,
+    ContentAudio,
+    ContentImage,
+    ContentText,
+    ContentVideo,
+)
 from inspect_ai._util.deprecation import relocated_module_attribute
 
 from ._tool import Tool, ToolError, ToolResult, tool
@@ -30,8 +36,10 @@ __all__ = [
     "ToolError",
     "ToolResult",
     "Content",
+    "ContentAudio",
     "ContentImage",
     "ContentText",
+    "ContentVideo",
     "ToolCall",
     "ToolCallContent",
     "ToolCallView",

inspect_ai/tool/_tool.py

@@ -11,7 +11,12 @@ from typing import (
     runtime_checkable,
 )
 
-from inspect_ai._util.content import ContentImage, ContentText
+from inspect_ai._util.content import (
+    ContentAudio,
+    ContentImage,
+    ContentText,
+    ContentVideo,
+)
 from inspect_ai._util.registry import (
     RegistryInfo,
     registry_add,
@@ -31,7 +36,9 @@ ToolResult = (
     | bool
     | ContentText
     | ContentImage
-    | list[ContentText | ContentImage]
+    | ContentAudio
+    | ContentVideo
+    | list[ContentText | ContentImage | ContentAudio | ContentVideo]
 )
 
 

inspect_ai/tool/_tool_info.py

@@ -8,6 +8,7 @@ from typing import (
     Dict,
     List,
     Optional,
+    Tuple,
     Type,
     Union,
     get_args,
@@ -155,7 +156,7 @@ def parse_type(type_hint: Type[Any]) -> ToolParam:
             return ToolParam(type="null")
         else:
             return ToolParam()
-    elif origin is list or origin is List:
+    elif origin is list or origin is List or origin is tuple or origin is Tuple:
         return ToolParam(
             type="array", items=parse_type(args[0]) if args else ToolParam()
         )

inspect_ai/tool/_tools/_web_browser/_resources/dm_env_servicer.py

@@ -38,9 +38,9 @@ class EnvironmentSpec:
             for i, obs_spec in enumerate(env_obs_spec.values()):
                 self.observation_spec[i + 1] = convert(obs_spec)
 
-        assert isinstance(
-            env.action_spec(), specs.Array
-        ), "Only a single action type is supported."
+        assert isinstance(env.action_spec(), specs.Array), (
+            "Only a single action type is supported."
+        )
         self.action_spec = {1: convert(env.action_spec())}
 
         self.observation_manager = spec_manager.SpecManager(self.observation_spec)
@@ -234,12 +234,12 @@ class EnvironmentService(dm_env_rpc_pb2_grpc.EnvironmentServicer):
             observations.
         """
         with self._lock:
-            assert (
-                cur_world in self._envs
-            ), "Current world does not have an assosiated environment"
-            assert (
-                cur_world in self._joined_worlds
-            ), "Please join world before calling step."
+            assert cur_world in self._envs, (
+                "Current world does not have an assosiated environment"
+            )
+            assert cur_world in self._joined_worlds, (
+                "Please join world before calling step."
+            )
             env = self._envs[cur_world]
             spec = self._specs[cur_world]
 

inspect_ai/tool/_tools/_web_browser/_web_browser.py

@@ -372,7 +372,9 @@ async def web_browser_cmd(cmd: str, *args: str) -> str:
         )
     else:
         response = parse_web_browser_output(result.stdout)
-        if "web_at" in response:
+        if "error" in response and response.get("error", "").strip() != "":
+            raise ToolError(str(response.get("error")) or "(unknown error)")
+        elif "web_at" in response:
             web_at = (
                 str(response.get("web_at")) or "(no web accessiblity tree available)"
             )
@@ -384,8 +386,6 @@ async def web_browser_cmd(cmd: str, *args: str) -> str:
             web_at = "\n".join(web_at_lines)
             store_as(WebBrowserStore).web_at = web_at
             return web_at
-        elif "error" in response:
-            raise ToolError(str(response.get("error")) or "(unknown error)")
         else:
             raise RuntimeError(
                 f"web_browser output must contain either 'error' or 'web_at' field: {result.stdout}"

inspect_ai/util/__init__.py

@@ -1,3 +1,5 @@
+from inspect_ai._util.trace import trace_action, trace_message
+
 from ._concurrency import concurrency
 from ._console import input_screen
 from ._display import DisplayType, display_type
@@ -24,7 +26,6 @@ from ._subprocess import (
 )
 from ._subtask import Subtask, subtask
 from ._throttle import throttle
-from ._trace import trace_enabled, trace_panel
 
 __all__ = [
     "ExecResult",
@@ -54,6 +55,6 @@ __all__ = [
     "Subtask",
     "subtask",
     "throttle",
-    "trace_enabled",
-    "trace_panel",
+    "trace_action",
+    "trace_message",
 ]

inspect_ai/util/{_trace.py → _conversation.py}

@@ -1,5 +1,3 @@
-from contextvars import ContextVar
-
 from rich import print
 from rich.console import RenderableType
 from rich.text import Text
@@ -7,12 +5,7 @@ from rich.text import Text
 from inspect_ai._util.transcript import transcript_panel
 
 
-def trace_enabled() -> bool:
-    """Is trace mode currently enabled."""
-    return _trace.get(None) is True
-
-
-def trace_panel(
+def conversation_panel(
     title: str,
     *,
     subtitle: str | None = None,
@@ -20,8 +13,8 @@ def trace_panel(
 ) -> None:
     """Trace content into a standard trace panel display.
 
-    Typically you would call `trace_enabled()` to confirm that trace mode
-    is enabled before calling `trace_panel()`.
+    Typically you would call `display_type() == "conversation"` to confirm that
+    we are in conversation mode before calling `conversation_panel()`.
 
     Args:
       title (str): Panel title.
@@ -32,10 +25,3 @@ def trace_panel(
         transcript_panel(title, subtitle, content),
         Text(),
     )
-
-
-def init_trace(trace: bool | None) -> None:
-    _trace.set(trace)
-
-
-_trace: ContextVar[bool | None] = ContextVar("_trace_mode")

inspect_ai/util/_display.py

@@ -3,10 +3,11 @@ from logging import getLogger
 from typing import Literal
 
 from inspect_ai._util.constants import DEFAULT_DISPLAY
+from inspect_ai._util.thread import is_main_thread
 
 logger = getLogger(__name__)
 
-DisplayType = Literal["full", "rich", "plain", "none"]
+DisplayType = Literal["full", "conversation", "rich", "plain", "none"]
 """Console display type."""
 
 
@@ -15,15 +16,24 @@ _display_type: DisplayType | None = None
 
 def init_display_type(display: str | None = None) -> DisplayType:
     global _display_type
-    global _display_metrics
     display = (
         display or os.environ.get("INSPECT_DISPLAY", DEFAULT_DISPLAY).lower().strip()
     )
+
+    # if we are on a background thread then throttle down to "plain"
+    # ("full" requires textual which cannot run in a background thread
+    # b/c it calls the Python signal function; "rich" assumes exclusive
+    # display access which may not be the case for threads)
+    if display in ["full", "rich"] and not is_main_thread():
+        display = "plain"
+
     match display:
-        case "full" | "rich" | "plain" | "none":
+        case "full" | "conversation" | "rich" | "plain" | "none":
             _display_type = display
         case _:
-            logger.warning(f"Unknown display type '{display}'")
+            logger.warning(
+                f"Unknown display type '{display}' (setting display to 'full')"
+            )
             _display_type = "full"
     return _display_type
 

inspect_ai/util/_sandbox/context.py

@@ -4,6 +4,8 @@ from typing import Any, NoReturn, cast
 
 from shortuuid import uuid
 
+from inspect_ai._util.constants import SANDBOX_SETUP_TIMEOUT
+
 from .environment import (
     SampleCleanup,
     SampleInit,
@@ -193,23 +195,20 @@ async def setup_sandbox_environment(
     setup_file = f"/tmp/{uuid()}"
     await env.write_file(setup_file, setup)
 
-    # chmod, execute, and remove
-    async def exec(cmd: list[str]) -> None:
-        try:
-            result = await env.exec(cmd, timeout=30)
-        except TimeoutError:
-            raise RuntimeError(
-                f"Timed out executing command {' '.join(cmd)} in sandbox"
-            )
-
+    # execute and then remove setup script (don't retry it on timeout
+    # in case it is not idempotent)
+    try:
+        await env.exec(["chmod", "+x", setup_file], timeout=30)
+        result = await env.exec(
+            ["env", setup_file], timeout=SANDBOX_SETUP_TIMEOUT, timeout_retry=False
+        )
         if not result.success:
             raise RuntimeError(
                 f"Failed to execute setup script for sample: {result.stderr}"
             )
-
-    await exec(["chmod", "+x", setup_file])
-    await exec(["env", setup_file])
-    await exec(["rm", setup_file])
+        await env.exec(["rm", setup_file], timeout=30)
+    except TimeoutError:
+        raise RuntimeError("Timed out executing setup command in sandbox")
 
 
 def default_sandbox_environment(

inspect_ai/util/_sandbox/docker/compose.py

@@ -25,18 +25,17 @@ COMPOSE_WAIT = "120"
 
 
 async def compose_up(project: ComposeProject) -> None:
-    # Start the environment
-    result = await compose_command(
+    # Start the environment. Note that we don't check the result because docker will
+    # return a non-zero exit code for services that exit (even successfully) when
+    # passing the --wait flag (see https://github.com/docker/compose/issues/10596).
+    # In practice, we will catch any errors when calling compose_check_running()
+    # immediately after we call compose_up().
+    await compose_command(
         ["up", "--detach", "--wait", "--wait-timeout", COMPOSE_WAIT],
         project=project,
         # wait up to 5 minutes for container to go up (compose wait + 3 minutes)
         timeout=300,
     )
-    if not result.success:
-        msg = (
-            f"Failed to start docker services for {project.config}: " f"{result.stderr}"
-        )
-        raise RuntimeError(msg)
 
 
 async def compose_down(project: ComposeProject, quiet: bool = True) -> None:
@@ -93,14 +92,21 @@ async def compose_cp(
         raise RuntimeError(msg)
 
 
-async def compose_check_running(services: list[str], project: ComposeProject) -> None:
+async def compose_check_running(
+    services: list[str], project: ComposeProject
+) -> list[str]:
     # Check to ensure that the status of containers is healthy
     running_services = await compose_ps(project=project, status="running")
-    if len(running_services) > 0:
-        if len(running_services) != len(services):
+    exited_services = await compose_ps(project=project, status="exited")
+    successful_services = running_services + [
+        service for service in exited_services if service["ExitCode"] == 0
+    ]
+
+    if len(successful_services) > 0:
+        if len(successful_services) != len(services):
             unhealthy_services = services
-            for running_service in running_services:
-                unhealthy_services.remove(running_service["Service"])
+            for successful_service in successful_services:
+                unhealthy_services.remove(successful_service["Service"])
 
             msg = (
                 "One or more docker containers failed to start from "
@@ -110,6 +116,8 @@ async def compose_check_running(services: list[str], project: ComposeProject) ->
     else:
         raise RuntimeError("No services started")
 
+    return [service["Service"] for service in running_services]
+
 
 async def compose_ps(
     project: ComposeProject,
@@ -168,6 +176,7 @@ async def compose_exec(
     *,
     project: ComposeProject,
     timeout: int | None,
+    timeout_retry: bool = True,
     input: str | bytes | None = None,
     output_limit: int | None = None,
 ) -> ExecResult[str]:
@@ -175,6 +184,7 @@ async def compose_exec(
         ["exec"] + command,
         project=project,
         timeout=timeout,
+        timeout_retry=timeout_retry,
         input=input,
         forward_env=False,
         output_limit=output_limit,
@@ -260,6 +270,7 @@ async def compose_command(
     *,
     project: ComposeProject,
     timeout: int | None,
+    timeout_retry: bool = True,
     input: str | bytes | None = None,
     cwd: str | Path | None = None,
     forward_env: bool = True,
@@ -327,7 +338,7 @@ async def compose_command(
                 return await run_command(command_timeout)
             except TimeoutError:
                 retries += 1
-                if retries <= MAX_RETRIES:
+                if timeout_retry and (retries <= MAX_RETRIES):
                     logger.info(
                         f"Retrying docker compose command: {shlex.join(compose_command)}"
                     )