inspect-ai 0.3.57__py3-none-any.whl → 0.3.58__py3-none-any.whl

inspect_ai/model/_providers/google.py

@@ -1,12 +1,17 @@
+import asyncio
 import functools
+import hashlib
 import json
 from copy import copy
+from io import BytesIO
+from logging import getLogger
 from typing import Any, cast
 
 import proto  # type: ignore
 from google.ai.generativelanguage import (
     Blob,
     Candidate,
+    File,
     FunctionCall,
     FunctionCallingConfig,
     FunctionDeclaration,
@@ -28,6 +33,8 @@ from google.generativeai import (  # type: ignore
     GenerationConfig,
     GenerativeModel,
     configure,
+    get_file,
+    upload_file,
 )
 from google.generativeai.types import (  # type: ignore
     AsyncGenerateContentResponse,
@@ -45,8 +52,16 @@ from pydantic import JsonValue
 from typing_extensions import override
 
 from inspect_ai._util.constants import BASE_64_DATA_REMOVED
-from inspect_ai._util.content import Content, ContentImage, ContentText
-from inspect_ai._util.images import image_as_data
+from inspect_ai._util.content import (
+    Content,
+    ContentAudio,
+    ContentImage,
+    ContentText,
+    ContentVideo,
+)
+from inspect_ai._util.images import file_as_data
+from inspect_ai._util.kvstore import inspect_kvstore
+from inspect_ai._util.trace import trace_message
 from inspect_ai.tool import ToolCall, ToolChoice, ToolInfo, ToolParam, ToolParams
 
 from .._chat_message import (
@@ -70,6 +85,8 @@ from .._model_output import (
 )
 from .util import model_base_url
 
+logger = getLogger(__name__)
+
 SAFETY_SETTINGS = "safety_settings"
 
 DEFAULT_SAFETY_SETTINGS: SafetySettingDict = {
@@ -364,19 +381,23 @@ def dict_to_struct(x: dict[str, Any]) -> Struct:
     return struct
 
 
-async def content_part(content: Content | str) -> PartDict:
+async def content_part(content: Content | str) -> PartType:
     if isinstance(content, str):
         return PartDict(text=content or NO_CONTENT)
     elif isinstance(content, ContentText):
         return PartDict(text=content.text or NO_CONTENT)
     else:
-        return PartDict(inline_data=await chat_content_image_to_blob(content))
+        return await chat_content_to_part(content)
 
 
-async def chat_content_image_to_blob(image: ContentImage) -> Blob:
-    image_url = image.image
-    image_bytes, mime_type = await image_as_data(image_url)
-    return Blob(mime_type=mime_type, data=image_bytes)
+async def chat_content_to_part(
+    content: ContentImage | ContentAudio | ContentVideo,
+) -> PartType:
+    if isinstance(content, ContentImage):
+        content_bytes, mime_type = await file_as_data(content.image)
+        return Blob(mime_type=mime_type, data=content_bytes)
+    else:
+        return await file_for_content(content)
 
 
 def prepend_system_messages(
@@ -630,3 +651,53 @@ def str_to_harm_block_threshold(threshold: str) -> HarmBlockThreshold:
         return HarmBlockThreshold.BLOCK_NONE
     else:
         raise ValueError(f"Unknown HarmBlockThreshold: {threshold}")
+
+
+async def file_for_content(content: ContentAudio | ContentVideo) -> File:
+    # helper to write trace messages
+    def trace(message: str) -> None:
+        trace_message(logger, "Google Files", message)
+
+    # get the file bytes and compute sha256 hash
+    if isinstance(content, ContentAudio):
+        file = content.audio
+    else:
+        file = content.video
+    content_bytes, mime_type = await file_as_data(file)
+    content_sha256 = hashlib.sha256(content_bytes).hexdigest()
+
+    # we cache uploads for re-use, open the db where we track that
+    # (track up to 1 million previous uploads)
+    with inspect_kvstore("google_files", 1000000) as files_db:
+        # can we serve from existing uploads?
+        uploaded_file = files_db.get(content_sha256)
+        if uploaded_file:
+            try:
+                upload = cast(File, get_file(uploaded_file))
+                if upload.state.name == "ACTIVE":
+                    trace(f"Using uploaded file: {uploaded_file}")
+                    return upload
+                else:
+                    trace(
+                        f"Not using uploaded file '{uploaded_file} (state was {upload.state})"
+                    )
+            except Exception as ex:
+                trace(f"Error attempting to access uploaded file: {ex}")
+                files_db.delete(content_sha256)
+
+        # do the upload (and record it)
+        upload = upload_file(BytesIO(content_bytes), mime_type=mime_type)
+        while upload.state.name == "PROCESSING":
+            await asyncio.sleep(3)
+            upload = get_file(upload.name)
+
+        if upload.state.name == "FAILED":
+            trace(f"Failed to upload file '{upload.name}: {upload.error}")
+            raise ValueError(f"Google file upload failed: {upload.error}")
+
+        # trace and record it
+        trace(f"Uploaded file: {upload.name}")
+        files_db.put(content_sha256, upload.name)
+
+        # return the file
+        return upload

inspect_ai/model/_providers/groq.py

@@ -23,8 +23,8 @@ from typing_extensions import override
 
 from inspect_ai._util.constants import DEFAULT_MAX_RETRIES, DEFAULT_MAX_TOKENS
 from inspect_ai._util.content import Content
-from inspect_ai._util.images import image_as_data_uri
-from inspect_ai._util.url import is_data_uri, is_http_url
+from inspect_ai._util.images import file_as_data_uri
+from inspect_ai._util.url import is_http_url
 from inspect_ai.tool import ToolCall, ToolChoice, ToolFunction, ToolInfo
 
 from .._chat_message import (
@@ -248,18 +248,20 @@ async def as_chat_completion_part(
 ) -> ChatCompletionContentPartParam:
     if content.type == "text":
         return ChatCompletionContentPartTextParam(type="text", text=content.text)
-    else:
+    elif content.type == "image":
         # API takes URL or base64 encoded file. If it's a remote file or data URL leave it alone, otherwise encode it
         image_url = content.image
         detail = content.detail
 
-        if not is_http_url(image_url) and not is_data_uri(image_url):
-            image_url = await image_as_data_uri(image_url)
+        if not is_http_url(image_url):
+            image_url = await file_as_data_uri(image_url)
 
         return ChatCompletionContentPartImageParam(
             type="image_url",
             image_url=dict(url=image_url, detail=detail),
         )
+    else:
+        raise RuntimeError("Groq models do not support audio or video inputs.")
 
 
 def chat_tools(tools: List[ToolInfo]) -> List[Dict[str, Any]]:

inspect_ai/model/_providers/hf.py

@@ -239,12 +239,17 @@ class HuggingFaceAPI(ModelAPI):
                 hf_messages = inspect_tools_to_string(hf_messages)
 
         # apply chat template
-        chat = self.tokenizer.apply_chat_template(
-            hf_messages,
-            add_generation_prompt=True,
-            tokenize=False,
-            tools=tools_list if len(tools_list) > 0 else None,
-        )
+        if self.tokenizer.chat_template is not None:
+            chat = self.tokenizer.apply_chat_template(
+                hf_messages,
+                add_generation_prompt=True,
+                tokenize=False,
+                tools=tools_list if len(tools_list) > 0 else None,
+            )
+        else:
+            chat = ""
+            for message in hf_messages:
+                chat += f"{message.role}: {message.content}\n"
         # return
         return cast(str, chat)
 

inspect_ai/model/_providers/mistral.py

@@ -42,8 +42,7 @@ from inspect_ai._util.constants import (
     DEFAULT_TIMEOUT,
 )
 from inspect_ai._util.content import Content, ContentImage, ContentText
-from inspect_ai._util.images import image_as_data_uri
-from inspect_ai._util.url import is_data_uri
+from inspect_ai._util.images import file_as_data_uri
 from inspect_ai.tool import ToolCall, ToolChoice, ToolFunction, ToolInfo
 
 from .._chat_message import (
@@ -351,16 +350,14 @@ def mistral_system_message_content(
 async def mistral_content_chunk(content: Content) -> ContentChunk:
     if isinstance(content, ContentText):
         return TextChunk(text=content.text or NO_CONTENT)
-    else:
+    elif isinstance(content, ContentImage):
         # resolve image to url
-        image_url = content.image
-        if not is_data_uri(image_url):
-            image_url = await image_as_data_uri(image_url)
+        image_url = await file_as_data_uri(content.image)
 
         # return chunk
-        return ImageURLChunk(
-            image_url=ImageURL(url=content.image, detail=content.detail)
-        )
+        return ImageURLChunk(image_url=ImageURL(url=image_url, detail=content.detail))
+    else:
+        raise RuntimeError("Mistral models do not support audio or video inputs.")
 
 
 def mistral_tool_call(tool_call: ToolCall) -> MistralToolCall:

inspect_ai/model/_providers/openai.py

@@ -17,6 +17,7 @@ from openai.types.chat import (
     ChatCompletion,
     ChatCompletionAssistantMessageParam,
     ChatCompletionContentPartImageParam,
+    ChatCompletionContentPartInputAudioParam,
     ChatCompletionContentPartParam,
     ChatCompletionContentPartTextParam,
     ChatCompletionDeveloperMessageParam,
@@ -36,9 +37,9 @@ from typing_extensions import override
 from inspect_ai._util.constants import DEFAULT_MAX_RETRIES
 from inspect_ai._util.content import Content
 from inspect_ai._util.error import PrerequisiteError
-from inspect_ai._util.images import image_as_data_uri
+from inspect_ai._util.images import file_as_data_uri
 from inspect_ai._util.logger import warn_once
-from inspect_ai._util.url import is_data_uri, is_http_url
+from inspect_ai._util.url import is_http_url
 from inspect_ai.tool import ToolCall, ToolChoice, ToolFunction, ToolInfo
 
 from .._chat_message import ChatMessage, ChatMessageAssistant
@@ -463,16 +464,27 @@ async def as_chat_completion_part(
 ) -> ChatCompletionContentPartParam:
     if content.type == "text":
         return ChatCompletionContentPartTextParam(type="text", text=content.text)
-    else:
+    elif content.type == "image":
         # API takes URL or base64 encoded file. If it's a remote file or
         # data URL leave it alone, otherwise encode it
         image_url = content.image
         detail = content.detail
 
-        if not is_http_url(image_url) and not is_data_uri(image_url):
-            image_url = await image_as_data_uri(image_url)
+        if not is_http_url(image_url):
+            image_url = await file_as_data_uri(image_url)
 
         return ChatCompletionContentPartImageParam(
             type="image_url",
             image_url=dict(url=image_url, detail=detail),
         )
+    elif content.type == "audio":
+        audio_data = await file_as_data_uri(content.audio)
+
+        return ChatCompletionContentPartInputAudioParam(
+            type="input_audio", input_audio=dict(data=audio_data, format=content.format)
+        )
+
+    else:
+        raise RuntimeError(
+            "Video content is not currently supported by Open AI chat models."
+        )

inspect_ai/model/_providers/vertex.py

@@ -24,8 +24,14 @@ from vertexai.generative_models import (  # type: ignore
 from vertexai.generative_models import Content as VertexContent
 
 from inspect_ai._util.constants import BASE_64_DATA_REMOVED
-from inspect_ai._util.content import Content, ContentText
-from inspect_ai._util.images import image_as_data
+from inspect_ai._util.content import (
+    Content,
+    ContentAudio,
+    ContentImage,
+    ContentText,
+    ContentVideo,
+)
+from inspect_ai._util.images import file_as_data
 from inspect_ai.tool import ToolCall, ToolChoice, ToolInfo
 
 from .._chat_message import (
@@ -308,9 +314,16 @@ async def content_part(content: Content | str) -> Part:
         return Part.from_text(content or NO_CONTENT)
     elif isinstance(content, ContentText):
         return Part.from_text(content.text or NO_CONTENT)
-    else:
-        image_bytes, mime_type = await image_as_data(content.image)
+    elif isinstance(content, ContentImage):
+        image_bytes, mime_type = await file_as_data(content.image)
         return Part.from_image(image=Image.from_bytes(data=image_bytes))
+    else:
+        if isinstance(content, ContentAudio):
+            file = content.audio
+        elif isinstance(content, ContentVideo):
+            file = content.video
+        file_bytes, mime_type = await file_as_data(file)
+        return Part.from_data(file_bytes, mime_type)
 
 
 def prepend_system_messages(

inspect_ai/scorer/__init__.py

@@ -1,3 +1,5 @@
+from inspect_ai._util.deprecation import relocated_module_attribute
+
 from ._answer import AnswerPattern, answer
 from ._choice import choice
 from ._classification import exact, f1
@@ -16,7 +18,7 @@ from ._metric import (
 )
 from ._metrics.accuracy import accuracy
 from ._metrics.mean import mean
-from ._metrics.std import bootstrap_std, std, stderr
+from ._metrics.std import bootstrap_stderr, std, stderr
 from ._model import model_graded_fact, model_graded_qa
 from ._multi import multi_scorer
 from ._pattern import pattern
@@ -50,7 +52,7 @@ __all__ = [
     "Target",
     "scorer",
     "accuracy",
-    "bootstrap_std",
+    "bootstrap_stderr",
     "std",
     "stderr",
     "mean",
@@ -76,3 +78,12 @@ __all__ = [
     "at_least",
     "pass_at",
 ]
+_BOOTSTRAP_RENAME_VERSION = "0.3.58"
+_REMOVED_IN = "0.4"
+
+relocated_module_attribute(
+    "bootstrap_std",
+    "inspect_ai.scorer.bootstrap_stderr",
+    _BOOTSTRAP_RENAME_VERSION,
+    _REMOVED_IN,
+)

inspect_ai/scorer/_metrics/__init__.py

@@ -1,12 +1,12 @@
 from .accuracy import accuracy
 from .mean import mean, var
-from .std import bootstrap_std, std, stderr
+from .std import bootstrap_stderr, std, stderr
 
 __all__ = [
     "accuracy",
     "mean",
     "var",
-    "bootstrap_std",
+    "bootstrap_stderr",
     "std",
     "stderr",
 ]

inspect_ai/scorer/_metrics/std.py

@@ -15,10 +15,10 @@ logger = getLogger(__name__)
 
 
 @metric
-def bootstrap_std(
+def bootstrap_stderr(
     num_samples: int = 1000, to_float: ValueToFloat = value_to_float()
 ) -> Metric:
-    """Standard deviation of a bootstrapped estimate of the mean.
+    """Standard error of the mean using bootstrap.
 
     Args:
        num_samples (int): Number of bootstrap samples to take.
@@ -31,7 +31,7 @@ def bootstrap_std(
          0 if the Value is a complex object (list or dict).
 
     Returns:
-       bootstrap_std metric
+       bootstrap_stderr metric
     """
 
     def metric(scores: list[Score]) -> float:

inspect_ai/tool/__init__.py

@@ -1,4 +1,10 @@
-from inspect_ai._util.content import Content, ContentImage, ContentText
+from inspect_ai._util.content import (
+    Content,
+    ContentAudio,
+    ContentImage,
+    ContentText,
+    ContentVideo,
+)
 from inspect_ai._util.deprecation import relocated_module_attribute
 
 from ._tool import Tool, ToolError, ToolResult, tool
@@ -30,8 +36,10 @@ __all__ = [
     "ToolError",
     "ToolResult",
     "Content",
+    "ContentAudio",
     "ContentImage",
     "ContentText",
+    "ContentVideo",
     "ToolCall",
     "ToolCallContent",
     "ToolCallView",

inspect_ai/tool/_tool.py

@@ -11,7 +11,12 @@ from typing import (
     runtime_checkable,
 )
 
-from inspect_ai._util.content import ContentImage, ContentText
+from inspect_ai._util.content import (
+    ContentAudio,
+    ContentImage,
+    ContentText,
+    ContentVideo,
+)
 from inspect_ai._util.registry import (
     RegistryInfo,
     registry_add,
@@ -31,7 +36,9 @@ ToolResult = (
     | bool
     | ContentText
     | ContentImage
-    | list[ContentText | ContentImage]
+    | ContentAudio
+    | ContentVideo
+    | list[ContentText | ContentImage | ContentAudio | ContentVideo]
 )
 
 

inspect_ai/util/__init__.py

@@ -26,7 +26,6 @@ from ._subprocess import (
 )
 from ._subtask import Subtask, subtask
 from ._throttle import throttle
-from ._trace import trace_enabled, trace_panel
 
 __all__ = [
     "ExecResult",
@@ -56,8 +55,6 @@ __all__ = [
     "Subtask",
     "subtask",
     "throttle",
-    "trace_enabled",
-    "trace_panel",
     "trace_action",
     "trace_message",
 ]

inspect_ai/util/{_trace.py → _conversation.py}

@@ -1,5 +1,3 @@
-from contextvars import ContextVar
-
 from rich import print
 from rich.console import RenderableType
 from rich.text import Text
@@ -7,12 +5,7 @@ from rich.text import Text
 from inspect_ai._util.transcript import transcript_panel
 
 
-def trace_enabled() -> bool:
-    """Is trace mode currently enabled."""
-    return _trace.get(None) is True
-
-
-def trace_panel(
+def conversation_panel(
     title: str,
     *,
     subtitle: str | None = None,
@@ -20,8 +13,8 @@ def trace_panel(
 ) -> None:
     """Trace content into a standard trace panel display.
 
-    Typically you would call `trace_enabled()` to confirm that trace mode
-    is enabled before calling `trace_panel()`.
+    Typically you would call `display_type() == "conversation"` to confirm that
+    we are in conversation mode before calling `conversation_panel()`.
 
     Args:
       title (str): Panel title.
@@ -32,10 +25,3 @@ def trace_panel(
         transcript_panel(title, subtitle, content),
         Text(),
     )
-
-
-def init_trace(trace: bool | None) -> None:
-    _trace.set(trace)
-
-
-_trace: ContextVar[bool | None] = ContextVar("_trace_mode")

inspect_ai/util/_display.py

@@ -3,10 +3,11 @@ from logging import getLogger
 from typing import Literal
 
 from inspect_ai._util.constants import DEFAULT_DISPLAY
+from inspect_ai._util.thread import is_main_thread
 
 logger = getLogger(__name__)
 
-DisplayType = Literal["full", "rich", "plain", "none"]
+DisplayType = Literal["full", "conversation", "rich", "plain", "none"]
 """Console display type."""
 
 
@@ -15,15 +16,24 @@ _display_type: DisplayType | None = None
 
 def init_display_type(display: str | None = None) -> DisplayType:
     global _display_type
-    global _display_metrics
     display = (
         display or os.environ.get("INSPECT_DISPLAY", DEFAULT_DISPLAY).lower().strip()
     )
+
+    # if we are on a background thread then throttle down to "plain"
+    # ("full" requires textual which cannot run in a background thread
+    # b/c it calls the Python signal function; "rich" assumes exclusive
+    # display access which may not be the case for threads)
+    if display in ["full", "rich"] and not is_main_thread():
+        display = "plain"
+
     match display:
-        case "full" | "rich" | "plain" | "none":
+        case "full" | "conversation" | "rich" | "plain" | "none":
             _display_type = display
         case _:
-            logger.warning(f"Unknown display type '{display}'")
+            logger.warning(
+                f"Unknown display type '{display}' (setting display to 'full')"
+            )
             _display_type = "full"
     return _display_type
 

inspect_ai/util/_sandbox/context.py

@@ -4,6 +4,8 @@ from typing import Any, NoReturn, cast
 
 from shortuuid import uuid
 
+from inspect_ai._util.constants import SANDBOX_SETUP_TIMEOUT
+
 from .environment import (
     SampleCleanup,
     SampleInit,
@@ -193,23 +195,20 @@ async def setup_sandbox_environment(
     setup_file = f"/tmp/{uuid()}"
     await env.write_file(setup_file, setup)
 
-    # chmod, execute, and remove
-    async def exec(cmd: list[str]) -> None:
-        try:
-            result = await env.exec(cmd, timeout=30)
-        except TimeoutError:
-            raise RuntimeError(
-                f"Timed out executing command {' '.join(cmd)} in sandbox"
-            )
-
+    # execute and then remove setup script (don't retry it on timeout
+    # in case it is not idempotent)
+    try:
+        await env.exec(["chmod", "+x", setup_file], timeout=30)
+        result = await env.exec(
+            ["env", setup_file], timeout=SANDBOX_SETUP_TIMEOUT, timeout_retry=False
+        )
         if not result.success:
             raise RuntimeError(
                 f"Failed to execute setup script for sample: {result.stderr}"
             )
-
-    await exec(["chmod", "+x", setup_file])
-    await exec(["env", setup_file])
-    await exec(["rm", setup_file])
+        await env.exec(["rm", setup_file], timeout=30)
+    except TimeoutError:
+        raise RuntimeError("Timed out executing setup command in sandbox")
 
 
 def default_sandbox_environment(

inspect_ai/util/_sandbox/docker/compose.py

@@ -25,16 +25,17 @@ COMPOSE_WAIT = "120"
 
 
 async def compose_up(project: ComposeProject) -> None:
-    # Start the environment
-    result = await compose_command(
+    # Start the environment. Note that we don't check the result because docker will
+    # return a non-zero exit code for services that exit (even successfully) when
+    # passing the --wait flag (see https://github.com/docker/compose/issues/10596).
+    # In practice, we will catch any errors when calling compose_check_running()
+    # immediately after we call compose_up().
+    await compose_command(
         ["up", "--detach", "--wait", "--wait-timeout", COMPOSE_WAIT],
         project=project,
         # wait up to 5 minutes for container to go up (compose wait + 3 minutes)
         timeout=300,
     )
-    if not result.success:
-        msg = f"Failed to start docker services for {project.config}: {result.stderr}"
-        raise RuntimeError(msg)
 
 
 async def compose_down(project: ComposeProject, quiet: bool = True) -> None:
@@ -91,14 +92,21 @@ async def compose_cp(
         raise RuntimeError(msg)
 
 
-async def compose_check_running(services: list[str], project: ComposeProject) -> None:
+async def compose_check_running(
+    services: list[str], project: ComposeProject
+) -> list[str]:
     # Check to ensure that the status of containers is healthy
     running_services = await compose_ps(project=project, status="running")
-    if len(running_services) > 0:
-        if len(running_services) != len(services):
+    exited_services = await compose_ps(project=project, status="exited")
+    successful_services = running_services + [
+        service for service in exited_services if service["ExitCode"] == 0
+    ]
+
+    if len(successful_services) > 0:
+        if len(successful_services) != len(services):
             unhealthy_services = services
-            for running_service in running_services:
-                unhealthy_services.remove(running_service["Service"])
+            for successful_service in successful_services:
+                unhealthy_services.remove(successful_service["Service"])
 
             msg = (
                 "One or more docker containers failed to start from "
@@ -108,6 +116,8 @@ async def compose_check_running(services: list[str], project: ComposeProject) ->
     else:
         raise RuntimeError("No services started")
 
+    return [service["Service"] for service in running_services]
+
 
 async def compose_ps(
     project: ComposeProject,
@@ -166,6 +176,7 @@ async def compose_exec(
     *,
     project: ComposeProject,
     timeout: int | None,
+    timeout_retry: bool = True,
     input: str | bytes | None = None,
     output_limit: int | None = None,
 ) -> ExecResult[str]:
@@ -173,6 +184,7 @@ async def compose_exec(
         ["exec"] + command,
         project=project,
         timeout=timeout,
+        timeout_retry=timeout_retry,
         input=input,
         forward_env=False,
         output_limit=output_limit,
@@ -258,6 +270,7 @@ async def compose_command(
     *,
     project: ComposeProject,
     timeout: int | None,
+    timeout_retry: bool = True,
     input: str | bytes | None = None,
     cwd: str | Path | None = None,
     forward_env: bool = True,
@@ -325,7 +338,7 @@ async def compose_command(
                 return await run_command(command_timeout)
             except TimeoutError:
                 retries += 1
-                if retries <= MAX_RETRIES:
+                if timeout_retry and (retries <= MAX_RETRIES):
                     logger.info(
                         f"Retrying docker compose command: {shlex.join(compose_command)}"
                     )