inspect-ai 0.3.52__py3-none-any.whl → 0.3.53__py3-none-any.whl

inspect_ai/_view/www/src/components/Tools.mjs

@@ -63,26 +63,12 @@ export const ToolCallView = ({
   output,
   mode,
 }) => {
-  const icon =
-    mode === "compact"
-      ? ""
-      : html`<i
-          class="bi bi-tools"
-          style=${{
-            marginRight: "0.2rem",
-            opacity: "0.4",
-          }}
-        ></i>`;
-  const codeIndent = mode === "compact" ? "" : "";
   return html`<div>
-    ${icon}
-    ${!view || view.title
-      ? html`<code style=${{ fontSize: FontSize.small }}
-          >${view?.title || functionCall}</code
-        >`
+    ${mode !== "compact" && (!view || view.title)
+      ? html`<${ToolTitle} title=${view?.title || functionCall} />`
       : ""}
     <div>
-      <div style=${{ marginLeft: `${codeIndent}` }}>
+      <div>
         <${ToolInput}
           type=${inputType}
           contents=${input}
@@ -92,7 +78,7 @@ export const ToolCallView = ({
         ${output
           ? html`
               <${ExpandablePanel} collapse=${true} border=${true} lines=${15}>
-              <${MessageContent} contents=${output} />
+              <${MessageContent} contents=${normalizeContent(output)} />
               </${ExpandablePanel}>`
           : ""}
       </div>
@@ -100,6 +86,48 @@ export const ToolCallView = ({
   </div>`;
 };
 
+/**
+ * Renders the ToolCallView component.
+ *
+ * @param {Object} props - The parameters for the component.
+ * @param {string} props.title - The title for the tool call
+ * @returns {import("preact").JSX.Element} The SampleTranscript component.
+ */
+const ToolTitle = ({ title }) => {
+  return html` <i
+      class="bi bi-tools"
+      style=${{
+        marginRight: "0.2rem",
+        opacity: "0.4",
+      }}
+    ></i>
+    <code style=${{ fontSize: FontSize.small }}>${title}</code>`;
+};
+
+/**
+ * Renders the ToolCallView component.
+ *
+ * @param {string | number | boolean | (import("../types/log").ContentText | import("../types/log").ContentImage)[]} output - The tool output
+ * @returns {(import("../Types.mjs").ContentTool | import("../types/log").ContentText | import("../types/log").ContentImage)[]} The SampleTranscript component.
+ */
+const normalizeContent = (output) => {
+  if (Array.isArray(output)) {
+    return output;
+  } else {
+    return [
+      {
+        type: "tool",
+        content: [
+          {
+            type: "text",
+            text: String(output),
+          },
+        ],
+      },
+    ];
+  }
+};
+
 /**
  * Renders the ToolInput component.
  *

inspect_ai/_view/www/src/navbar/Navbar.mjs

@@ -255,6 +255,8 @@ const ResultsPanel = ({ results }) => {
         justifyContent: "end",
         height: "100%",
         alignItems: "center",
+        maxHeight: "15em",
+        overflow: "scroll",
       }}
     >
       ${metrics.map((metric, i) => {
@@ -273,6 +275,8 @@ const ResultsPanel = ({ results }) => {
         marginTop: "0.2rem",
         paddingBottom: "0.4rem",
         rowGap: "1em",
+        maxHeight: "15em",
+        overflow: "scroll",
       }}
     >
       ${results?.scores?.map((score, index) => {
@@ -285,6 +289,14 @@ const ResultsPanel = ({ results }) => {
   }
 };
 
+/** Renders a Vertial Metric
+ *
+ * @param {Object} props - The parameters for the component.
+ * @param {import("../types/log").EvalMetric} props.metric - The metric
+ * @param {boolean} props.isFirst - Whether this is the first metric
+ *
+ * @returns {import("preact").JSX.Element} The TranscriptView component.
+ */
 const VerticalMetric = ({ metric, isFirst }) => {
   const reducer_component = metric.reducer
     ? html` <div

inspect_ai/_view/www/src/samples/SampleList.mjs

@@ -145,7 +145,7 @@ export const SampleList = (props) => {
   );
 
   const listStyle = { ...style, flex: "1", overflowY: "auto", outline: "none" };
-  const { limit, answer } = gridColumns(sampleDescriptor);
+  const { limit, answer, target } = gridColumns(sampleDescriptor);
 
   const headerRow = html`<div
     style=${{
@@ -161,7 +161,7 @@ export const SampleList = (props) => {
   >
     <div>Id</div>
     <div>Input</div>
-    <div>Target</div>
+    <div>${target !== "0" ? "Target" : ""}</div>
     <div>${answer !== "0" ? "Answer" : ""}</div>
     <div>${limit !== "0" ? "Limit" : ""}</div>
     <div style=${{ justifySelf: "center" }}>Score</div>

inspect_ai/_view/www/src/samples/transcript/ToolEventView.mjs

@@ -29,10 +29,10 @@ export const ToolEventView = ({ id, event, style, depth }) => {
     return e.event === "approval";
   });
 
-  const title = `Tool: ${event.function}`;
+  const title = `Tool: ${event.view?.title || event.function}`;
   return html`
   <${EventPanel} id=${id} title="${title}" subTitle=${formatDateTime(new Date(event.timestamp))} icon=${ApplicationIcons.solvers.use_tools} style=${style}>  
-  <div name="Summary" style=${{ margin: "0.5em 0" }}>
+  <div name="Summary" style=${{ margin: "0.5em 0", width: "100%" }}>
     <${ToolCallView}
       functionCall=${functionCall}
       input=${input}

inspect_ai/log/_log.py

@@ -79,6 +79,9 @@ class EvalConfig(BaseModel):
     max_subprocesses: int | None = Field(default=None)
     """Maximum number of subprocesses to run concurrently."""
 
+    max_sandboxes: int | None = Field(default=None)
+    """Maximum number of sandboxes to run concurrently."""
+
     sandbox_cleanup: bool | None = Field(default=None)
     """Cleanup sandbox environments after task completes."""
 

inspect_ai/log/_recorders/eval.py

@@ -362,13 +362,14 @@ class ZipLogFile:
                     f"Error occurred during async write to {self._file}: {ex}. Falling back to sync write."
                 )
 
-            # write sync if we need to
-            if not written:
-                with file(self._file, "wb") as f:
-                    f.write(log_bytes)
-
-            # re-open zip file w/ self.temp_file pointer at end
-            self._open()
+            try:
+                # write sync if we need to
+                if not written:
+                    with file(self._file, "wb") as f:
+                        f.write(log_bytes)
+            finally:
+                # re-open zip file w/ self.temp_file pointer at end
+                self._open()
 
     async def close(self) -> EvalLog:
         async with self._lock:

inspect_ai/model/_generate_config.py

@@ -72,6 +72,9 @@ class GenerateConfigArgs(TypedDict, total=False):
     cache_prompt: Literal["auto"] | bool | None
     """Whether to cache the prompt prefix. Defaults to "auto", which will enable caching for requests with tools. Anthropic only."""
 
+    reasoning_effort: Literal["low", "medium", "high"] | None
+    """Constrains effort on reasoning for reasoning models. Open AI o1 models only."""
+
 
 class GenerateConfig(BaseModel):
     """Base class for model generation configs."""
@@ -139,6 +142,9 @@ class GenerateConfig(BaseModel):
     cache_prompt: Literal["auto"] | bool | None = Field(default=None)
     """Whether to cache the prompt prefix. Defaults to "auto", which will enable caching for requests with tools. Anthropic only."""
 
+    reasoning_effort: Literal["low", "medium", "high"] | None = Field(default=None)
+    """Constrains effort on reasoning for reasoning models. Open AI o1 models only."""
+
     def merge(
         self, other: Union["GenerateConfig", GenerateConfigArgs]
     ) -> "GenerateConfig":

inspect_ai/model/_providers/azureai.py

@@ -93,7 +93,7 @@ class AzureAIAPI(ModelAPI):
         def collect_model_arg(name: str) -> Any | None:
             nonlocal model_args
             value = model_args.get(name, None)
-            if value:
+            if value is not None:
                 model_args.pop(name)
             return value
 

inspect_ai/model/_providers/bedrock.py

@@ -236,15 +236,21 @@ class BedrockAPI(ModelAPI):
         self,
         model_name: str,
         base_url: str | None,
+        api_key: str | None = None,
         config: GenerateConfig = GenerateConfig(),
         **model_args: Any,
     ):
         super().__init__(
             model_name=model_name,
             base_url=model_base_url(base_url, "BEDROCK_BASE_URL"),
+            api_key=api_key,
+            api_key_vars=[],
             config=config,
         )
 
+        # save model_args
+        self.model_args = model_args
+
         # import aioboto3 on demand
         try:
             import aioboto3
@@ -263,6 +269,9 @@ class BedrockAPI(ModelAPI):
 
     @override
     def max_tokens(self) -> int | None:
+        if "llama3-70" in self.model_name or "llama3-8" in self.model_name:
+            return 2048
+
         if "llama3" in self.model_name or "claude3" in self.model_name:
             return 4096
 
@@ -316,6 +325,7 @@ class BedrockAPI(ModelAPI):
                     mode="adaptive",
                 ),
             ),
+            **self.model_args,
         ) as client:
             # Process the tools
             resolved_tools = converse_tools(tools)
@@ -658,6 +668,8 @@ def converse_image_type(type: str) -> ConverseImageFormat:
             return "png"
         case "image/webp":
             return "webp"
+        case "image/jpeg":
+            return "jpeg"
         case _:
             raise ValueError(
                 f"Image mime type {type} is not supported for Bedrock Converse models."
@@ -673,7 +685,11 @@ def converse_tools(tools: list[ToolInfo]) -> list[ConverseTool] | None:
         tool_spec = ConverseToolSpec(
             name=tool.name,
             description=tool.description,
-            inputSchema={"json": tool.parameters.model_dump(exclude_none=True)},
+            inputSchema={
+                "json": tool.parameters.model_dump(
+                    exclude_none=True, exclude={"additionalProperties"}
+                )
+            },
         )
         result.append(ConverseTool(toolSpec=tool_spec))
     return result

inspect_ai/model/_providers/hf.py

@@ -64,7 +64,7 @@ class HuggingFaceAPI(ModelAPI):
         def collect_model_arg(name: str) -> Any | None:
             nonlocal model_args
             value = model_args.get(name, None)
-            if value:
+            if value is not None:
                 model_args.pop(name)
             return value
 

inspect_ai/model/_providers/openai.py

@@ -18,6 +18,7 @@ from openai.types.chat import (
     ChatCompletionContentPartImageParam,
     ChatCompletionContentPartParam,
     ChatCompletionContentPartTextParam,
+    ChatCompletionDeveloperMessageParam,
     ChatCompletionMessage,
     ChatCompletionMessageParam,
     ChatCompletionMessageToolCallParam,
@@ -141,6 +142,18 @@ class OpenAIAPI(ModelAPI):
                 **model_args,
             )
 
+    def is_o1(self) -> bool:
+        return self.model_name.startswith("o1")
+
+    def is_o1_full(self) -> bool:
+        return self.is_o1() and not self.is_o1_mini() and not self.is_o1_preview()
+
+    def is_o1_mini(self) -> bool:
+        return self.model_name.startswith("o1-mini")
+
+    def is_o1_preview(self) -> bool:
+        return self.model_name.startswith("o1-preview")
+
     async def generate(
         self,
         input: list[ChatMessage],
@@ -148,8 +161,8 @@ class OpenAIAPI(ModelAPI):
         tool_choice: ToolChoice,
         config: GenerateConfig,
     ) -> ModelOutput | tuple[ModelOutput, ModelCall]:
-        # short-circuit to call o1- model
-        if self.model_name.startswith("o1-"):
+        # short-circuit to call o1- models that are text only
+        if self.is_o1_preview() or self.is_o1_mini():
             return await generate_o1(
                 client=self.client,
                 input=input,
@@ -179,7 +192,7 @@ class OpenAIAPI(ModelAPI):
 
         # prepare request (we do this so we can log the ModelCall)
         request = dict(
-            messages=await as_openai_chat_messages(input),
+            messages=await as_openai_chat_messages(input, self.is_o1_full()),
             tools=chat_tools(tools) if len(tools) > 0 else NOT_GIVEN,
             tool_choice=chat_tool_choice(tool_choice) if len(tools) > 0 else NOT_GIVEN,
             **self.completion_params(config, len(tools) > 0),
@@ -271,8 +284,10 @@ class OpenAIAPI(ModelAPI):
             params["logprobs"] = config.logprobs
         if config.top_logprobs is not None:
             params["top_logprobs"] = config.top_logprobs
-        if tools and config.parallel_tool_calls is not None:
+        if tools and config.parallel_tool_calls is not None and not self.is_o1():
             params["parallel_tool_calls"] = config.parallel_tool_calls
+        if config.reasoning_effort is not None and self.is_o1_full():
+            params["reasoning_effort"] = config.reasoning_effort
 
         return params
 
@@ -291,14 +306,23 @@ class OpenAIAPI(ModelAPI):
 
 
 async def as_openai_chat_messages(
-    messages: list[ChatMessage],
+    messages: list[ChatMessage], o1_full: bool
 ) -> list[ChatCompletionMessageParam]:
-    return [await openai_chat_message(message) for message in messages]
+    return [await openai_chat_message(message, o1_full) for message in messages]
 
 
-async def openai_chat_message(message: ChatMessage) -> ChatCompletionMessageParam:
+async def openai_chat_message(
+    message: ChatMessage, o1_full: bool
+) -> ChatCompletionMessageParam:
     if message.role == "system":
-        return ChatCompletionSystemMessageParam(role=message.role, content=message.text)
+        if o1_full:
+            return ChatCompletionDeveloperMessageParam(
+                role="developer", content=message.text
+            )
+        else:
+            return ChatCompletionSystemMessageParam(
+                role=message.role, content=message.text
+            )
     elif message.role == "user":
         return ChatCompletionUserMessageParam(
             role=message.role,

inspect_ai/model/_providers/providers.py

@@ -242,7 +242,7 @@ def mockllm() -> type[ModelAPI]:
 def validate_openai_client(feature: str) -> None:
     FEATURE = feature
     PACKAGE = "openai"
-    MIN_VERSION = "1.45.0"
+    MIN_VERSION = "1.58.1"
 
     # verify we have the package
     try:

inspect_ai/model/_providers/vllm.py

@@ -75,7 +75,7 @@ class VLLMAPI(ModelAPI):
         def collect_model_arg(name: str) -> Any | None:
             nonlocal model_args
             value = model_args.get(name, None)
-            if value:
+            if value is not None:
                 model_args.pop(name)
             return value
 

inspect_ai/util/_sandbox/context.py

@@ -109,7 +109,7 @@ def raise_no_sandbox() -> NoReturn:
 
 
 async def init_sandbox_environments_sample(
-    type: str,
+    sandboxenv_type: type[SandboxEnvironment],
     task_name: str,
     config: SandboxEnvironmentConfigType | None,
     files: dict[str, bytes],
@@ -117,7 +117,6 @@ async def init_sandbox_environments_sample(
     metadata: dict[str, Any],
 ) -> dict[str, SandboxEnvironment]:
     # get setup and cleanup functions
-    sandboxenv_type = registry_find_sandboxenv(type)
     sample_init = cast(SampleInit, getattr(sandboxenv_type, "sample_init"))
     sample_cleanup = cast(SampleCleanup, getattr(sandboxenv_type, "sample_cleanup"))
 

inspect_ai/util/_sandbox/docker/config.py

@@ -2,8 +2,6 @@ import os
 from logging import getLogger
 from pathlib import Path
 
-import aiofiles
-
 logger = getLogger(__name__)
 
 
@@ -17,7 +15,7 @@ CONFIG_FILES = [
 DOCKERFILE = "Dockerfile"
 
 
-async def resolve_compose_file(parent: str = "") -> str:
+def resolve_compose_file(parent: str = "") -> str:
     # existing compose file provides all the config we need
     compose = find_compose_file(parent)
     if compose is not None:
@@ -29,11 +27,11 @@ async def resolve_compose_file(parent: str = "") -> str:
 
     # dockerfile just needs a compose.yaml synthesized
     elif has_dockerfile(parent):
-        return await auto_compose_file(COMPOSE_DOCKERFILE_YAML, parent)
+        return auto_compose_file(COMPOSE_DOCKERFILE_YAML, parent)
 
     # otherwise provide a generic python container
     else:
-        return await auto_compose_file(COMPOSE_GENERIC_YAML, parent)
+        return auto_compose_file(COMPOSE_GENERIC_YAML, parent)
 
 
 def find_compose_file(parent: str = "") -> str | None:
@@ -59,9 +57,9 @@ def is_auto_compose_file(file: str) -> bool:
     return os.path.basename(file) == AUTO_COMPOSE_YAML
 
 
-async def ensure_auto_compose_file(file: str | None) -> None:
+def ensure_auto_compose_file(file: str | None) -> None:
     if file is not None and is_auto_compose_file(file) and not os.path.exists(file):
-        await resolve_compose_file(os.path.dirname(file))
+        resolve_compose_file(os.path.dirname(file))
 
 
 def safe_cleanup_auto_compose(file: str | None) -> None:
@@ -100,8 +98,8 @@ services:
 """
 
 
-async def auto_compose_file(contents: str, parent: str = "") -> str:
+def auto_compose_file(contents: str, parent: str = "") -> str:
     path = os.path.join(parent, AUTO_COMPOSE_YAML)
-    async with aiofiles.open(path, "w", encoding="utf-8") as f:
-        await f.write(contents)
+    with open(path, "w", encoding="utf-8") as f:
+        f.write(contents)
     return Path(path).resolve().as_posix()

inspect_ai/util/_sandbox/docker/docker.py

@@ -5,7 +5,6 @@ from logging import getLogger
 from pathlib import Path, PurePosixPath
 from typing import Literal, Union, cast, overload
 
-import aiofiles
 from typing_extensions import override
 
 from inspect_ai.util._subprocess import ExecResult
@@ -54,6 +53,11 @@ class DockerSandboxEnvironment(SandboxEnvironment):
     def config_files(cls) -> list[str]:
         return CONFIG_FILES + [DOCKERFILE]
 
+    @classmethod
+    def default_concurrency(cls) -> int | None:
+        count = os.cpu_count() or 1
+        return 2 * count
+
     @classmethod
     async def task_init(
         cls, task_name: str, config: SandboxEnvironmentConfigType | None
@@ -403,11 +407,11 @@ class DockerSandboxEnvironment(SandboxEnvironment):
 
             # read and return w/ appropriate encoding
             if text:
-                async with aiofiles.open(dest_file, "r", encoding="utf-8") as f:
-                    return await f.read()
+                with open(dest_file, "r", encoding="utf-8") as f:
+                    return f.read()
             else:
-                async with aiofiles.open(dest_file, "rb") as f:
-                    return await f.read()
+                with open(dest_file, "rb") as f:
+                    return f.read()
 
     @override
     async def connection(self) -> SandboxConnection:

inspect_ai/util/_sandbox/docker/util.py

@@ -41,7 +41,7 @@ class ComposeProject:
 
         # if its a Dockerfile, then config is the auto-generated .compose.yaml
         if config_path and is_dockerfile(config_path.name):
-            config = await auto_compose_file(
+            config = auto_compose_file(
                 COMPOSE_DOCKERFILE_YAML, config_path.parent.as_posix()
             )
 
@@ -51,12 +51,12 @@ class ComposeProject:
 
         # no config passed, look for 'auto-config' (compose.yaml, Dockerfile, etc.)
         else:
-            config = await resolve_compose_file()
+            config = resolve_compose_file()
 
         # this could be a cleanup where docker has tracked a .compose.yaml file
         # as part of its ConfigFiles and passed it back to us -- we in the
         # meantime have cleaned it up so we re-create it here as required
-        await ensure_auto_compose_file(config)
+        ensure_auto_compose_file(config)
 
         # return project
         return ComposeProject(name, config, env)

inspect_ai/util/_sandbox/environment.py

@@ -53,6 +53,11 @@ class SandboxEnvironment(abc.ABC):
         """Standard config files for this provider (used for automatic discovery)"""
         return []
 
+    @classmethod
+    def default_concurrency(cls) -> int | None:
+        """Default max_sandboxes for this provider (`None` means no maximum)"""
+        return None
+
     @classmethod
     async def task_init(
         cls, task_name: str, config: SandboxEnvironmentConfigType | None
@@ -143,7 +148,7 @@ class SandboxEnvironment(abc.ABC):
         The current working directory for execution will be the per-sample
         filesystem context.
 
-        Each output stream (stdout and stderr) is limited to 1 MiB. If exceeded, an
+        Each output stream (stdout and stderr) is limited to 10 MiB. If exceeded, an
         `OutputLimitExceededError` will be raised.
 
         Args:
@@ -164,7 +169,7 @@ class SandboxEnvironment(abc.ABC):
           PermissionError: If the user does not have
             permission to execute the command.
           OutputLimitExceededError: If an output stream
-            exceeds the 1 MiB limit.
+            exceeds the 10 MiB limit.
         """
         ...
 

inspect_ai/util/_sandbox/limits.py

@@ -29,7 +29,7 @@ def verify_exec_result_size(exec_result: ExecResult[str]) -> None:
     """Verify the size of the output streams in an `ExecResult`.
 
     Raises:
-      OutputLimitExceededError: If an output stream exceeds the 1 MiB limit.
+      OutputLimitExceededError: If an output stream exceeds the limit.
     """
     limit = SandboxEnvironmentLimits.MAX_EXEC_OUTPUT_SIZE
     stdout_truncated = truncate_string_to_bytes(exec_result.stdout, limit)

inspect_ai/util/_sandbox/local.py

@@ -3,7 +3,6 @@ import warnings
 from pathlib import Path
 from typing import Literal, Union, cast, overload
 
-import aiofiles
 from typing_extensions import override
 
 from .._subprocess import ExecResult, subprocess
@@ -85,11 +84,11 @@ class LocalSandboxEnvironment(SandboxEnvironment):
         Path(file).parent.mkdir(parents=True, exist_ok=True)
 
         if isinstance(contents, str):
-            async with aiofiles.open(file, "w", encoding="utf-8") as f:
-                await f.write(contents)
+            with open(file, "w", encoding="utf-8") as f:
+                f.write(contents)
         else:
-            async with aiofiles.open(file, "wb") as f:
-                await f.write(contents)
+            with open(file, "wb") as f:
+                f.write(contents)
 
     @overload
     async def read_file(self, file: str, text: Literal[True] = True) -> str: ...
@@ -102,11 +101,11 @@ class LocalSandboxEnvironment(SandboxEnvironment):
         file = self._resolve_file(file)
         verify_read_file_size(file)
         if text:
-            async with aiofiles.open(file, "r", encoding="utf-8") as f:
-                return await f.read()
+            with open(file, "r", encoding="utf-8") as f:
+                return f.read()
         else:
-            async with aiofiles.open(file, "rb") as f:
-                return await f.read()
+            with open(file, "rb") as f:
+                return f.read()
 
     def _resolve_file(self, file: str) -> str:
         path = Path(file)

{inspect_ai-0.3.52.dist-info → inspect_ai-0.3.53.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: inspect_ai
-Version: 0.3.52
+Version: 0.3.53
 Summary: Framework for large language model evaluations
 Author: UK AI Safety Institute
 License: MIT License
@@ -20,7 +20,6 @@ Classifier: Operating System :: OS Independent
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: aiofiles
 Requires-Dist: aiohttp>=3.9.0
 Requires-Dist: anyio>=4.4.0
 Requires-Dist: beautifulsoup4
@@ -71,7 +70,6 @@ Requires-Dist: pytest-xdist; extra == "dev"
 Requires-Dist: ruff==0.8.3; extra == "dev"
 Requires-Dist: textual-dev>=0.86.2; extra == "dev"
 Requires-Dist: types-PyYAML; extra == "dev"
-Requires-Dist: types-aiofiles; extra == "dev"
 Requires-Dist: types-beautifulsoup4; extra == "dev"
 Requires-Dist: types-aioboto3; extra == "dev"
 Requires-Dist: types-boto3; extra == "dev"