inspect-ai 0.3.56__py3-none-any.whl → 0.3.57__py3-none-any.whl

inspect_ai/_view/www/src/samples/transcript/ToolEventView.mjs

@@ -37,7 +37,7 @@ export const ToolEventView = ({ id, event, style, depth }) => {
       functionCall=${functionCall}
       input=${input}
       inputType=${inputType}
-      output=${event.result}
+      output=${event.error?.message || event.result}
       mode="compact"
       view=${event.view}
       />

inspect_ai/_view/www/src/types/log.d.ts

@@ -396,7 +396,7 @@ export type Answer1 = string | null;
 export type Explanation2 = string | null;
 export type Metadata8 = {} | null;
 export type SampleId1 = string | number | null;
-export type Samples2 = SampleScore[];
+export type Samples2 = EvalSampleScore[];
 export type Location1 = string;
 
 export interface EvalLog {
@@ -1034,13 +1034,7 @@ export interface EvalSampleReductions {
   reducer: Reducer1;
   samples: Samples2;
 }
-/**
- * Score for a Sample
- *
- * Args:
- *    sample_id: (str | int | None) Unique id of a sample
- */
-export interface SampleScore {
+export interface EvalSampleScore {
   value: Value2;
   answer: Answer1;
   explanation: Explanation2;

inspect_ai/_view/www/src/workspace/WorkSpace.mjs

@@ -150,7 +150,7 @@ export const WorkSpace = ({
 
     // The samples tab
     // Currently only appears when the result is successful
-    if (evalStatus !== "error" && sampleMode !== "none") {
+    if (sampleMode !== "none") {
       resolvedTabs.samples = {
         id: kEvalWorkspaceTabId,
         scrollable: samples.length === 1,

inspect_ai/log/_log.py

@@ -16,6 +16,7 @@ from inspect_ai._util.constants import CONSOLE_DISPLAY_WIDTH, PKG_NAME
 from inspect_ai._util.error import EvalError, exception_message
 from inspect_ai._util.logger import warn_once
 from inspect_ai.approval._policy import ApprovalPolicyConfig
+from inspect_ai.dataset._dataset import MT, metadata_as
 from inspect_ai.model import (
     ChatMessage,
     GenerateConfig,
@@ -24,6 +25,8 @@ from inspect_ai.model import (
 )
 from inspect_ai.scorer import Score
 from inspect_ai.util._sandbox.environment import SandboxEnvironmentSpec
+from inspect_ai.util._store import Store
+from inspect_ai.util._store_model import SMT
 
 from ._transcript import Event
 
@@ -158,9 +161,31 @@ class EvalSample(BaseModel):
     metadata: dict[str, Any]
     """Additional sample metadata."""
 
+    def metadata_as(self, metadata_cls: Type[MT]) -> MT:
+        """Pydantic model interface to metadata.
+
+        Args:
+          metadata_cls: Pydantic model type
+
+        Returns:
+          BaseModel: Instance of metadata_cls bound to sample metadata.
+        """
+        return metadata_as(self.metadata, metadata_cls)
+
     store: dict[str, Any] = Field(default_factory=dict)
     """State at end of sample execution."""
 
+    def store_as(self, model_cls: Type[SMT]) -> SMT:
+        """Pydantic model interface to the store.
+
+        Args:
+          model_cls: Pydantic model type (must derive from StoreModel)
+
+        Returns:
+          StoreModel: Instance of model_cls bound to sample store data.
+        """
+        return model_cls(store=Store(self.store))
+
     events: list[Event] = Field(default_factory=list)
     """Events that occurred during sample execution."""
 

inspect_ai/log/_recorders/eval.py

@@ -252,6 +252,8 @@ def text_inputs(inputs: str | list[ChatMessage]) -> str | list[ChatMessage]:
                     filtered_content.append(ContentText(text="(Image)"))
                 message.content = filtered_content
                 input.append(message)
+            else:
+                input.append(message)
 
         return input
     else:

inspect_ai/model/_call_tools.py

@@ -1,15 +1,20 @@
 import asyncio
 import inspect
+import types
 from dataclasses import is_dataclass
 from logging import getLogger
 from textwrap import dedent
+from types import UnionType
 from typing import (
     Any,
     Callable,
     Dict,
     List,
     NamedTuple,
+    Optional,
+    Tuple,
     Type,
+    Union,
     get_args,
     get_origin,
     get_type_hints,
@@ -25,10 +30,7 @@ from inspect_ai._util.text import truncate_string_to_bytes
 from inspect_ai._util.trace import trace_action
 from inspect_ai.model._trace import trace_tool_mesage
 from inspect_ai.tool import Tool, ToolCall, ToolError, ToolInfo
-from inspect_ai.tool._tool import (
-    ToolApprovalError,
-    ToolParsingError,
-)
+from inspect_ai.tool._tool import ToolApprovalError, ToolParsingError
 from inspect_ai.tool._tool_call import ToolCallContent, ToolCallError
 from inspect_ai.tool._tool_def import ToolDef, tool_defs
 from inspect_ai.tool._tool_info import parse_docstring
@@ -268,6 +270,16 @@ def disable_parallel_tools(
     return False
 
 
+def type_hint_includes_none(type_hint: Type[Any] | None) -> bool:
+    origin = get_origin(type_hint)
+
+    if origin in {Union, UnionType}:
+        return type(None) in get_args(type_hint)
+    elif origin is Optional:
+        return True
+    return False
+
+
 def tool_params(input: dict[str, Any], func: Callable[..., Any]) -> dict[str, Any]:
     # parse function typeinfo
     signature = inspect.signature(func)
@@ -296,7 +308,7 @@ def tool_params(input: dict[str, Any], func: Callable[..., Any]) -> dict[str, An
         # yield parameter (fail if not passed and there is no default)
         if param_name in input:
             params[param_name] = tool_param(type_hint, input.get(param_name))
-        elif param.default is not None:
+        elif param.default is not None or type_hint_includes_none(type_hint):
             params[param_name] = param.default
         else:
             raise ToolParsingError(
@@ -339,11 +351,21 @@ def tool_param(type_hint: Type[Any], input: Any) -> Any:
             return [tool_param(args[0], x) for x in input]
         else:
             return input
+    elif origin is tuple or origin is Tuple:
+        if args:
+            return tuple([tool_param(args[0], x) for x in input])
+        else:
+            return tuple(input)
     elif origin is dict or origin is Dict:
         if args and len(args) > 1:
             return {k: tool_param(args[1], v) for k, v in input}
         else:
             return input
+    elif origin is Union or origin is types.UnionType:
+        if args[1] is type(None):
+            return tool_param(args[0], input)
+        else:
+            return input
     else:
         return input
 

inspect_ai/model/_providers/google.py

@@ -194,7 +194,9 @@ class GoogleAPI(ModelAPI):
                 model=self.model_name, content=ex.message, stop_reason="model_length"
             )
         else:
-            raise ex
+            return ModelOutput.from_content(
+                model=self.model_name, content=ex.message, stop_reason="unknown"
+            )
 
     @override
     def is_rate_limit(self, ex: BaseException) -> bool:
@@ -408,25 +410,34 @@ def chat_tools(tools: list[ToolInfo]) -> list[Tool]:
 # https://ai.google.dev/gemini-api/tutorials/extract_structured_data#define_the_schema
 
 
-def schema_from_param(param: ToolParam | ToolParams) -> Schema:
+def schema_from_param(param: ToolParam | ToolParams, nullable: bool = False) -> Schema:
     if isinstance(param, ToolParams):
         param = ToolParam(
             type=param.type, properties=param.properties, required=param.required
         )
 
     if param.type == "number":
-        return Schema(type=Type.NUMBER, description=param.description)
+        return Schema(
+            type=Type.NUMBER, description=param.description, nullable=nullable
+        )
     elif param.type == "integer":
-        return Schema(type=Type.INTEGER, description=param.description)
+        return Schema(
+            type=Type.INTEGER, description=param.description, nullable=nullable
+        )
     elif param.type == "boolean":
-        return Schema(type=Type.BOOLEAN, description=param.description)
+        return Schema(
+            type=Type.BOOLEAN, description=param.description, nullable=nullable
+        )
     elif param.type == "string":
-        return Schema(type=Type.STRING, description=param.description)
+        return Schema(
+            type=Type.STRING, description=param.description, nullable=nullable
+        )
     elif param.type == "array":
         return Schema(
             type=Type.ARRAY,
             description=param.description,
             items=schema_from_param(param.items) if param.items else None,
+            nullable=nullable,
         )
     elif param.type == "object":
         return Schema(
@@ -436,7 +447,14 @@ def schema_from_param(param: ToolParam | ToolParams) -> Schema:
             if param.properties is not None
             else None,
             required=param.required,
+            nullable=nullable,
         )
+    # convert unions to optional params if the second type is 'null'
+    elif param.anyOf:
+        if len(param.anyOf) == 2 and param.anyOf[1].type == "null":
+            return schema_from_param(param.anyOf[0], nullable=True)
+        else:
+            return Schema(type=Type.TYPE_UNSPECIFIED)
     else:
         return Schema(type=Type.TYPE_UNSPECIFIED)
 

inspect_ai/model/_providers/openai.py

@@ -51,6 +51,7 @@ from .._model_output import (
     Logprobs,
     ModelOutput,
     ModelUsage,
+    StopReason,
 )
 from .openai_o1 import generate_o1
 from .util import (
@@ -262,7 +263,10 @@ class OpenAIAPI(ModelAPI):
             model=self.model_name,
         )
         if config.max_tokens is not None:
-            params["max_tokens"] = config.max_tokens
+            if self.is_o1():
+                params["max_completion_tokens"] = config.max_tokens
+            else:
+                params["max_tokens"] = config.max_tokens
         if config.frequency_penalty is not None:
             params["frequency_penalty"] = config.frequency_penalty
         if config.stop_seqs is not None:
@@ -303,13 +307,23 @@ class OpenAIAPI(ModelAPI):
 
     # convert some well known bad request errors into ModelOutput
     def handle_bad_request(self, e: BadRequestError) -> ModelOutput:
-        if e.status_code == 400 and e.code == "context_length_exceeded":
+        if e.status_code == 400:
+            # extract message
             if isinstance(e.body, dict) and "message" in e.body.keys():
                 content = str(e.body.get("message"))
             else:
                 content = e.message
+
+            # narrow stop_reason
+            if e.code == "context_length_exceeded":
+                stop_reason: StopReason = "model_length"
+            elif e.code == "invalid_prompt":
+                stop_reason = "content_filter"
+            else:
+                stop_reason = "unknown"
+
             return ModelOutput.from_content(
-                model=self.model_name, content=content, stop_reason="model_length"
+                model=self.model_name, content=content, stop_reason=stop_reason
             )
         else:
             raise e

inspect_ai/model/_providers/openai_o1.py

@@ -25,7 +25,7 @@ from inspect_ai.model import (
 from inspect_ai.tool import ToolCall, ToolInfo
 
 from .._model_call import ModelCall
-from .._model_output import ModelUsage
+from .._model_output import ModelUsage, StopReason
 from .._providers.util import (
     ChatAPIHandler,
     ChatAPIMessage,
@@ -48,12 +48,6 @@ async def generate_o1(
     # create chatapi handler
     handler = O1PreviewChatAPIHandler()
 
-    # map max_tokens => max_completion_tokens
-    max_tokens = params.get("max_tokens", None)
-    if max_tokens:
-        params["max_completion_tokens"] = max_tokens
-        del params["max_tokens"]
-
     # call model
     request = dict(
         model=model,
@@ -89,12 +83,16 @@ async def generate_o1(
 
 
 def handle_bad_request(model: str, ex: BadRequestError) -> ModelOutput:
-    if ex.code == "invalid_prompt":
-        return ModelOutput.from_content(
-            model=model, content=str(ex), stop_reason="content_filter"
-        )
+    if ex.code == "context_length_exceeded":
+        stop_reason: StopReason = "model_length"
+    elif ex.code == "invalid_prompt":
+        stop_reason = "content_filter"
     else:
-        raise ex
+        stop_reason = "unknown"
+
+    return ModelOutput.from_content(
+        model=model, content=str(ex), stop_reason=stop_reason
+    )
 
 
 def chat_messages(

inspect_ai/tool/_tool_info.py

@@ -8,6 +8,7 @@ from typing import (
     Dict,
     List,
     Optional,
+    Tuple,
     Type,
     Union,
     get_args,
@@ -155,7 +156,7 @@ def parse_type(type_hint: Type[Any]) -> ToolParam:
             return ToolParam(type="null")
         else:
             return ToolParam()
-    elif origin is list or origin is List:
+    elif origin is list or origin is List or origin is tuple or origin is Tuple:
         return ToolParam(
             type="array", items=parse_type(args[0]) if args else ToolParam()
         )

inspect_ai/tool/_tools/_web_browser/_resources/dm_env_servicer.py

@@ -38,9 +38,9 @@ class EnvironmentSpec:
             for i, obs_spec in enumerate(env_obs_spec.values()):
                 self.observation_spec[i + 1] = convert(obs_spec)
 
-        assert isinstance(
-            env.action_spec(), specs.Array
-        ), "Only a single action type is supported."
+        assert isinstance(env.action_spec(), specs.Array), (
+            "Only a single action type is supported."
+        )
         self.action_spec = {1: convert(env.action_spec())}
 
         self.observation_manager = spec_manager.SpecManager(self.observation_spec)
@@ -234,12 +234,12 @@ class EnvironmentService(dm_env_rpc_pb2_grpc.EnvironmentServicer):
             observations.
         """
         with self._lock:
-            assert (
-                cur_world in self._envs
-            ), "Current world does not have an assosiated environment"
-            assert (
-                cur_world in self._joined_worlds
-            ), "Please join world before calling step."
+            assert cur_world in self._envs, (
+                "Current world does not have an assosiated environment"
+            )
+            assert cur_world in self._joined_worlds, (
+                "Please join world before calling step."
+            )
             env = self._envs[cur_world]
             spec = self._specs[cur_world]
 

inspect_ai/tool/_tools/_web_browser/_web_browser.py

@@ -372,7 +372,9 @@ async def web_browser_cmd(cmd: str, *args: str) -> str:
         )
     else:
         response = parse_web_browser_output(result.stdout)
-        if "web_at" in response:
+        if "error" in response and response.get("error", "").strip() != "":
+            raise ToolError(str(response.get("error")) or "(unknown error)")
+        elif "web_at" in response:
             web_at = (
                 str(response.get("web_at")) or "(no web accessiblity tree available)"
             )
@@ -384,8 +386,6 @@ async def web_browser_cmd(cmd: str, *args: str) -> str:
             web_at = "\n".join(web_at_lines)
             store_as(WebBrowserStore).web_at = web_at
             return web_at
-        elif "error" in response:
-            raise ToolError(str(response.get("error")) or "(unknown error)")
         else:
             raise RuntimeError(
                 f"web_browser output must contain either 'error' or 'web_at' field: {result.stdout}"

inspect_ai/util/__init__.py

@@ -1,3 +1,5 @@
+from inspect_ai._util.trace import trace_action, trace_message
+
 from ._concurrency import concurrency
 from ._console import input_screen
 from ._display import DisplayType, display_type
@@ -56,4 +58,6 @@ __all__ = [
     "throttle",
     "trace_enabled",
     "trace_panel",
+    "trace_action",
+    "trace_message",
 ]

inspect_ai/util/_sandbox/docker/compose.py

@@ -33,9 +33,7 @@ async def compose_up(project: ComposeProject) -> None:
         timeout=300,
     )
     if not result.success:
-        msg = (
-            f"Failed to start docker services for {project.config}: " f"{result.stderr}"
-        )
+        msg = f"Failed to start docker services for {project.config}: {result.stderr}"
         raise RuntimeError(msg)
 
 

inspect_ai/util/_sandbox/docker/util.py

@@ -84,7 +84,8 @@ def task_project_name(task: str) -> str:
     if len(task) == 0:
         task = "task"
 
-    return f"inspect-{task[:12]}-i{uuid().lower()[:6]}"
+    # _- breaks docker project name constraints so we strip trailing underscores.
+    return f"inspect-{task[:12].rstrip('_')}-i{uuid().lower()[:6]}"
 
 
 inspect_project_pattern = r"^inspect-[a-z\d\-_]*-i[a-z\d]{6,}$"

inspect_ai/util/_sandbox/self_check.py

@@ -75,9 +75,9 @@ async def test_read_and_write_file_text(sandbox_env: SandboxEnvironment) -> None
     written_file_string = await sandbox_env.read_file(
         "test_read_and_write_file_text.file", text=True
     )
-    assert (
-        "great #content\nincluding newlines" == written_file_string
-    ), f"unexpected content: [{written_file_string}]"
+    assert "great #content\nincluding newlines" == written_file_string, (
+        f"unexpected content: [{written_file_string}]"
+    )
     await _cleanup_file(sandbox_env, "test_read_and_write_file_text.file")
 
 
@@ -219,9 +219,9 @@ async def test_exec_output(sandbox_env: SandboxEnvironment) -> None:
     exec_result = await sandbox_env.exec(["sh", "-c", "echo foo; echo bar"])
     expected = "foo\nbar\n"
     # in the assertion message, we show the actual bytes to help debug newline issues
-    assert (
-        exec_result.stdout == expected
-    ), f"Unexpected output:expected {expected.encode('UTF-8')!r}; got {exec_result.stdout.encode('UTF-8')!r}"
+    assert exec_result.stdout == expected, (
+        f"Unexpected output:expected {expected.encode('UTF-8')!r}; got {exec_result.stdout.encode('UTF-8')!r}"
+    )
 
 
 async def test_exec_timeout(sandbox_env: SandboxEnvironment) -> None:
@@ -248,13 +248,13 @@ async def test_exec_as_user(sandbox_env: SandboxEnvironment) -> None:
 
         # Test exec as different users
         root_result = await sandbox_env.exec(["whoami"], user="root")
-        assert (
-            root_result.stdout.strip() == "root"
-        ), f"Expected 'root', got '{root_result.stdout.strip()}'"
+        assert root_result.stdout.strip() == "root", (
+            f"Expected 'root', got '{root_result.stdout.strip()}'"
+        )
         myuser_result = await sandbox_env.exec(["whoami"], user=username)
-        assert (
-            myuser_result.stdout.strip() == username
-        ), f"Expected '{username}', got '{myuser_result.stdout.strip()}'"
+        assert myuser_result.stdout.strip() == username, (
+            f"Expected '{username}', got '{myuser_result.stdout.strip()}'"
+        )
     finally:
         # Clean up
         await sandbox_env.exec(["userdel", "-r", username], user="root")
@@ -266,9 +266,9 @@ async def test_exec_as_nonexistent_user(sandbox_env: SandboxEnvironment) -> None
     expected_error = (
         "unable to find user nonexistent: no matching entries in passwd file"
     )
-    assert (
-        expected_error in result.stdout
-    ), f"Error string '{expected_error}' not found in error output: '{result.stdout}'"
+    assert expected_error in result.stdout, (
+        f"Error string '{expected_error}' not found in error output: '{result.stdout}'"
+    )
 
 
 async def test_cwd_unspecified(sandbox_env: SandboxEnvironment) -> None:
@@ -291,9 +291,9 @@ async def test_cwd_relative(sandbox_env: SandboxEnvironment) -> None:
     file_path = cwd_subdirectory + "/" + file_name
     await sandbox_env.write_file(file_path, "ls me plz")
     current_dir_contents = (await sandbox_env.exec(["ls"], cwd=cwd_subdirectory)).stdout
-    assert (
-        file_name in current_dir_contents
-    ), f"{file_name} not found in {current_dir_contents}"
+    assert file_name in current_dir_contents, (
+        f"{file_name} not found in {current_dir_contents}"
+    )
     await _cleanup_file(sandbox_env, file_path)
 
 

inspect_ai/util/_store.py

@@ -34,8 +34,8 @@ class Store:
     inheriting from Pydantic `BaseModel`)
     """
 
-    def __init__(self) -> None:
-        self._data: dict[str, Any] = {}
+    def __init__(self, data: dict[str, Any] | None = None) -> None:
+        self._data = deepcopy(data) if data else {}
 
     @overload
     def get(self, key: str, default: None = None) -> Any: ...

inspect_ai/util/_subprocess.py

@@ -101,9 +101,9 @@ async def subprocess(
     input = input.encode() if isinstance(input, str) else input
 
     # function to run command (we may or may not run it w/ concurrency)
-    async def run_command() -> (
-        AsyncGenerator[Union[Process, ExecResult[str], ExecResult[bytes]], None]
-    ):
+    async def run_command() -> AsyncGenerator[
+        Union[Process, ExecResult[str], ExecResult[bytes]], None
+    ]:
         if isinstance(args, str):
             proc = await asyncio.create_subprocess_shell(
                 args,

{inspect_ai-0.3.56.dist-info → inspect_ai-0.3.57.dist-info}/METADATA

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.2
 Name: inspect_ai
-Version: 0.3.56
+Version: 0.3.57
 Summary: Framework for large language model evaluations
 Author: UK AI Safety Institute
 License: MIT License
@@ -67,7 +67,7 @@ Requires-Dist: pytest-asyncio; extra == "dev"
 Requires-Dist: pytest-cov; extra == "dev"
 Requires-Dist: pytest-dotenv; extra == "dev"
 Requires-Dist: pytest-xdist; extra == "dev"
-Requires-Dist: ruff==0.8.4; extra == "dev"
+Requires-Dist: ruff==0.9.0; extra == "dev"
 Requires-Dist: textual-dev>=0.86.2; extra == "dev"
 Requires-Dist: types-PyYAML; extra == "dev"
 Requires-Dist: types-beautifulsoup4; extra == "dev"