inspect-ai 0.3.62__py3-none-any.whl → 0.3.63__py3-none-any.whl

inspect_ai/tool/{beta → _tools}/_computer/_computer.py

@@ -2,10 +2,7 @@ from typing import Awaitable, Callable
 
 from inspect_ai._util.content import Content, ContentImage, ContentText
 from inspect_ai.tool import Tool, ToolResult, tool
-from inspect_ai.tool._tool import (
-    TOOL_INIT_MODEL_INPUT,
-    ToolParsingError,
-)
+from inspect_ai.tool._tool import TOOL_INIT_MODEL_INPUT, ToolParsingError
 from inspect_ai.tool._tool_call import ToolCallModelInput
 
 from . import _common as common
@@ -84,7 +81,7 @@ def computer(max_screenshots: int | None = 1, timeout: int | None = 180) -> Tool
             if coordinate is not None:
                 raise ToolParsingError(f"coordinate is not accepted for {action}")
             if not isinstance(text, str):
-                raise ToolParsingError(output=f"{text} must be a string")
+                raise ToolParsingError(f"{text} must be a string")
 
             if action == "key":
                 return await common.press_key(text, timeout=timeout)

inspect_ai/tool/{beta → _tools}/_computer/_resources/tool/_x11_client.py

@@ -138,7 +138,7 @@ class X11Client:
             if coordinate is not None:
                 raise ToolError(f"coordinate is not accepted for {action}")
             if not isinstance(text, str):
-                raise ToolError(output=f"{text} must be a string")
+                raise ToolError(f"{text} must be a string")
 
             if action == "key":
                 return await self.shell(

inspect_ai/tool/_tools/_execute.py

@@ -74,8 +74,14 @@ def python(timeout: int | None = None, user: str | None = None) -> Tool:
         """
         Use the python function to execute Python code.
 
-        The python function will only return you the stdout of the script,
-        so make sure to use print to see the output.
+        The Python tool executes single-run Python scripts. Important notes:
+        1. Each execution is independent - no state is preserved between runs
+        2. You must explicitly use print() statements to see any output
+        3. Simply writing expressions (like in notebooks) will not display results
+        4. The script cannot accept interactive input during execution
+        5. Return statements alone won't produce visible output
+        6. All variables and imports are cleared between executions
+        7. Standard output (via print()) is the only way to see results
 
         Args:
           code (str): The python code to execute.

inspect_ai/tool/beta.py

@@ -0,0 +1,3 @@
+from inspect_ai._util.deprecation import relocated_module_attribute
+
+relocated_module_attribute("computer", "inspect_ai.tool.computer", "0.3.62", "0.4")

inspect_ai/util/_sandbox/docker/docker.py

@@ -1,3 +1,4 @@
+import base64
 import errno
 import json
 import os
@@ -34,7 +35,6 @@ from .compose import (
     compose_build,
     compose_check_running,
     compose_cleanup_images,
-    compose_command,
     compose_cp,
     compose_exec,
     compose_ps,
@@ -270,103 +270,50 @@ class DockerSandboxEnvironment(SandboxEnvironment):
 
     @override
     async def write_file(self, file: str, contents: str | bytes) -> None:
-        # exec function w/ timeout
-        async def exec(cmd: list[str]) -> ExecResult[str]:
-            return await self.exec(cmd, timeout=60)
-
         # resolve relative file paths
         file = self.container_file(file)
 
-        # We want to be able to write a file in the container,
-        # but only if the container's user would be allowed to do that.
-        # We need to avoid implicitly trusting the provided "file" string.
-        # For example, it shouldn't be passed as part of a shell command,
-        # because of the risk of shell injection.
-
-        local_tmpfile = tempfile.NamedTemporaryFile()
+        # ensure that the directory exists
+        parent = Path(file).parent.as_posix()
+        if parent != ".":
+            result = await self.exec(["mkdir", "-p", parent])
+            if not result.success:
+                msg = f"Failed to create container directory {parent}: {result.stderr}"
+                raise RuntimeError(msg)
 
-        # write contents into a local tmp file (not in the container)
+        # write the file
         if isinstance(contents, str):
-            local_tmpfile.write(contents.encode("utf-8"))
+            result = await self.exec(
+                ["sh", "-e", "-c", 'tee -- "$1"', "write_file_script", file],
+                input=contents,
+            )
         else:
-            local_tmpfile.write(contents)
-
-        local_tmpfile.flush()
-
-        # Copy the local tmp file into a tmp file on the container.
-        # Both tmp files have safe names as we created them ourselves
-
-        # We write the tmp file in the default directory,
-        # because of strangeness with /tmp on GitHub action runners.
-
-        # We are reusing the generated local tmp file name within
-        # the sandbox to save on a container roundtrip. There is a very slight
-        # risk of collision if another write_file call happens
-        # to get the same local tmp file name. But we assume tmp file
-        # names have enough randomness for us to ignore that.
-
-        container_tmpfile = (
-            f".tmp_inspect_sandbox_{os.path.basename(local_tmpfile.name)}"
-        )
-
-        # compose cp will leave the file owned by root
-        await compose_cp(
-            src=local_tmpfile.name,
-            dest=f"{self._service}:{self.container_file(container_tmpfile)}",
-            project=self._project,
-        )
-
-        local_tmpfile.close()  # this will also delete the file
-
-        if not hasattr(self, "_docker_user"):
-            uid = (await exec(["id", "-u"])).stdout.strip()
-            gid = (await exec(["id", "-g"])).stdout.strip()
-            self._docker_user = (uid, gid)
-
-        await compose_command(
-            [
-                "exec",
-                "--user",
-                "root",
-                self._service,
-                "chown",
-                f"{self._docker_user[0]}:{self._docker_user[1]}",
-                container_tmpfile,
-            ],
-            project=self._project,
-            timeout=60,
-        )
-
-        parent = PurePosixPath(file).parent
-
-        # We do these steps in a shell script for efficiency to avoid round-trips to docker.
-        res_cp = await exec(
-            [
-                "sh",
-                "-e",
-                "-c",
-                'mkdir -p -- "$1"; cp -T -- "$2" "$3"; rm -- "$2"',
-                "copy_script",
-                str(parent),
-                container_tmpfile,
-                file,
-            ]
-        )
-
-        if res_cp.returncode != 0:
-            if "Permission denied" in res_cp.stderr:
-                ls_result = await exec(["ls", "-la", "."])
-                error_string = f"Permission was denied. Error details: {res_cp.stderr}; ls -la: {ls_result.stdout}; {self._docker_user=}"
+            base64_contents = base64.b64encode(contents).decode("US-ASCII")
+            result = await self.exec(
+                [
+                    "sh",
+                    "-e",
+                    "-c",
+                    'base64 -d | tee -- "$1" > /dev/null',
+                    "write_file_script",
+                    file,
+                ],
+                input=base64_contents,
+            )
+        if result.returncode != 0:
+            if "permission denied" in result.stderr.casefold():
+                ls_result = await self.exec(["ls", "-la", "."])
+                error_string = f"Permission was denied. Error details: {result.stderr}; ls -la: {ls_result.stdout}"
                 raise PermissionError(error_string)
             elif (
-                "cannot overwrite directory" in res_cp.stderr
-                or "is a directory" in res_cp.stderr
+                "cannot overwrite directory" in result.stderr.casefold()
+                or "is a directory" in result.stderr.casefold()
             ):
                 raise IsADirectoryError(
                     f"Failed to write file: {file} because it is a directory already"
                 )
             else:
-                raise RuntimeError(f"failed to copy during write_file: {res_cp}")
+                raise RuntimeError(f"failed to copy during write_file: {result}")
 
     @overload
     async def read_file(self, file: str, text: Literal[True] = True) -> str: ...

inspect_ai/util/_sandbox/self_check.py

@@ -32,6 +32,7 @@ async def self_check(sandbox_env: SandboxEnvironment) -> dict[str, bool | str]:
     for fn in [
         test_read_and_write_file_text,
         test_read_and_write_file_binary,
+        test_read_and_write_large_file_binary,
         test_write_file_text_utf,
         test_read_and_write_file_including_directory_absolute,
         test_read_and_write_file_including_directory_relative,
@@ -41,12 +42,19 @@ async def self_check(sandbox_env: SandboxEnvironment) -> dict[str, bool | str]:
         test_read_file_is_directory,
         test_read_file_nonsense_name,
         test_read_file_limit,
-        test_write_file_zero_length,
-        test_write_file_space,
-        test_write_file_is_directory,
-        test_write_file_without_permissions,
-        test_write_file_exists,
+        test_write_text_file_zero_length,
+        test_write_text_file_space,
+        test_write_text_file_is_directory,
+        test_write_text_file_without_permissions,
+        test_write_text_file_exists,
+        test_write_binary_file_zero_length,
+        test_write_binary_file_space,
+        test_write_binary_file_is_directory,
+        test_write_binary_file_without_permissions,
+        test_write_binary_file_exists,
         test_exec_output,
+        test_exec_stderr,
+        test_exec_returncode,
         test_exec_timeout,
         test_exec_permission_error,
         test_exec_as_user,
@@ -100,6 +108,17 @@ async def test_read_and_write_file_binary(sandbox_env: SandboxEnvironment) -> No
     await _cleanup_file(sandbox_env, file_name)
 
 
+async def test_read_and_write_large_file_binary(
+    sandbox_env: SandboxEnvironment,
+) -> None:
+    file_name = "test_read_and_write_large_file_binary.file"
+    long_bytes = b"\xc3" * 5_000_000
+    await sandbox_env.write_file(file_name, long_bytes)
+    written_file_bytes = await sandbox_env.read_file(file_name, text=False)
+    assert long_bytes == written_file_bytes
+    await _cleanup_file(sandbox_env, file_name)
+
+
 async def test_read_and_write_file_including_directory_absolute(
     sandbox_env: SandboxEnvironment,
 ) -> None:
@@ -176,7 +195,7 @@ async def test_read_file_limit(sandbox_env: SandboxEnvironment) -> None:
     await _cleanup_file(sandbox_env, file_name)
 
 
-async def test_write_file_zero_length(sandbox_env: SandboxEnvironment) -> None:
+async def test_write_text_file_zero_length(sandbox_env: SandboxEnvironment) -> None:
     file_name = "zero_length_file.file"
     await sandbox_env.write_file(file_name, "")
     zero_length = await sandbox_env.read_file(file_name, text=True)
@@ -185,7 +204,7 @@ async def test_write_file_zero_length(sandbox_env: SandboxEnvironment) -> None:
     await _cleanup_file(sandbox_env, file_name)
 
 
-async def test_write_file_space(sandbox_env: SandboxEnvironment) -> None:
+async def test_write_text_file_space(sandbox_env: SandboxEnvironment) -> None:
     space = "to the moon"
     file_name = "file with space.file"
     await sandbox_env.write_file(file_name, space)
@@ -195,28 +214,28 @@ async def test_write_file_space(sandbox_env: SandboxEnvironment) -> None:
     await _cleanup_file(sandbox_env, file_name)
 
 
-async def test_write_file_is_directory(
+async def test_write_text_file_is_directory(
     sandbox_env: SandboxEnvironment,
 ) -> None:
     # ensure /tmp/directory exists
     await sandbox_env.write_file(
-        "/tmp/inspect_ai_test_write_file_is_directory/file", "unused content"
+        "/tmp/inspect_ai_test_write_text_file_is_directory/file", "unused content"
     )
     with Raises(IsADirectoryError) as e_info:
         await sandbox_env.write_file(
-            "/tmp/inspect_ai_test_write_file_is_directory",
+            "/tmp/inspect_ai_test_write_text_file_is_directory",
             "content cannot go in a directory, dummy",
         )
     assert "directory" in str(e_info.value)
     await sandbox_env.exec(
-        ["rm", "-rf", "/tmp/inspect_ai_test_write_file_is_directory"]
+        ["rm", "-rf", "/tmp/inspect_ai_test_write_text_file_is_directory"]
     )
 
 
-async def test_write_file_without_permissions(
+async def test_write_text_file_without_permissions(
     sandbox_env: SandboxEnvironment,
 ) -> None:
-    file_name = "test_write_file_without_permissions.file"
+    file_name = "test_write_text_file_without_permissions.file"
     await sandbox_env.write_file(file_name, "impervious #content")
     await sandbox_env.exec(["chmod", "-w", file_name])
     with Raises(PermissionError) as e_info:
@@ -226,7 +245,7 @@ async def test_write_file_without_permissions(
     await _cleanup_file(sandbox_env, file_name)
 
 
-async def test_write_file_exists(
+async def test_write_text_file_exists(
     sandbox_env: SandboxEnvironment,
 ) -> None:
     file_name = "file_exists.file"
@@ -237,6 +256,67 @@ async def test_write_file_exists(
     await _cleanup_file(sandbox_env, file_name)
 
 
+async def test_write_binary_file_zero_length(sandbox_env: SandboxEnvironment) -> None:
+    file_name = "zero_length_file.file"
+    await sandbox_env.write_file(file_name, b"")
+    zero_length = await sandbox_env.read_file(file_name, text=False)
+    assert isinstance(zero_length, bytes)
+    assert zero_length == b""
+    await _cleanup_file(sandbox_env, file_name)
+
+
+async def test_write_binary_file_space(sandbox_env: SandboxEnvironment) -> None:
+    binary_content = b"\xc3\x28"
+    file_name = "file with space.file"
+    await sandbox_env.write_file(file_name, binary_content)
+    file_with_space = await sandbox_env.read_file(file_name, text=False)
+    assert isinstance(file_with_space, bytes)
+    assert file_with_space == binary_content
+    await _cleanup_file(sandbox_env, file_name)
+
+
+async def test_write_binary_file_is_directory(
+    sandbox_env: SandboxEnvironment,
+) -> None:
+    # ensure /tmp/directory exists
+    await sandbox_env.write_file(
+        "/tmp/inspect_ai_test_write_binary_file_is_directory/file", "unused content"
+    )
+    with Raises(IsADirectoryError) as e_info:
+        await sandbox_env.write_file(
+            "/tmp/inspect_ai_test_write_binary_file_is_directory",
+            b"\xc3\x28",
+        )
+    assert "directory" in str(e_info.value)
+    await sandbox_env.exec(
+        ["rm", "-rf", "/tmp/inspect_ai_test_write_binary_file_is_directory"]
+    )
+
+
+async def test_write_binary_file_without_permissions(
+    sandbox_env: SandboxEnvironment,
+) -> None:
+    file_name = "test_write_binary_file_without_permissions.file"
+    await sandbox_env.write_file(file_name, "impervious #content")
+    await sandbox_env.exec(["chmod", "-w", file_name])
+    with Raises(PermissionError) as e_info:
+        await sandbox_env.write_file(file_name, b"\xc3\x28")
+    assert file_name in str(e_info.value)
+    await sandbox_env.exec(["chmod", "+w", file_name])
+    await _cleanup_file(sandbox_env, file_name)
+
+
+async def test_write_binary_file_exists(
+    sandbox_env: SandboxEnvironment,
+) -> None:
+    file_name = "file_exists.file"
+    await sandbox_env.write_file(file_name, b"\xc3\x28")
+    await sandbox_env.write_file(file_name, b"\xc3\x29")
+    altered_content = await sandbox_env.read_file(file_name, text=False)
+    assert altered_content == b"\xc3\x29"
+    await _cleanup_file(sandbox_env, file_name)
+
+
 async def test_exec_output(sandbox_env: SandboxEnvironment) -> None:
     exec_result = await sandbox_env.exec(["sh", "-c", "echo foo; echo bar"])
     expected = "foo\nbar\n"
@@ -246,9 +326,19 @@ async def test_exec_output(sandbox_env: SandboxEnvironment) -> None:
     )
 
 
+async def test_exec_stderr(sandbox_env: SandboxEnvironment) -> None:
+    exec_result = await sandbox_env.exec(["sh", "-c", "echo boof; echo baz >&2"])
+    assert exec_result.stderr == "baz\n"
+
+
+async def test_exec_returncode(sandbox_env: SandboxEnvironment) -> None:
+    exec_result = await sandbox_env.exec(["sh", "-c", "echo foo; exit 70"])
+    assert exec_result.returncode == 70
+
+
 async def test_exec_timeout(sandbox_env: SandboxEnvironment) -> None:
     with Raises(TimeoutError):
-        await sandbox_env.exec(["sleep", "2"], timeout=1)
+        await sandbox_env.exec(["sleep", "4"], timeout=2)
 
 
 async def test_exec_permission_error(sandbox_env: SandboxEnvironment) -> None:
@@ -259,10 +349,28 @@ async def test_exec_permission_error(sandbox_env: SandboxEnvironment) -> None:
 
 async def test_exec_as_user(sandbox_env: SandboxEnvironment) -> None:
     username = "inspect-ai-test-exec-as-user"
+
+    # Neither adduser nor useradd are part of POSIX, so we need some brittle logic here
+    adduser_help_exec_result = await sandbox_env.exec(["adduser", "--help"])
+    adduser_help_text = (
+        adduser_help_exec_result.stdout + adduser_help_exec_result.stderr
+    )
+
+    if "BusyBox" in adduser_help_text:
+        adduser_command = ["adduser", "-D", username]
+    else:
+        adduser_command = [
+            "adduser",
+            "--comment",
+            "self_check.py",
+            "--disabled-password",
+            username,
+        ]
+
     try:
         # Create a new user
         add_user_result = await sandbox_env.exec(
-            ["adduser", "--comment", "self_check.py", "--disabled-password", username],
+            adduser_command,
             user="root",
             timeout=10,  # in one case adduser decided to ask for input which caused the test to hang indefinitely
         )

{inspect_ai-0.3.62.dist-info → inspect_ai-0.3.63.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: inspect_ai
-Version: 0.3.62
+Version: 0.3.63
 Summary: Framework for large language model evaluations
 Author: UK AI Safety Institute
 License: MIT License
@@ -63,6 +63,7 @@ Requires-Dist: mypy; extra == "dev"
 Requires-Dist: nbformat; extra == "dev"
 Requires-Dist: openai; extra == "dev"
 Requires-Dist: pre-commit; extra == "dev"
+Requires-Dist: pylint; extra == "dev"
 Requires-Dist: pytest; extra == "dev"
 Requires-Dist: pytest-asyncio; extra == "dev"
 Requires-Dist: pytest-cov; extra == "dev"