inspect-ai 0.3.62__py3-none-any.whl → 0.3.64__py3-none-any.whl

inspect_ai/util/_sandbox/environment.py

@@ -65,91 +65,6 @@ class SandboxEnvironment(abc.ABC):
     filesystem context to copy samples files into and resolve relative paths to.
     """
 
-    @classmethod
-    def config_files(cls) -> list[str]:
-        """Standard config files for this provider (used for automatic discovery)"""
-        return []
-
-    @classmethod
-    def default_concurrency(cls) -> int | None:
-        """Default max_sandboxes for this provider (`None` means no maximum)"""
-        return None
-
-    @classmethod
-    async def task_init(
-        cls, task_name: str, config: SandboxEnvironmentConfigType | None
-    ) -> None:
-        """Called at task startup initialize resources.
-
-        Args:
-          task_name (str): Name of task using the sandbox environment.
-          config (SandboxEnvironmentConfigType): Implementation defined configuration (optional).
-        """
-        pass
-
-    @classmethod
-    async def sample_init(
-        cls,
-        task_name: str,
-        config: SandboxEnvironmentConfigType | None,
-        metadata: dict[str, str],
-    ) -> dict[str, "SandboxEnvironment"]:
-        """Initialize sandbox environments for a sample.
-
-        Args:
-          task_name (str): Name of task using the sandbox environment.
-          config (SandboxEnvironmentConfigType): Implementation defined configuration (optional).
-          metadata (dict[str,str]): Sample `metadata` field
-
-        Returns:
-          Dictionary of named sandbox environments. The environment which represents
-          the default environment (resolved by `sandbox("default")` or `sandbox()`) must
-          be the first key/value pair in the dictionary.
-        """
-        return {}
-
-    @classmethod
-    @abc.abstractmethod
-    async def sample_cleanup(
-        cls,
-        task_name: str,
-        config: SandboxEnvironmentConfigType | None,
-        environments: dict[str, "SandboxEnvironment"],
-        interrupted: bool,
-    ) -> None:
-        """Cleanup sandbox environments.
-
-        Args:
-          task_name (str): Name of task using the sandbox environment.
-          config (SandboxEnvironmentConfigType): Implementation defined configuration (optional).
-          environments (dict[str,SandboxEnvironment]): Sandbox environments created for this sample.
-          interrupted (bool): Was the task interrupted by an error or cancellation
-        """
-        ...
-
-    @classmethod
-    async def task_cleanup(
-        cls, task_name: str, config: SandboxEnvironmentConfigType | None, cleanup: bool
-    ) -> None:
-        """Called at task exit as a last chance to cleanup resources.
-
-        Args:
-          task_name (str): Name of task using the sandbox environment.
-          config (SandboxEnvironmentConfigType): Implementation defined configuration (optional).
-          cleanup (bool): Whether to actually cleanup environment resources
-            (False if `--no-sandbox-cleanup` was specified)
-        """
-        pass
-
-    @classmethod
-    async def cli_cleanup(cls, id: str | None) -> None:
-        """Handle a cleanup invoked from the CLI (e.g. inspect sandbox cleanup).
-
-        Args:
-          id (str | None): Optional ID to limit scope of cleanup.
-        """
-        pass
-
     @abc.abstractmethod
     async def exec(
         self,
@@ -170,13 +85,13 @@ class SandboxEnvironment(abc.ABC):
         `OutputLimitExceededError` will be raised.
 
         Args:
-          cmd (str | list[str]): Command or command and arguments to execute.
-          input (str | bytes | None): Standard input (optional).
-          cwd (str | None): Current working dir (optional). If relative, will be relative to the per-sample filesystem context.
-          env (dict[str,str]): Environment variables for execution.
-          user (str | None): Optional username or UID to run the command as.
-          timeout (int | None): Optional execution timeout (seconds).
-          timeout_retry (bool): Retry the command in the case that it times out.
+          cmd: Command or command and arguments to execute.
+          input: Standard input (optional).
+          cwd: Current working dir (optional). If relative, will be relative to the per-sample filesystem context.
+          env: Environment variables for execution.
+          user: Optional username or UID to run the command as.
+          timeout: Optional execution timeout (seconds).
+          timeout_retry: Retry the command in the case that it times out.
             Commands will be retried up to twice, with a timeout of no greater
             than 60 seconds for the first retry and 30 for the second.
 
@@ -204,9 +119,9 @@ class SandboxEnvironment(abc.ABC):
         should be automatically created.
 
         Args:
-          file (str): Path to file (relative file paths will resolve to the
+          file: Path to file (relative file paths will resolve to the
             per-sample working directory).
-          contents (str | bytes): Text or binary file contents.
+          contents: Text or binary file contents.
 
         Raises:
           PermissionError: If the current user does not have permission to
@@ -233,9 +148,9 @@ class SandboxEnvironment(abc.ABC):
         to specifying `newline=""` in a call to the Python `open()` function.
 
         Args:
-          file (str): Path to file (relative file paths will resolve to the
+          file: Path to file (relative file paths will resolve to the
             per-sample working directory).
-          text (bool): Read as a utf-8 encoded text file.
+          text: Read as a utf-8 encoded text file.
 
         Returns:
           Contents of file (as str or bytes for binary files)
@@ -265,6 +180,91 @@ class SandboxEnvironment(abc.ABC):
         """
         raise NotImplementedError("connection not implemented")
 
+    @classmethod
+    def config_files(cls) -> list[str]:
+        """Standard config files for this provider (used for automatic discovery)"""
+        return []
+
+    @classmethod
+    def default_concurrency(cls) -> int | None:
+        """Default max_sandboxes for this provider (`None` means no maximum)"""
+        return None
+
+    @classmethod
+    async def task_init(
+        cls, task_name: str, config: SandboxEnvironmentConfigType | None
+    ) -> None:
+        """Called at task startup initialize resources.
+
+        Args:
+          task_name: Name of task using the sandbox environment.
+          config: Implementation defined configuration (optional).
+        """
+        pass
+
+    @classmethod
+    async def sample_init(
+        cls,
+        task_name: str,
+        config: SandboxEnvironmentConfigType | None,
+        metadata: dict[str, str],
+    ) -> dict[str, "SandboxEnvironment"]:
+        """Initialize sandbox environments for a sample.
+
+        Args:
+          task_name: Name of task using the sandbox environment.
+          config: Implementation defined configuration (optional).
+          metadata: Sample `metadata` field
+
+        Returns:
+          Dictionary of named sandbox environments. The environment which represents
+          the default environment (resolved by `sandbox("default")` or `sandbox()`) must
+          be the first key/value pair in the dictionary.
+        """
+        return {}
+
+    @classmethod
+    @abc.abstractmethod
+    async def sample_cleanup(
+        cls,
+        task_name: str,
+        config: SandboxEnvironmentConfigType | None,
+        environments: dict[str, "SandboxEnvironment"],
+        interrupted: bool,
+    ) -> None:
+        """Cleanup sandbox environments.
+
+        Args:
+          task_name: Name of task using the sandbox environment.
+          config: Implementation defined configuration (optional).
+          environments: Sandbox environments created for this sample.
+          interrupted: Was the task interrupted by an error or cancellation
+        """
+        ...
+
+    @classmethod
+    async def task_cleanup(
+        cls, task_name: str, config: SandboxEnvironmentConfigType | None, cleanup: bool
+    ) -> None:
+        """Called at task exit as a last chance to cleanup resources.
+
+        Args:
+          task_name: Name of task using the sandbox environment.
+          config: Implementation defined configuration (optional).
+          cleanup: Whether to actually cleanup environment resources
+            (False if `--no-sandbox-cleanup` was specified)
+        """
+        pass
+
+    @classmethod
+    async def cli_cleanup(cls, id: str | None) -> None:
+        """Handle a cleanup invoked from the CLI (e.g. inspect sandbox cleanup).
+
+        Args:
+          id: Optional ID to limit scope of cleanup.
+        """
+        pass
+
 
 @dataclass
 class SandboxEnvironments:
@@ -284,7 +284,10 @@ class SandboxEnvironmentSpec(NamedTuple):
     """Specification of a SandboxEnvironment."""
 
     type: str
+    """Sandbox type (e.g. 'local', 'docker')"""
+
     config: SandboxEnvironmentConfigType | None = None
+    """Sandbox configuration (filename or config object)."""
 
 
 SandboxEnvironmentConfigType = BaseModel | str

inspect_ai/util/_sandbox/self_check.py

@@ -32,6 +32,7 @@ async def self_check(sandbox_env: SandboxEnvironment) -> dict[str, bool | str]:
     for fn in [
         test_read_and_write_file_text,
         test_read_and_write_file_binary,
+        test_read_and_write_large_file_binary,
         test_write_file_text_utf,
         test_read_and_write_file_including_directory_absolute,
         test_read_and_write_file_including_directory_relative,
@@ -41,12 +42,19 @@ async def self_check(sandbox_env: SandboxEnvironment) -> dict[str, bool | str]:
         test_read_file_is_directory,
         test_read_file_nonsense_name,
         test_read_file_limit,
-        test_write_file_zero_length,
-        test_write_file_space,
-        test_write_file_is_directory,
-        test_write_file_without_permissions,
-        test_write_file_exists,
+        test_write_text_file_zero_length,
+        test_write_text_file_space,
+        test_write_text_file_is_directory,
+        test_write_text_file_without_permissions,
+        test_write_text_file_exists,
+        test_write_binary_file_zero_length,
+        test_write_binary_file_space,
+        test_write_binary_file_is_directory,
+        test_write_binary_file_without_permissions,
+        test_write_binary_file_exists,
         test_exec_output,
+        test_exec_stderr,
+        test_exec_returncode,
         test_exec_timeout,
         test_exec_permission_error,
         test_exec_as_user,
@@ -100,6 +108,17 @@ async def test_read_and_write_file_binary(sandbox_env: SandboxEnvironment) -> No
     await _cleanup_file(sandbox_env, file_name)
 
 
+async def test_read_and_write_large_file_binary(
+    sandbox_env: SandboxEnvironment,
+) -> None:
+    file_name = "test_read_and_write_large_file_binary.file"
+    long_bytes = b"\xc3" * 5_000_000
+    await sandbox_env.write_file(file_name, long_bytes)
+    written_file_bytes = await sandbox_env.read_file(file_name, text=False)
+    assert long_bytes == written_file_bytes
+    await _cleanup_file(sandbox_env, file_name)
+
+
 async def test_read_and_write_file_including_directory_absolute(
     sandbox_env: SandboxEnvironment,
 ) -> None:
@@ -176,7 +195,7 @@ async def test_read_file_limit(sandbox_env: SandboxEnvironment) -> None:
     await _cleanup_file(sandbox_env, file_name)
 
 
-async def test_write_file_zero_length(sandbox_env: SandboxEnvironment) -> None:
+async def test_write_text_file_zero_length(sandbox_env: SandboxEnvironment) -> None:
     file_name = "zero_length_file.file"
     await sandbox_env.write_file(file_name, "")
     zero_length = await sandbox_env.read_file(file_name, text=True)
@@ -185,7 +204,7 @@ async def test_write_file_zero_length(sandbox_env: SandboxEnvironment) -> None:
     await _cleanup_file(sandbox_env, file_name)
 
 
-async def test_write_file_space(sandbox_env: SandboxEnvironment) -> None:
+async def test_write_text_file_space(sandbox_env: SandboxEnvironment) -> None:
     space = "to the moon"
     file_name = "file with space.file"
     await sandbox_env.write_file(file_name, space)
@@ -195,28 +214,28 @@ async def test_write_file_space(sandbox_env: SandboxEnvironment) -> None:
     await _cleanup_file(sandbox_env, file_name)
 
 
-async def test_write_file_is_directory(
+async def test_write_text_file_is_directory(
     sandbox_env: SandboxEnvironment,
 ) -> None:
     # ensure /tmp/directory exists
     await sandbox_env.write_file(
-        "/tmp/inspect_ai_test_write_file_is_directory/file", "unused content"
+        "/tmp/inspect_ai_test_write_text_file_is_directory/file", "unused content"
     )
     with Raises(IsADirectoryError) as e_info:
         await sandbox_env.write_file(
-            "/tmp/inspect_ai_test_write_file_is_directory",
+            "/tmp/inspect_ai_test_write_text_file_is_directory",
             "content cannot go in a directory, dummy",
         )
     assert "directory" in str(e_info.value)
     await sandbox_env.exec(
-        ["rm", "-rf", "/tmp/inspect_ai_test_write_file_is_directory"]
+        ["rm", "-rf", "/tmp/inspect_ai_test_write_text_file_is_directory"]
     )
 
 
-async def test_write_file_without_permissions(
+async def test_write_text_file_without_permissions(
     sandbox_env: SandboxEnvironment,
 ) -> None:
-    file_name = "test_write_file_without_permissions.file"
+    file_name = "test_write_text_file_without_permissions.file"
     await sandbox_env.write_file(file_name, "impervious #content")
     await sandbox_env.exec(["chmod", "-w", file_name])
     with Raises(PermissionError) as e_info:
@@ -226,7 +245,7 @@ async def test_write_file_without_permissions(
     await _cleanup_file(sandbox_env, file_name)
 
 
-async def test_write_file_exists(
+async def test_write_text_file_exists(
     sandbox_env: SandboxEnvironment,
 ) -> None:
     file_name = "file_exists.file"
@@ -237,6 +256,67 @@ async def test_write_file_exists(
     await _cleanup_file(sandbox_env, file_name)
 
 
+async def test_write_binary_file_zero_length(sandbox_env: SandboxEnvironment) -> None:
+    file_name = "zero_length_file.file"
+    await sandbox_env.write_file(file_name, b"")
+    zero_length = await sandbox_env.read_file(file_name, text=False)
+    assert isinstance(zero_length, bytes)
+    assert zero_length == b""
+    await _cleanup_file(sandbox_env, file_name)
+
+
+async def test_write_binary_file_space(sandbox_env: SandboxEnvironment) -> None:
+    binary_content = b"\xc3\x28"
+    file_name = "file with space.file"
+    await sandbox_env.write_file(file_name, binary_content)
+    file_with_space = await sandbox_env.read_file(file_name, text=False)
+    assert isinstance(file_with_space, bytes)
+    assert file_with_space == binary_content
+    await _cleanup_file(sandbox_env, file_name)
+
+
+async def test_write_binary_file_is_directory(
+    sandbox_env: SandboxEnvironment,
+) -> None:
+    # ensure /tmp/directory exists
+    await sandbox_env.write_file(
+        "/tmp/inspect_ai_test_write_binary_file_is_directory/file", "unused content"
+    )
+    with Raises(IsADirectoryError) as e_info:
+        await sandbox_env.write_file(
+            "/tmp/inspect_ai_test_write_binary_file_is_directory",
+            b"\xc3\x28",
+        )
+    assert "directory" in str(e_info.value)
+    await sandbox_env.exec(
+        ["rm", "-rf", "/tmp/inspect_ai_test_write_binary_file_is_directory"]
+    )
+
+
+async def test_write_binary_file_without_permissions(
+    sandbox_env: SandboxEnvironment,
+) -> None:
+    file_name = "test_write_binary_file_without_permissions.file"
+    await sandbox_env.write_file(file_name, "impervious #content")
+    await sandbox_env.exec(["chmod", "-w", file_name])
+    with Raises(PermissionError) as e_info:
+        await sandbox_env.write_file(file_name, b"\xc3\x28")
+    assert file_name in str(e_info.value)
+    await sandbox_env.exec(["chmod", "+w", file_name])
+    await _cleanup_file(sandbox_env, file_name)
+
+
+async def test_write_binary_file_exists(
+    sandbox_env: SandboxEnvironment,
+) -> None:
+    file_name = "file_exists.file"
+    await sandbox_env.write_file(file_name, b"\xc3\x28")
+    await sandbox_env.write_file(file_name, b"\xc3\x29")
+    altered_content = await sandbox_env.read_file(file_name, text=False)
+    assert altered_content == b"\xc3\x29"
+    await _cleanup_file(sandbox_env, file_name)
+
+
 async def test_exec_output(sandbox_env: SandboxEnvironment) -> None:
     exec_result = await sandbox_env.exec(["sh", "-c", "echo foo; echo bar"])
     expected = "foo\nbar\n"
@@ -246,9 +326,19 @@ async def test_exec_output(sandbox_env: SandboxEnvironment) -> None:
     )
 
 
+async def test_exec_stderr(sandbox_env: SandboxEnvironment) -> None:
+    exec_result = await sandbox_env.exec(["sh", "-c", "echo boof; echo baz >&2"])
+    assert exec_result.stderr == "baz\n"
+
+
+async def test_exec_returncode(sandbox_env: SandboxEnvironment) -> None:
+    exec_result = await sandbox_env.exec(["sh", "-c", "echo foo; exit 70"])
+    assert exec_result.returncode == 70
+
+
 async def test_exec_timeout(sandbox_env: SandboxEnvironment) -> None:
     with Raises(TimeoutError):
-        await sandbox_env.exec(["sleep", "2"], timeout=1)
+        await sandbox_env.exec(["sleep", "4"], timeout=2)
 
 
 async def test_exec_permission_error(sandbox_env: SandboxEnvironment) -> None:
@@ -259,10 +349,28 @@ async def test_exec_permission_error(sandbox_env: SandboxEnvironment) -> None:
 
 async def test_exec_as_user(sandbox_env: SandboxEnvironment) -> None:
     username = "inspect-ai-test-exec-as-user"
+
+    # Neither adduser nor useradd are part of POSIX, so we need some brittle logic here
+    adduser_help_exec_result = await sandbox_env.exec(["adduser", "--help"])
+    adduser_help_text = (
+        adduser_help_exec_result.stdout + adduser_help_exec_result.stderr
+    )
+
+    if "BusyBox" in adduser_help_text:
+        adduser_command = ["adduser", "-D", username]
+    else:
+        adduser_command = [
+            "adduser",
+            "--comment",
+            "self_check.py",
+            "--disabled-password",
+            username,
+        ]
+
     try:
         # Create a new user
         add_user_result = await sandbox_env.exec(
-            ["adduser", "--comment", "self_check.py", "--disabled-password", username],
+            adduser_command,
             user="root",
             timeout=10,  # in one case adduser decided to ask for input which caused the test to hang indefinitely
         )

inspect_ai/util/_subprocess.py

@@ -20,6 +20,8 @@ T = TypeVar("T", str, bytes)
 
 @dataclass
 class ExecResult(Generic[T]):
+    """Execution result from call to `subprocess()`."""
+
     success: bool
     """Did the process exit with success."""
 
@@ -85,11 +87,11 @@ async def subprocess(
        cwd (str | Path | None): Switch to directory for execution.
        env (dict[str, str]): Additional environment variables.
        capture_output (bool): Capture stderr and stdout into ExecResult
-         (if False, then output is redirected to parent stderr/stdout)
+          (if False, then output is redirected to parent stderr/stdout)
        output_limit (int | None): Stop reading output if it exceeds
-         the specified limit (in bytes).
+          the specified limit (in bytes).
        timeout (int | None): Timeout. If the timeout expires then
-         a `TimeoutError` will be raised.
+          a `TimeoutError` will be raised.
 
     Returns:
        Subprocess result (text or binary depending on `text` param)

inspect_ai/util/_subtask.py

@@ -27,21 +27,21 @@ logger = getLogger(__name__)
 
 @runtime_checkable
 class Subtask(Protocol):
-    """Subtask with distinct `Store` and `Transcript`.
-
-    Args:
-      *args (Any): Arguments for the subtask.
-      **kwargs (Any): Keyword arguments for the subtask.
-
-    Returns:
-      Result of subtask.
-    """
-
     async def __call__(
         self,
         *args: Any,
         **kwargs: Any,
-    ) -> Any: ...
+    ) -> Any:
+        """Subtask with distinct `Store` and `Transcript`.
+
+        Args:
+            *args (Any): Arguments for the subtask.
+            **kwargs (Any): Keyword arguments for the subtask.
+
+        Returns:
+            Result of subtask.
+        """
+        ...
 
 
 @overload
@@ -71,11 +71,10 @@ def subtask(
     r"""Decorator for subtasks.
 
     Args:
-        func (Subtask): Subtask implementation.
-        name (str | None): Name for subtask (defaults to function name)
-        store (store | None): Store to use for subtask
-        type (str | None): Type to use for subtask
-        input (dict[str, Any] | None): Input to log for subtask
+        name: Name for subtask (defaults to function name)
+        store: Store to use for subtask
+        type: Type to use for subtask
+        input: Input to log for subtask
 
     Returns:
         Function which runs the Subtask, providing an isolated

{inspect_ai-0.3.62.dist-info → inspect_ai-0.3.64.dist-info}/LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2024 UK AI Safety Institute
+Copyright (c) 2024 UK AI Security Institute
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

{inspect_ai-0.3.62.dist-info → inspect_ai-0.3.64.dist-info}/METADATA

@@ -1,8 +1,8 @@
 Metadata-Version: 2.2
 Name: inspect_ai
-Version: 0.3.62
+Version: 0.3.64
 Summary: Framework for large language model evaluations
-Author: UK AI Safety Institute
+Author: UK AI Security Institute
 License: MIT License
 Project-URL: Documentation, https://inspect.ai-safety-institute.org.uk/
 Project-URL: Source Code, https://github.com/UKGovernmentBEIS/inspect_ai
@@ -55,6 +55,7 @@ Requires-Dist: azure-ai-inference; extra == "dev"
 Requires-Dist: google-cloud-aiplatform; extra == "dev"
 Requires-Dist: google-generativeai; extra == "dev"
 Requires-Dist: goodfire; extra == "dev"
+Requires-Dist: griffe; extra == "dev"
 Requires-Dist: groq; extra == "dev"
 Requires-Dist: ipython; extra == "dev"
 Requires-Dist: mistralai; extra == "dev"
@@ -63,13 +64,15 @@ Requires-Dist: mypy; extra == "dev"
 Requires-Dist: nbformat; extra == "dev"
 Requires-Dist: openai; extra == "dev"
 Requires-Dist: pre-commit; extra == "dev"
+Requires-Dist: pylint; extra == "dev"
 Requires-Dist: pytest; extra == "dev"
 Requires-Dist: pytest-asyncio; extra == "dev"
 Requires-Dist: pytest-cov; extra == "dev"
 Requires-Dist: pytest-dotenv; extra == "dev"
 Requires-Dist: pytest-xdist; extra == "dev"
-Requires-Dist: ruff==0.9.4; extra == "dev"
+Requires-Dist: ruff==0.9.5; extra == "dev"
 Requires-Dist: textual-dev>=0.86.2; extra == "dev"
+Requires-Dist: types-Markdown; extra == "dev"
 Requires-Dist: types-PyYAML; extra == "dev"
 Requires-Dist: types-beautifulsoup4; extra == "dev"
 Requires-Dist: types-aioboto3; extra == "dev"
@@ -81,15 +84,17 @@ Requires-Dist: types-protobuf; extra == "dev"
 Requires-Dist: types-psutil; extra == "dev"
 Requires-Dist: types-python-dateutil; extra == "dev"
 Provides-Extra: doc
-Requires-Dist: quarto-cli; extra == "doc"
+Requires-Dist: quarto-cli==1.5.57; extra == "doc"
 Requires-Dist: jupyter; extra == "doc"
+Requires-Dist: panflute; extra == "doc"
+Requires-Dist: markdown; extra == "doc"
 Provides-Extra: dist
 Requires-Dist: twine; extra == "dist"
 Requires-Dist: build; extra == "dist"
 
-[<img width="295" src="https://inspect.ai-safety-institute.org.uk/images/aisi-logo.png" />](https://aisi.gov.uk/)
+[<img width="295" src="https://inspect.ai-safety-institute.org.uk/images/aisi-logo.svg" />](https://aisi.gov.uk/)
 
-Welcome to Inspect, a framework for large language model evaluations created by the [UK AI Safety Institute](https://aisi.gov.uk/).
+Welcome to Inspect, a framework for large language model evaluations created by the [UK AI Security Institute](https://aisi.gov.uk/).
 
 Inspect provides many built-in components, including facilities for prompt engineering, tool usage, multi-turn dialog, and model graded evaluations. Extensions to Inspect (e.g. to support new elicitation and scoring techniques) can be provided by other Python packages.