inspect-ai 0.3.74__py3-none-any.whl → 0.3.76__py3-none-any.whl

inspect_ai/tool/_tools/_web_browser/_web_browser.py

@@ -1,23 +1,40 @@
 import re
-from textwrap import dedent
 
-from pydantic import Field
+from pydantic import BaseModel, Field
 
 from inspect_ai._util.content import ContentText
 from inspect_ai._util.error import PrerequisiteError
 from inspect_ai.tool._tool import Tool, ToolError, ToolResult, tool
 from inspect_ai.tool._tool_call import ToolCall, ToolCallContent, ToolCallView
 from inspect_ai.tool._tool_info import parse_tool_info
+from inspect_ai.tool._tool_support_helpers import (
+    exec_sandbox_rpc,
+    tool_container_sandbox,
+)
 from inspect_ai.tool._tool_with import tool_with
-from inspect_ai.util._sandbox import SandboxEnvironment, sandbox_with
-from inspect_ai.util._sandbox.docker.internal import INSPECT_WEB_BROWSER_IMAGE_DOCKERHUB
 from inspect_ai.util._store_model import StoreModel, store_as
 
+from ._back_compat import old_web_browser_cmd
+
+
+# These two models are cloned from the container code. If/when we decide to create
+# a package that is shared between the inspect and tool-container codebases, we'll
+# just have to live with it.
+class NewSessionResult(BaseModel):
+    session_name: str
+
+
+class CrawlerResult(BaseModel):
+    web_url: str
+    main_content: str | None = None
+    web_at: str
+    error: str | None = None
+
 
 def web_browser(interactive: bool = True) -> list[Tool]:
     """Tools used for web browser navigation.
 
-     See documentation at <https://inspect.ai-safety-institute.org.uk/tools.html#sec-web-browser>.
+     See documentation at <https://inspect.aisi.org.uk/tools-standard.html#sec-web-browser>.
 
     Args:
        interactive: Provide interactive tools (enable
@@ -85,7 +102,7 @@ def web_browser_go() -> Tool:
         Returns:
           Web accessibility tree of the visible elements of the web page. The element_id of each element is displayed in brackets at the beginning of the line.
         """
-        return await web_browser_cmd("web_go", url)
+        return await _web_browser_cmd("web_go", locals())
 
     return execute
 
@@ -165,7 +182,7 @@ def web_browser_click() -> Tool:
         Returns:
            Web accessibility tree of the visible elements of the web page. The element_id of each element is displayed in brackets at the beginning of the line.
         """
-        return await web_browser_cmd("web_click", str(element_id))
+        return await _web_browser_cmd("web_click", locals())
 
     return execute
 
@@ -203,7 +220,7 @@ def web_browser_type_submit() -> Tool:
         Returns:
            Web accessibility tree of the visible elements of the web page. The element_id of each element is displayed in brackets at the beginning of the line.
         """
-        return await web_browser_cmd("web_type_submit", str(element_id), text)
+        return await _web_browser_cmd("web_type_submit", locals())
 
     return execute
 
@@ -241,7 +258,7 @@ def web_browser_type() -> Tool:
         Returns:
            Web accessibility tree of the visible elements of the web page. The element_id of each element is displayed in brackets at the beginning of the line.
         """
-        return await web_browser_cmd("web_type", str(element_id), text)
+        return await _web_browser_cmd("web_type", locals())
 
     return execute
 
@@ -271,7 +288,7 @@ def web_browser_scroll() -> Tool:
         Returns:
            Web accessibility tree of the visible elements of the web page. The element_id of each element is displayed in brackets at the beginning of the line.
         """
-        return await web_browser_cmd("web_scroll", direction)
+        return await _web_browser_cmd("web_scroll", locals())
 
     return execute
 
@@ -292,7 +309,7 @@ def web_browser_back() -> Tool:
         Returns:
            Web accessibility tree of the visible elements of the web page. The element_id of each element is displayed in brackets at the beginning of the line.
         """
-        return await web_browser_cmd("web_back")
+        return await _web_browser_cmd("web_back", locals())
 
     return execute
 
@@ -313,7 +330,7 @@ def web_browser_forward() -> Tool:
         Returns:
            Web accessibility tree of the visible elements of the web page. The element_id of each element is displayed in brackets at the beginning of the line.
         """
-        return await web_browser_cmd("web_forward")
+        return await _web_browser_cmd("web_forward", locals())
 
     return execute
 
@@ -334,133 +351,61 @@ def web_browser_refresh() -> Tool:
         Returns:
            Web accessibility tree of the visible elements of the web page. The element_id of each element is displayed in brackets at the beginning of the line.
         """
-        return await web_browser_cmd("web_refresh")
+        return await _web_browser_cmd("web_refresh", locals())
 
     return execute
 
 
-WEB_CLIENT_REQUEST = "/app/web_browser/web_client.py"
-WEB_CLIENT_NEW_SESSION = "/app/web_browser/web_client_new_session.py"
-
-
-async def web_browser_cmd(cmd: str, *args: str) -> ToolResult:
-    sandbox_env = await sandbox_with(WEB_CLIENT_NEW_SESSION)
-    session_flag = ""
-    if sandbox_env:
-        store = store_as(WebBrowserStore)
-        if not store.session_id:
-            result = await sandbox_env.exec(
-                ["python3", WEB_CLIENT_NEW_SESSION], timeout=180
+async def _web_browser_cmd(tool_name: str, params: dict[str, object]) -> ToolResult:
+    try:
+        sandbox_env = await tool_container_sandbox("web browser")
+    except PrerequisiteError as e:
+        # The user may have the old, incompatible, sandbox. If so, use that and
+        # execute the old compatible code.
+        try:
+            return await old_web_browser_cmd(tool_name, *params)
+        except PrerequisiteError:
+            raise e
+
+    store = store_as(WebBrowserStore)
+
+    if not store.session_id:
+        store.session_id = (
+            await exec_sandbox_rpc(
+                sandbox_env,
+                "web_new_session",
+                {"headful": False},
+                NewSessionResult,
             )
+        ).session_name
 
-            if not result.success:
-                raise RuntimeError(
-                    f"Error creating new web browser session: {result.stderr}"
-                )
-
-            store.session_id = result.stdout.strip("\n")
+    params["session_name"] = store.session_id
 
-        session_flag = f"--session_name={store.session_id}"
-
-    else:
-        sandbox_env = await web_browser_sandbox()
-
-    arg_list = None
-    if session_flag:
-        arg_list = ["python3", WEB_CLIENT_REQUEST, session_flag, cmd] + list(args)
+    crawler_result = await exec_sandbox_rpc(
+        sandbox_env, tool_name, params, CrawlerResult
+    )
+    if crawler_result.error and crawler_result.error.strip() != "":
+        raise ToolError(crawler_result.error)
     else:
-        arg_list = ["python3", WEB_CLIENT_REQUEST, cmd] + list(args)
+        main_content = crawler_result.main_content
+        web_at = crawler_result.web_at or "(no web accessibility tree available)"
+        # Remove base64 data from images.
+        web_at_lines = web_at.split("\n")
+        web_at_lines = [
+            line.partition("data:image/png;base64")[0] for line in web_at_lines
+        ]
 
-    result = await sandbox_env.exec(arg_list, timeout=180)
-    if not result.success:
-        raise RuntimeError(
-            f"Error executing web browser command {cmd}({', '.join(args)}): {result.stderr}"
+        store_as(WebBrowserStore).main_content = (
+            main_content or "(no main text summary)"
         )
-    else:
-        response = parse_web_browser_output(result.stdout)
-        if "error" in response and response.get("error", "").strip() != "":
-            raise ToolError(str(response.get("error")) or "(unknown error)")
-        elif "web_at" in response:
-            main_content = str(response.get("main_content")) or None
-            web_at = (
-                str(response.get("web_at")) or "(no web accessibility tree available)"
-            )
-            # Remove base64 data from images.
-            web_at_lines = web_at.split("\n")
-            web_at_lines = [
-                line.partition("data:image/png;base64")[0] for line in web_at_lines
-            ]
+        store_as(WebBrowserStore).web_at = web_at
 
-            store_as(WebBrowserStore).main_content = (
-                main_content or "(no main text summary)"
-            )
-            store_as(WebBrowserStore).web_at = web_at
-
-            web_at = "\n".join(web_at_lines)
-            return (
-                [
-                    ContentText(text=f"main content:\n{main_content}\n\n"),
-                    ContentText(text=f"accessibility tree:\n{web_at}"),
-                ]
-                if main_content
-                else web_at
-            )
-        else:
-            raise RuntimeError(
-                f"web_browser output must contain either 'error' or 'web_at' field: {result.stdout}"
-            )
-
-
-async def web_browser_sandbox() -> SandboxEnvironment:
-    sb = await sandbox_with(WEB_CLIENT_REQUEST)
-    if sb:
-        return sb
-    else:
-        msg = dedent(f"""
-                The web browser service was not found in any of the sandboxes for this sample. Please add the web browser service to your configuration. For example, the following Docker compose file uses the {INSPECT_WEB_BROWSER_IMAGE_DOCKERHUB} image as its default sandbox:
-
-                services:
-                  default:
-                    image: "{INSPECT_WEB_BROWSER_IMAGE_DOCKERHUB}"
-                    init: true
-
-                Alternatively, this Docker compose file creates a dedicated image for the web browser service:
-
-                services:
-                  default:
-                    image: "python:3.12-bookworm"
-                    init: true
-                    command: "tail -f /dev/null"
-
-                  web_browser:
-                    image: "{INSPECT_WEB_BROWSER_IMAGE_DOCKERHUB}"
-                    init: true
-                """).strip()
-        raise PrerequisiteError(msg)
-
-
-def parse_web_browser_output(output: str) -> dict[str, str]:
-    response: dict[str, str] = dict(
-        web_url="", main_content="", web_at="", info="", error=""
-    )
-    active_field: str | None = None
-    active_field_lines: list[str] = []
-
-    def collect_active_field() -> None:
-        if active_field is not None:
-            response[active_field] = "\n".join(active_field_lines)
-        active_field_lines.clear()
-
-    for line in output.splitlines():
-        field_match = re.match(
-            r"^(error|main_content|web_at|web_url|info)\s*:\s*(.+)$", line
+        web_at = "\n".join(web_at_lines)
+        return (
+            [
+                ContentText(text=f"main content:\n{main_content}\n\n"),
+                ContentText(text=f"accessibility tree:\n{web_at}"),
+            ]
+            if main_content
+            else web_at
         )
-        if field_match:
-            collect_active_field()
-            active_field = field_match.group(1)
-            active_field_lines.append(field_match.group(2))
-        else:
-            active_field_lines.append(line)
-    collect_active_field()
-
-    return response

inspect_ai/tool/_tools/_web_search.py

@@ -52,7 +52,7 @@ def web_search(
     A web search is conducted using the specified provider, the results are parsed for relevance
     using the specified model, and the top 'num_results' relevant pages are returned.
 
-    See further documentation at <https://inspect.ai-safety-institute.org.uk/tools.html#sec-web-search>.
+    See further documentation at <https://inspect.aisi.org.uk/tools-standard.html#sec-web-search>.
 
     Args:
       provider: Search provider (defaults to "google", currently
@@ -190,7 +190,7 @@ def google_search_provider(client: httpx.AsyncClient) -> SearchProvider:
     google_cse_id = os.environ.get("GOOGLE_CSE_ID", None)
     if not google_api_key or not google_cse_id:
         raise PrerequisiteError(
-            "GOOGLE_CSE_ID and/or GOOGLE_CSE_API_KEY not set in the environment. Please ensure these variables are defined to use Google Custom Search with the web_search tool.\n\nLearn more about the Google web search provider at https://inspect.ai-safety-institute.org.uk/tools.html#google-provider"
+            "GOOGLE_CSE_ID and/or GOOGLE_CSE_API_KEY not set in the environment. Please ensure these variables are defined to use Google Custom Search with the web_search tool.\n\nLearn more about the Google web search provider at https://inspect.aisi.org.uk/tools.html#google-provider"
         )
 
     async def search(query: str, start_idx: int) -> list[SearchLink]:

inspect_ai/util/_json.py

@@ -1,5 +1,6 @@
 import types
 import typing
+from copy import deepcopy
 from dataclasses import is_dataclass
 from typing import (
     Any,
@@ -10,6 +11,7 @@ from typing import (
     Tuple,
     Type,
     Union,
+    cast,
     get_args,
     get_origin,
     get_type_hints,
@@ -127,6 +129,7 @@ def cls_json_schema(cls: Type[Any]) -> JSONSchema:
                 required.append(name)
     elif isinstance(cls, type) and issubclass(cls, BaseModel):
         schema = cls.model_json_schema()
+        schema = resolve_schema_references(schema)
         for name, prop in schema.get("properties", {}).items():
             properties[name] = JSONSchema(**prop)
         required = schema.get("required", [])
@@ -168,3 +171,28 @@ def python_type_to_json_type(python_type: str | None) -> JSONType:
             raise ValueError(
                 f"Unsupported type: {python_type} for Python to JSON conversion."
             )
+
+
+def resolve_schema_references(schema: dict[str, Any]) -> dict[str, Any]:
+    """Resolves all $ref references in a JSON schema by inlining the definitions."""
+    schema = deepcopy(schema)
+    definitions = schema.pop("$defs", {})
+
+    def _resolve_refs(obj: Any) -> Any:
+        if isinstance(obj, dict):
+            if "$ref" in obj and obj["$ref"].startswith("#/$defs/"):
+                ref_key = obj["$ref"].split("/")[-1]
+                if ref_key in definitions:
+                    # Replace with a deep copy of the definition
+                    resolved = deepcopy(definitions[ref_key])
+                    # Process any nested references in the definition
+                    return _resolve_refs(resolved)
+
+            # Process all entries in the dictionary
+            return {k: _resolve_refs(v) for k, v in obj.items()}
+        elif isinstance(obj, list):
+            return [_resolve_refs(item) for item in obj]
+        else:
+            return obj
+
+    return cast(dict[str, Any], _resolve_refs(schema))

inspect_ai/util/_sandbox/context.py

@@ -49,11 +49,14 @@ def sandbox(name: str | None = None) -> SandboxEnvironment:
         return environment
 
 
-async def sandbox_with(file: str) -> SandboxEnvironment | None:
+async def sandbox_with(file: str, on_path: bool = False) -> SandboxEnvironment | None:
     """Get the SandboxEnvironment for the current sample that has the specified file.
 
     Args:
-      file (str): Path to file to check for.
+      file (str): Path to file to check for if on_path is False. If on_path is
+        True, file should be a filename that exists on the system path.
+      on_path (bool): If True, file is a filename to be verified using "which".
+        If False, file is a path to be checked within the sandbox environments.
 
     Return:
       SandboxEnvironment instance or None if no sandboxes had the file.
@@ -66,19 +69,25 @@ async def sandbox_with(file: str) -> SandboxEnvironment | None:
     if environments_with is None:
         raise_no_sandbox()
 
-    # if we've already disovered the sandbox for this file then return it
-    environment = environments_with.get(file, None)
+    # if we've already discovered the sandbox for this file then return it
+    environment_with_key = f"{file}:{on_path}"
+    environment = environments_with.get(environment_with_key, None)
     if environment is not None:
         return environment
 
     # look in each sandbox
     for _, environment in environments.items():
         try:
-            # can we read the file?
-            await environment.read_file(file)
+            if on_path:
+                # can we find the file on the path?
+                if not (await environment.exec(["which", file])).success:
+                    continue
+            else:
+                # can we read the file?
+                await environment.read_file(file)
 
             # if so this is our environment, cache and return it
-            environments_with[file] = environment
+            environments_with[environment_with_key] = environment
             return environment
 
         # allow exception types known to be raised from read_file
@@ -192,7 +201,8 @@ async def copy_sandbox_environment_files(
             target_env = environments.get(envname, None)
             if not target_env:
                 raise RuntimeError(
-                    f"Environment referenced in sample file not found: '{envname}:{file}'"
+                    f"Environment referenced in sample file not found: '{envname}:{file}'. "
+                    + "Note that ':' can be optionally used to specify an explicit environment name for sample files (e.g. 'envname:file') so cannot be used as a character within filenames."
                 )
         else:
             target_env = default_environment

inspect_ai/util/_sandbox/docker/config.py

@@ -82,7 +82,7 @@ COMPOSE_COMMENT = """# inspect auto-generated docker compose file
 COMPOSE_GENERIC_YAML = f"""{COMPOSE_COMMENT}
 services:
   default:
-    image: "python:3.12-bookworm"
+    image: "aisiuk/inspect-tool-support"
     command: "tail -f /dev/null"
     init: true
     network_mode: none

inspect_ai/util/_sandbox/docker/internal.py

@@ -3,13 +3,13 @@ from inspect_ai._util.error import PrerequisiteError
 from inspect_ai.util._display import display_type
 from inspect_ai.util._subprocess import subprocess
 
-INSPECT_WEB_BROWSER_IMAGE_DOCKERHUB = "aisiuk/inspect-web-browser-tool"
+INSPECT_WEB_BROWSER_IMAGE_DOCKERHUB_DEPRECATED = "aisiuk/inspect-web-browser-tool"
 
-INSPECT_WEB_BROWSER_IMAGE = "inspect_web_browser"
+INSPECT_WEB_BROWSER_IMAGE_DEPRECATED = "inspect_web_browser"
 INSPECT_COMPUTER_IMAGE = "inspect-computer-tool"
 
 INTERNAL_IMAGES = {
-    INSPECT_WEB_BROWSER_IMAGE: PKG_PATH
+    INSPECT_WEB_BROWSER_IMAGE_DEPRECATED: PKG_PATH
     / "tool"
     / "_tools"
     / "_web_browser"

inspect_ai/util/_sandbox/environment.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import abc
+import logging
 from dataclasses import dataclass, field
 from typing import (
     Annotated,
@@ -17,8 +18,12 @@ from typing import (
 
 from pydantic import BaseModel, Field, model_validator
 
+from inspect_ai._util.logger import warn_once
+
 from .._subprocess import ExecResult
 
+logger = logging.getLogger(__name__)
+
 ST = TypeVar("ST", bound="SandboxEnvironment")
 
 TaskInit = Callable[[str, Union["SandboxEnvironmentConfigType", None]], Awaitable[None]]
@@ -381,11 +386,21 @@ def resolve_sandbox_environment(
         return None
 
 
-def deserialize_sandbox_specific_config(type: str, config: dict[str, Any]) -> BaseModel:
+def deserialize_sandbox_specific_config(
+    type: str, config: dict[str, Any]
+) -> BaseModel | dict[str, Any]:
     # Avoid circular import
     from inspect_ai.util._sandbox.registry import registry_find_sandboxenv
 
-    sandboxenv_type = registry_find_sandboxenv(type)
+    try:
+        sandboxenv_type = registry_find_sandboxenv(type)
+    except ValueError:
+        warn_once(
+            logger,
+            f"Could not find sandbox environment plugin for type '{type}'. "
+            "Ensure the plugin is installed in your environment.",
+        )
+        return config
     config_deserialize = cast(
         ConfigDeserialize, getattr(sandboxenv_type, "config_deserialize")
     )

{inspect_ai-0.3.74.dist-info → inspect_ai-0.3.76.dist-info}/METADATA

@@ -1,10 +1,10 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: inspect_ai
-Version: 0.3.74
+Version: 0.3.76
 Summary: Framework for large language model evaluations
 Author: UK AI Security Institute
 License: MIT License
-Project-URL: Documentation, https://inspect.ai-safety-institute.org.uk/
+Project-URL: Documentation, https://inspect.aisi.org.uk/
 Project-URL: Source Code, https://github.com/UKGovernmentBEIS/inspect_ai
 Project-URL: Issue Tracker, https://github.com/UKGovernmentBEIS/inspect_ai/issues
 Classifier: Development Status :: 4 - Beta
@@ -60,11 +60,13 @@ Requires-Dist: goodfire; extra == "dev"
 Requires-Dist: griffe; extra == "dev"
 Requires-Dist: groq; extra == "dev"
 Requires-Dist: ipython; extra == "dev"
+Requires-Dist: markdown; extra == "dev"
 Requires-Dist: mistralai; extra == "dev"
 Requires-Dist: moto[server]; extra == "dev"
 Requires-Dist: mypy; extra == "dev"
 Requires-Dist: nbformat; extra == "dev"
 Requires-Dist: openai; extra == "dev"
+Requires-Dist: panflute; extra == "dev"
 Requires-Dist: pip; extra == "dev"
 Requires-Dist: pre-commit; extra == "dev"
 Requires-Dist: pylint; extra == "dev"
@@ -96,14 +98,15 @@ Requires-Dist: griffe; extra == "doc"
 Provides-Extra: dist
 Requires-Dist: twine; extra == "dist"
 Requires-Dist: build; extra == "dist"
+Dynamic: license-file
 
-[<img width="295" src="https://inspect.ai-safety-institute.org.uk/images/aisi-logo.svg" />](https://aisi.gov.uk/)
+[<img width="295" src="https://inspect.aisi.org.uk/images/aisi-logo.svg" />](https://aisi.gov.uk/)
 
 Welcome to Inspect, a framework for large language model evaluations created by the [UK AI Security Institute](https://aisi.gov.uk/).
 
 Inspect provides many built-in components, including facilities for prompt engineering, tool usage, multi-turn dialog, and model graded evaluations. Extensions to Inspect (e.g. to support new elicitation and scoring techniques) can be provided by other Python packages.
 
-To get started with Inspect, please see the documentation at <https://inspect.ai-safety-institute.org.uk/>.
+To get started with Inspect, please see the documentation at <https://inspect.aisi.org.uk/>.
 
 ***