inspect-ai 0.3.59__py3-none-any.whl → 0.3.61__py3-none-any.whl

inspect_ai/solver/_basic_agent.py

@@ -13,6 +13,7 @@ from inspect_ai.solver._chain import chain
 from inspect_ai.tool._tool import Tool, ToolResult, tool
 from inspect_ai.tool._tool_with import tool_with
 
+from ._limit import SampleLimitExceededError
 from ._prompt import system_message
 from ._solver import Generate, Solver, solver
 from ._task_state import TaskState
@@ -119,7 +120,7 @@ def basic_agent(
     # resolve tools
     if tools is None:
         tools = []
-    tools = tools if isinstance(tools, Solver) else use_tools(tools)
+    tools = tools if isinstance(tools, Solver) else use_tools(tools, append=True)
 
     # resolve score_value function
     score_value_fn = score_value or value_to_float()
@@ -167,61 +168,70 @@ def basic_agent(
             # track attempts
             attempts = 0
 
-            # main loop (state.completed checks message_limit and token_limit)
-            while not state.completed:
-                # generate output and append assistant message
-                state.output = await get_model().generate(
-                    input=state.messages, tools=state.tools, cache=cache
-                )
-                state.messages.append(state.output.message)
-
-                # check for context window overflow
-                if state.output.stop_reason == "model_length":
-                    from inspect_ai.log._transcript import transcript
-
-                    transcript().info("Agent terminated: model context window exceeded")
-                    break
-
-                # resolve tools calls (if any)
-                if state.output.message.tool_calls:
-                    # call tool functions
-                    tool_results = await call_tools(
-                        state.output.message, state.tools, max_output=max_tool_output
+            try:
+                # main loop (state.completed checks message_limit and token_limit)
+                while not state.completed:
+                    # generate output and append assistant message
+                    state.output = await get_model().generate(
+                        input=state.messages, tools=state.tools, cache=cache
                     )
-                    state.messages.extend(tool_results)
-
-                    # was an answer submitted?
-                    answer = submission(tool_results)
-                    if answer:
-                        # set the output to the answer for scoring
-                        state.output.completion = answer
-
-                        # exit if we are at max_attempts
-                        attempts += 1
-                        if attempts >= max_attempts:
-                            state.completed = True
-                            break
-
-                        # exit if the submission is successful
-                        answer_scores = await score(state)
-                        if score_value_fn(answer_scores[0].value) == 1.0:
-                            state.completed = True
-                            break
-
-                        # otherwise notify the model that it was incorrect and continue
-                        else:
-                            response_message = (
-                                incorrect_message(state, answer_scores)
-                                if callable(incorrect_message)
-                                else incorrect_message
-                            )
-                            state.messages.append(
-                                ChatMessageUser(content=response_message)
-                            )
-
-                # no tool calls, urge the model to continue
-                else:
-                    state.messages.append(ChatMessageUser(content=continue_message))
+                    state.messages.append(state.output.message)
+
+                    # check for context window overflow
+                    if state.output.stop_reason == "model_length":
+                        from inspect_ai.log._transcript import transcript
+
+                        transcript().info(
+                            "Agent terminated: model context window exceeded"
+                        )
+                        break
+
+                    # resolve tools calls (if any)
+                    if state.output.message.tool_calls:
+                        # call tool functions
+                        tool_results = await call_tools(
+                            state.output.message,
+                            state.tools,
+                            max_output=max_tool_output,
+                        )
+                        state.messages.extend(tool_results)
+
+                        # was an answer submitted?
+                        answer = submission(tool_results)
+                        if answer:
+                            # set the output to the answer for scoring
+                            state.output.completion = answer
+
+                            # exit if we are at max_attempts
+                            attempts += 1
+                            if attempts >= max_attempts:
+                                state.completed = True
+                                break
+
+                            # exit if the submission is successful
+                            answer_scores = await score(state)
+                            if score_value_fn(answer_scores[0].value) == 1.0:
+                                state.completed = True
+                                break
+
+                            # otherwise notify the model that it was incorrect and continue
+                            else:
+                                response_message = (
+                                    incorrect_message(state, answer_scores)
+                                    if callable(incorrect_message)
+                                    else incorrect_message
+                                )
+                                state.messages.append(
+                                    ChatMessageUser(content=response_message)
+                                )
+
+                    # no tool calls, urge the model to continue
+                    else:
+                        state.messages.append(ChatMessageUser(content=continue_message))
+
+            # propagate current state along with sample limit exceeded
+            except SampleLimitExceededError as ex:
+                raise ex.with_state(state)
 
             return state
 

inspect_ai/solver/_bridge/__init__.py

@@ -0,0 +1,3 @@
+from .bridge import bridge
+
+__all__ = ["bridge"]

inspect_ai/solver/_bridge/bridge.py

@@ -0,0 +1,100 @@
+from typing import Any, Awaitable, Callable
+
+from jsonschema import Draft7Validator
+from pydantic import BaseModel, Field, ValidationError
+from pydantic_core import to_json
+
+from inspect_ai._util._async import is_callable_coroutine
+from inspect_ai.model._chat_message import ChatMessage, ChatMessageUser
+from inspect_ai.model._providers.providers import validate_openai_client
+from inspect_ai.scorer._metric import Score
+
+from .._solver import Generate, Solver, solver
+from .._task_state import TaskState
+
+
+@solver
+def bridge(agent: Callable[[dict[str, Any]], Awaitable[dict[str, Any]]]) -> Solver:
+    """Bridge an external agent into an Inspect Solver.
+
+    See documentation at https://inspect.ai-safety-institute.org.uk/agent-bridge.html
+
+    Args:
+      agent: Callable which takes a sample `dict` and returns a result `dict`.
+
+    Returns:
+      Standard Inspect solver.
+    """
+    validate_openai_client("Solver bridge()")
+
+    from openai.types.chat import ChatCompletionMessageParam
+
+    from inspect_ai.model._openai import (
+        chat_messages_from_openai,
+        openai_chat_messages,
+    )
+
+    from .patch import openai_request_to_inspect_model
+
+    class BridgeSample(BaseModel):
+        sample_id: str
+        epoch: int
+        input: list[ChatCompletionMessageParam]
+        metadata: dict[str, Any]
+        target: list[str]
+
+    class BridgeResult(BaseModel):
+        output: str
+        messages: list[ChatCompletionMessageParam] | None = Field(default=None)
+        scores: dict[str, Score] | None = Field(default=None)
+
+    result_schema = BridgeResult.model_json_schema()
+    result_validator = Draft7Validator(result_schema)
+
+    # validate that the agent is an async function
+    if not is_callable_coroutine(agent):
+        raise TypeError(f"'{agent.__name__}' is not declared as an async callable.")
+
+    async def solve(state: TaskState, generate: Generate) -> TaskState:
+        # resolve input to array
+        input: list[ChatMessage] = (
+            [ChatMessageUser(content=state.input)]
+            if isinstance(state.input, str)
+            else state.input
+        )
+
+        # create sample
+        sample = BridgeSample(
+            sample_id=str(state.sample_id),
+            epoch=state.epoch,
+            input=await openai_chat_messages(input, state.model.name),
+            metadata=state.metadata,
+            target=list(state.target),
+        )
+
+        # run target function
+        async with openai_request_to_inspect_model():
+            # call the function
+            result_dict = await agent(sample.model_dump())
+            try:
+                result = BridgeResult.model_validate(result_dict)
+            except ValidationError:
+                # if we fail to validate provide a better human readable error
+                errors = list(result_validator.iter_errors(result_dict))
+                message = "\n".join(
+                    ["Result returned from bridged solver is not valid:"]
+                    + [f" - {error.message}" for error in errors]
+                    + ["", to_json(result_dict, indent=2).decode()]
+                )
+                raise ValueError(message)
+
+        # update and return state
+        state.output.completion = result.output
+        if result.messages is not None:
+            state.messages = chat_messages_from_openai(result.messages)
+        if result.scores is not None:
+            state.scores = result.scores
+
+        return state
+
+    return solve

inspect_ai/solver/_bridge/patch.py

@@ -0,0 +1,170 @@
+import contextlib
+import re
+from contextvars import ContextVar
+from functools import wraps
+from time import time
+from typing import Any, AsyncGenerator, Optional, Type, cast
+
+from openai._base_client import AsyncAPIClient, _AsyncStreamT
+from openai._models import FinalRequestOptions
+from openai._types import ResponseT
+from openai.types.chat import (
+    ChatCompletion,
+    ChatCompletionMessageParam,
+    ChatCompletionToolParam,
+)
+from shortuuid import uuid
+
+from inspect_ai.model._generate_config import GenerateConfig
+from inspect_ai.model._model import get_model
+from inspect_ai.model._openai import (
+    chat_messages_from_openai,
+    openai_chat_choices,
+    openai_completion_usage,
+)
+from inspect_ai.solver._task_state import sample_state
+from inspect_ai.tool._tool_info import ToolInfo
+from inspect_ai.tool._tool_params import ToolParams
+
+
+@contextlib.asynccontextmanager
+async def openai_request_to_inspect_model() -> AsyncGenerator[None, None]:
+    # ensure one time init
+    init_openai_request_patch()
+
+    # set the patch enabled for this context and child coroutines
+    token = _patch_enabled.set(True)
+    try:
+        yield
+    finally:
+        _patch_enabled.reset(token)
+
+
+_patch_initialised: bool = False
+
+_patch_enabled: ContextVar[bool] = ContextVar(
+    "openai_request_patch_enabled", default=False
+)
+
+
+def init_openai_request_patch() -> None:
+    global _patch_initialised
+    if not _patch_initialised:
+        # get reference to original method
+        original_request = getattr(AsyncAPIClient, "request")
+        if original_request is None:
+            raise RuntimeError("Couldn't find 'request' method on AsyncAPIClient")
+
+        @wraps(original_request)
+        async def patched_request(
+            self: AsyncAPIClient,
+            cast_to: Type[ResponseT],
+            options: FinalRequestOptions,
+            *,
+            stream: bool = False,
+            stream_cls: type[_AsyncStreamT] | None = None,
+            remaining_retries: Optional[int] = None,
+        ) -> Any:
+            # we have patched the underlying request method so now need to figure out when to
+            # patch and when to stand down
+            if (
+                # enabled for this coroutine
+                _patch_enabled.get()
+                # completions request
+                and options.url == "/chat/completions"
+                # call to openai not another service (e.g. TogetherAI)
+                and self.base_url == "https://api.openai.com/v1/"
+            ):
+                # must also be an explicit request for an inspect model
+                json_data = cast(dict[str, Any], options.json_data)
+                model_name = str(json_data["model"])
+                if re.match(r"^inspect/?", model_name):
+                    return await inspect_model_request(model_name, options)
+
+            # otherwise just delegate
+            return await original_request(
+                self,
+                cast_to,
+                options,
+                stream=stream,
+                stream_cls=stream_cls,
+                remaining_retries=remaining_retries,
+            )
+
+        setattr(AsyncAPIClient, "request", patched_request)
+
+
+async def inspect_model_request(
+    model_name: str, options: FinalRequestOptions
+) -> ChatCompletion:
+    # convert openai messages to inspect messages
+    json_data = cast(dict[str, Any], options.json_data)
+    messages: list[ChatCompletionMessageParam] = json_data["messages"]
+    input = chat_messages_from_openai(messages)
+
+    # convert openai tools to inspect tools
+    tools: list[ChatCompletionToolParam] = json_data.get("tools", [])
+    inspect_tools: list[ToolInfo] = []
+    for tool in tools:
+        function = tool["function"].copy()
+        inspect_tools.append(
+            ToolInfo(
+                name=function["name"],
+                description=function["description"],
+                parameters=ToolParams.model_validate(function["parameters"]),
+            )
+        )
+
+    # resolve model
+    if model_name == "inspect":
+        model = get_model()
+    else:
+        model = get_model(model_name.removeprefix("inspect/"))
+
+    output = await model.generate(
+        input=input,
+        tools=inspect_tools,
+        config=generate_config_from_openai(options),
+    )
+
+    # if we are using the "default" inspect model for the task, update state.messages
+    if model_name == "inspect":
+        state = sample_state()
+        if state:
+            state.messages = input + [output.choices[0].message]
+
+    # inspect completion to openai completion
+    return ChatCompletion(
+        id=uuid(),
+        created=int(time()),
+        object="chat.completion",
+        choices=openai_chat_choices(output.choices),
+        model=model_name,
+        usage=openai_completion_usage(output.usage) if output.usage else None,
+    )
+
+
+def generate_config_from_openai(options: FinalRequestOptions) -> GenerateConfig:
+    # get options dict
+    json_data = cast(dict[str, Any], options.json_data)
+
+    config = GenerateConfig()
+    config.max_tokens = json_data.get(
+        "max_completion_tokens", json_data.get("max_tokens", None)
+    )
+    config.top_p = json_data.get("top_p", None)
+    config.temperature = json_data.get("temperature", None)
+    stop = json_data.get("stop", None)
+    if stop:
+        config.stop_seqs = [stop] if isinstance(stop, str) else stop
+    config.frequency_penalty = json_data.get("frequency_penalty", None)
+    config.presence_penalty = json_data.get("presence_penalty", None)
+    config.seed = json_data.get("seed", None)
+    config.num_choices = json_data.get("n", None)
+    config.logprobs = json_data.get("logprobs", None)
+    config.top_logprobs = json_data.get("top_logprobs", None)
+    config.logit_bias = json_data.get("logit_bias", None)
+    config.parallel_tool_calls = json_data.get("parallel_tool_calls", None)
+    config.reasoning_effort = json_data.get("reasoning_effort", None)
+
+    return config

inspect_ai/{util → solver}/_limit.py

@@ -1,5 +1,7 @@
 from typing import Literal
 
+from ._task_state import TaskState
+
 
 class SampleLimitExceededError(Exception):
     """Exception raised when a sample limit is exceeded.
@@ -18,9 +20,20 @@ class SampleLimitExceededError(Exception):
         value: int,
         limit: int,
         message: str | None = None,
+        state: TaskState | None = None,
     ) -> None:
         self.type = type
         self.value = value
         self.limit = limit
         self.message = f"Exceeded {type} limit: {limit:,}"
+        self.state = state
         super().__init__(message)
+
+    def with_state(self, state: TaskState) -> "SampleLimitExceededError":
+        return SampleLimitExceededError(
+            self.type,
+            value=self.value,
+            limit=self.limit,
+            message=self.message,
+            state=state,
+        )

inspect_ai/solver/_solver.py

@@ -180,6 +180,7 @@ def solver(
             solver_type, name if name else getattr(solver_type, "__name__")
         )
 
+        @wraps(solver_type)
         def solver_wrapper(*args: P.args, **kwargs: P.kwargs) -> Solver:
             solver = solver_type(*args, **kwargs)
 
@@ -193,6 +194,7 @@ def solver(
             if inspect.isclass(type(solver)):
                 original_call = solver.__call__
 
+                @wraps(original_call)
                 async def call_with_state(
                     state: TaskState, generate: Generate
                 ) -> TaskState:
@@ -225,6 +227,10 @@ def solver(
 
             return registered_solver
 
+        # functools.wraps overrides the return type annotation of the inner function, so
+        # we explicitly set it again
+        solver_wrapper.__annotations__["return"] = Solver
+
         return solver_register(cast(Callable[P, Solver], solver_wrapper), solver_name)
 
     # for decorators with an explicit name, one more wrapper for the name

inspect_ai/solver/_task_state.py

@@ -22,7 +22,6 @@ from inspect_ai.scorer._metric import Score
 from inspect_ai.scorer._target import Target
 from inspect_ai.tool import Tool, ToolChoice
 from inspect_ai.tool._tool_def import ToolDef
-from inspect_ai.util._limit import SampleLimitExceededError
 from inspect_ai.util._store import Store, store_jsonable
 from inspect_ai.util._store_model import SMT
 
@@ -173,7 +172,7 @@ class TaskState:
         self.metadata = metadata
         """Metadata from the `Sample` for this `TaskState`"""
 
-        self._messages: list[ChatMessage] = ChatMessageList(messages)
+        self._messages: list[ChatMessage] = ChatMessageList(messages, self)
         """
         Chat conversation history for sample.
 
@@ -272,7 +271,7 @@ class TaskState:
     @messages.setter
     def messages(self, messages: list[ChatMessage]) -> None:
         """Set messages in chat history."""
-        self._messages = ChatMessageList(messages)
+        self._messages = ChatMessageList(messages, self)
 
     @property
     def max_messages(self) -> int | None:
@@ -319,8 +318,32 @@ class TaskState:
 
     @property
     def completed(self) -> bool:
-        """Is the task completed."""
-        return self._completed
+        """Is the task completed.
+
+        Additionally, checks message and token limits and raises if they are exceeded.
+        """
+        from inspect_ai.log._samples import set_active_sample_total_messages
+
+        from ._limit import SampleLimitExceededError
+
+        # update messages
+        set_active_sample_total_messages(len(self.messages))
+
+        if self._completed:
+            return True
+        elif self.message_limit and len(self.messages) >= self.message_limit:
+            raise SampleLimitExceededError(
+                "message",
+                value=len(self.messages),
+                limit=self.message_limit,
+                state=self,
+            )
+        elif self.token_limit and self.token_usage >= self.token_limit:
+            raise SampleLimitExceededError(
+                "token", value=self.token_usage, limit=self.token_limit, state=self
+            )
+        else:
+            return self._completed
 
     @completed.setter
     def completed(self, completed: bool) -> None:
@@ -403,7 +426,8 @@ def sample_jsonable(sample: Sample) -> dict[str, Any]:
 
 
 class ChatMessageList(list[ChatMessage]):
-    def __init__(self, iterable: Iterable[ChatMessage]):
+    def __init__(self, iterable: Iterable[ChatMessage], parent_state: TaskState):
+        self.parent_state = parent_state
         items, length = self._iterable_length(iterable)
         self._check_size(length)
         super().__init__(items)
@@ -411,12 +435,18 @@ class ChatMessageList(list[ChatMessage]):
     def _check_size(self, additional_items: int = 1) -> None:
         from inspect_ai.log._samples import active_sample_message_limit
 
+        from ._limit import SampleLimitExceededError
+
         messages_limit = active_sample_message_limit()
         if messages_limit is not None:
             messages = len(self) + additional_items
             if messages > messages_limit:
                 raise SampleLimitExceededError(
-                    "message", value=messages, limit=messages_limit
+                    "message",
+                    value=messages,
+                    limit=messages_limit,
+                    message=None,
+                    state=self.parent_state,
                 )
 
     def append(self, item: ChatMessage) -> None:

inspect_ai/tool/_tools/_web_browser/_web_browser.py

@@ -345,7 +345,9 @@ async def web_browser_cmd(cmd: str, *args: str) -> str:
     if sandbox_env:
         store = store_as(WebBrowserStore)
         if not store.session_id:
-            result = await sandbox_env.exec(["python3", WEB_CLIENT_NEW_SESSION])
+            result = await sandbox_env.exec(
+                ["python3", WEB_CLIENT_NEW_SESSION], timeout=180
+            )
 
             if not result.success:
                 raise RuntimeError(

inspect_ai/tool/beta/_computer/_resources/Dockerfile

@@ -33,8 +33,6 @@ RUN apt-get update && \
 
 # Userland apt-get'able apps
 RUN apt-get install -y --no-install-recommends \
-      # A simple image viewer.
-      xpaint \
       # A calculator application.
       galculator && \
     apt-get clean
@@ -78,7 +76,7 @@ RUN useradd -m -s /bin/bash -d $HOME $USERNAME
 RUN echo "${USERNAME} ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers
 USER ${USERNAME}
 WORKDIR $HOME
-COPY --chown=$USERNAME:$USERNAME image_home_dir/ $HOME
+ADD --chown=$USERNAME:$USERNAME image_home_dir/ $HOME
 
 # configure Firefox to skip all 'first run' UI
 RUN mkdir -p $HOME/.mozilla/firefox-esr/profile.default && \

inspect_ai/tool/beta/_computer/_resources/entrypoint/x11vnc_startup.sh

@@ -5,7 +5,7 @@ echo "starting vnc"
     -forever \
     -shared \
     -wait 50 \
-    -cursor most \
+    -multiptr \
     -cursor arrow \
     -rfbport 5900 \
     -nopw \

inspect_ai/tool/beta/_computer/_resources/image_home_dir/.config/xfce4/xfconf/xfce-perchannel-xml/xfce4-screensaver.xml

@@ -0,0 +1,10 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<channel name="xfce4-screensaver" version="1.0">
+  <property name="saver" type="empty">
+    <property name="mode" type="int" value="0" />
+  </property>
+  <property name="lock" type="empty">
+    <property name="enabled" type="bool" value="false" />
+  </property>
+</channel>

inspect_ai/util/__init__.py

@@ -3,7 +3,6 @@ from inspect_ai._util.trace import trace_action, trace_message
 from ._concurrency import concurrency
 from ._console import input_screen
 from ._display import DisplayType, display_type
-from ._limit import SampleLimitExceededError
 from ._panel import InputPanel, input_panel
 from ._resource import resource
 from ._sandbox import (
@@ -37,7 +36,6 @@ __all__ = [
     "input_panel",
     "input_screen",
     "OutputLimitExceededError",
-    "SampleLimitExceededError",
     "resource",
     "subprocess",
     "SandboxEnvironment",

inspect_ai/util/_display.py

@@ -49,3 +49,8 @@ def display_type() -> DisplayType:
         return _display_type
     else:
         return init_display_type()
+
+
+def display_type_initialized() -> bool:
+    global _display_type
+    return _display_type is not None

inspect_ai/util/_sandbox/docker/prereqs.py

@@ -57,7 +57,7 @@ async def validate_docker_compose(
     version: str = DOCKER_COMPOSE_REQUIRED_VERSION,
 ) -> None:
     def parse_version(stdout: str) -> semver.Version:
-        version = json.loads(stdout)["version"].removeprefix("v")
+        version = json.loads(stdout)["version"].removeprefix("v").split("+")[0]
         return semver.Version.parse(version)
 
     await validate_version(