inspect-ai 0.3.60__py3-none-any.whl → 0.3.62__py3-none-any.whl

inspect_ai/model/_providers/anthropic.py

@@ -12,6 +12,7 @@ else:
 
 from anthropic import (
     APIConnectionError,
+    APIStatusError,
     AsyncAnthropic,
     AsyncAnthropicBedrock,
     AsyncAnthropicVertex,
@@ -215,6 +216,17 @@ class AnthropicAPI(ModelAPI):
             # return output and call
             return output, model_call()
 
+        except APIStatusError as ex:
+            if ex.status_code == 413:
+                return ModelOutput.from_content(
+                    model=self.model_name,
+                    content=ex.message,
+                    stop_reason="model_length",
+                    error=ex.message,
+                ), model_call()
+            else:
+                raise ex
+
         except BadRequestError as ex:
             return self.handle_bad_request(ex), model_call()
 
@@ -291,9 +303,6 @@ class AnthropicAPI(ModelAPI):
         elif "content filtering" in error:
             content = "Sorry, but I am unable to help with that request."
             stop_reason = "content_filter"
-        else:
-            content = error
-            stop_reason = "unknown"
 
         if content and stop_reason:
             return ModelOutput.from_content(

inspect_ai/model/_providers/groq.py

@@ -294,8 +294,12 @@ def chat_tool_calls(message: Any, tools: list[ToolInfo]) -> Optional[List[ToolCa
 
 
 def chat_message_assistant(message: Any, tools: list[ToolInfo]) -> ChatMessageAssistant:
+    reasoning = getattr(message, "reasoning", None)
+    if reasoning is not None:
+        reasoning = str(reasoning)
     return ChatMessageAssistant(
         content=message.content or "",
         source="generate",
         tool_calls=chat_tool_calls(message, tools),
+        reasoning=reasoning,
     )

inspect_ai/model/_providers/openai.py

@@ -35,10 +35,12 @@ from .._model_output import (
     StopReason,
 )
 from .._openai import (
-    is_o1,
+    is_gpt,
     is_o1_full,
     is_o1_mini,
     is_o1_preview,
+    is_o3,
+    is_o_series,
     openai_chat_messages,
     openai_chat_tool_choice,
     openai_chat_tools,
@@ -140,8 +142,8 @@ class OpenAIAPI(ModelAPI):
     def is_azure(self) -> bool:
         return self.service == "azure"
 
-    def is_o1(self) -> bool:
-        return is_o1(self.model_name)
+    def is_o_series(self) -> bool:
+        return is_o_series(self.model_name)
 
     def is_o1_full(self) -> bool:
         return is_o1_full(self.model_name)
@@ -149,9 +151,15 @@ class OpenAIAPI(ModelAPI):
     def is_o1_mini(self) -> bool:
         return is_o1_mini(self.model_name)
 
+    def is_o3(self) -> bool:
+        return is_o3(self.model_name)
+
     def is_o1_preview(self) -> bool:
         return is_o1_preview(self.model_name)
 
+    def is_gpt(self) -> bool:
+        return is_gpt(self.model_name)
+
     async def generate(
         self,
         input: list[ChatMessage],
@@ -258,7 +266,7 @@ class OpenAIAPI(ModelAPI):
             model=self.model_name,
         )
         if config.max_tokens is not None:
-            if self.is_o1():
+            if self.is_o_series():
                 params["max_completion_tokens"] = config.max_tokens
             else:
                 params["max_tokens"] = config.max_tokens
@@ -273,10 +281,10 @@ class OpenAIAPI(ModelAPI):
         if config.seed is not None:
             params["seed"] = config.seed
         if config.temperature is not None:
-            if self.is_o1():
+            if self.is_o_series():
                 warn_once(
                     logger,
-                    "o1 models do not support the 'temperature' parameter (temperature is always 1).",
+                    "o series models do not support the 'temperature' parameter (temperature is always 1).",
                 )
             else:
                 params["temperature"] = config.temperature
@@ -293,9 +301,9 @@ class OpenAIAPI(ModelAPI):
             params["logprobs"] = config.logprobs
         if config.top_logprobs is not None:
             params["top_logprobs"] = config.top_logprobs
-        if tools and config.parallel_tool_calls is not None and not self.is_o1():
+        if tools and config.parallel_tool_calls is not None and not self.is_o_series():
             params["parallel_tool_calls"] = config.parallel_tool_calls
-        if config.reasoning_effort is not None and self.is_o1_full():
+        if config.reasoning_effort is not None and not self.is_gpt():
             params["reasoning_effort"] = config.reasoning_effort
 
         return params
@@ -312,7 +320,11 @@ class OpenAIAPI(ModelAPI):
         stop_reason: StopReason | None = None
         if e.code == "context_length_exceeded":
             stop_reason = "model_length"
-        elif e.code == "invalid_prompt":
+        elif (
+            e.code == "invalid_prompt"  # seems to happen for o1/o3
+            or e.code == "content_policy_violation"  # seems to happen for vision
+            or e.code == "content_filter"  # seems to happen on azure
+        ):
             stop_reason = "content_filter"
 
         if stop_reason:

inspect_ai/model/_providers/providers.py

@@ -16,7 +16,7 @@ from .._registry import modelapi
 def groq() -> type[ModelAPI]:
     FEATURE = "Groq API"
     PACKAGE = "groq"
-    MIN_VERSION = "0.11.0"
+    MIN_VERSION = "0.16.0"
 
     # verify we have the package
     try:

inspect_ai/model/_reasoning.py

@@ -0,0 +1,17 @@
+import re
+from typing import NamedTuple
+
+
+class ContentWithReasoning(NamedTuple):
+    content: str
+    reasoning: str
+
+
+def parse_content_with_reasoning(content: str) -> ContentWithReasoning | None:
+    match = re.match(r"\s*<think>(.*?)</think>(.*)", content, re.DOTALL)
+    if match:
+        return ContentWithReasoning(
+            content=match.group(2).strip(), reasoning=match.group(1).strip()
+        )
+    else:
+        return None

inspect_ai/solver/__init__.py

@@ -6,6 +6,7 @@ from ._chain import chain
 from ._critique import self_critique
 from ._fork import fork
 from ._human_agent.agent import human_agent
+from ._limit import SampleLimitExceededError
 from ._multiple_choice import MultipleChoiceTemplate, multiple_choice
 from ._plan import Plan, plan
 from ._prompt import chain_of_thought, prompt_template, system_message, user_message
@@ -37,6 +38,7 @@ __all__ = [
     "TaskState",
     "Generate",
     "MultipleChoiceTemplate",
+    "SampleLimitExceededError",
 ]
 
 

inspect_ai/solver/_basic_agent.py

@@ -1,8 +1,9 @@
 from logging import getLogger
-from typing import Callable, cast
+from typing import Awaitable, Callable, cast
 
 from typing_extensions import TypedDict, Unpack
 
+from inspect_ai._util._async import is_callable_coroutine
 from inspect_ai.model._cache import CachePolicy
 from inspect_ai.model._call_tools import call_tools
 from inspect_ai.model._chat_message import ChatMessageTool, ChatMessageUser
@@ -13,6 +14,7 @@ from inspect_ai.solver._chain import chain
 from inspect_ai.tool._tool import Tool, ToolResult, tool
 from inspect_ai.tool._tool_with import tool_with
 
+from ._limit import SampleLimitExceededError
 from ._prompt import system_message
 from ._solver import Generate, Solver, solver
 from ._task_state import TaskState
@@ -57,7 +59,9 @@ def basic_agent(
     max_tool_output: int | None = None,
     score_value: ValueToFloat | None = None,
     incorrect_message: str
-    | Callable[[TaskState, list[Score]], str] = DEFAULT_INCORRECT_MESSAGE,
+    | Callable[
+        [TaskState, list[Score]], str | Awaitable[str]
+    ] = DEFAULT_INCORRECT_MESSAGE,
     continue_message: str = DEFAULT_CONTINUE_MESSAGE,
     submit_name: str = DEFAULT_SUBMIT_NAME,
     submit_description: str = DEFAULT_SUBMIT_DESCRIPTION,
@@ -92,8 +96,9 @@ def basic_agent(
           Defaults to max_tool_output from active GenerateConfig.
        score_value (ValueToFloat): Function used to extract float from scores (defaults
          to standard value_to_float())
-       incorrect_message (str | Callable[[TaskState, list[Score]], str]): User message reply for an
-         incorrect submission from the model. Alternatively, a function which returns a message.
+       incorrect_message (str | Callable[[TaskState, list[Score]], str | Awaitable[str]]):
+         User message reply for an incorrect submission from the model. Alternatively,
+         a function which returns a message (function may optionally be async)
        continue_message (str): User message to urge the model to continue when it
          doesn't make a tool call.
        submit_name (str): Name for tool used to make submissions
@@ -167,61 +172,76 @@ def basic_agent(
             # track attempts
             attempts = 0
 
-            # main loop (state.completed checks message_limit and token_limit)
-            while not state.completed:
-                # generate output and append assistant message
-                state.output = await get_model().generate(
-                    input=state.messages, tools=state.tools, cache=cache
-                )
-                state.messages.append(state.output.message)
-
-                # check for context window overflow
-                if state.output.stop_reason == "model_length":
-                    from inspect_ai.log._transcript import transcript
-
-                    transcript().info("Agent terminated: model context window exceeded")
-                    break
-
-                # resolve tools calls (if any)
-                if state.output.message.tool_calls:
-                    # call tool functions
-                    tool_results = await call_tools(
-                        state.output.message, state.tools, max_output=max_tool_output
+            try:
+                # main loop (state.completed checks message_limit and token_limit)
+                while not state.completed:
+                    # generate output and append assistant message
+                    state.output = await get_model().generate(
+                        input=state.messages, tools=state.tools, cache=cache
                     )
-                    state.messages.extend(tool_results)
-
-                    # was an answer submitted?
-                    answer = submission(tool_results)
-                    if answer:
-                        # set the output to the answer for scoring
-                        state.output.completion = answer
-
-                        # exit if we are at max_attempts
-                        attempts += 1
-                        if attempts >= max_attempts:
-                            state.completed = True
-                            break
-
-                        # exit if the submission is successful
-                        answer_scores = await score(state)
-                        if score_value_fn(answer_scores[0].value) == 1.0:
-                            state.completed = True
-                            break
-
-                        # otherwise notify the model that it was incorrect and continue
-                        else:
-                            response_message = (
-                                incorrect_message(state, answer_scores)
-                                if callable(incorrect_message)
-                                else incorrect_message
-                            )
-                            state.messages.append(
-                                ChatMessageUser(content=response_message)
-                            )
-
-                # no tool calls, urge the model to continue
-                else:
-                    state.messages.append(ChatMessageUser(content=continue_message))
+                    state.messages.append(state.output.message)
+
+                    # check for context window overflow
+                    if state.output.stop_reason == "model_length":
+                        from inspect_ai.log._transcript import transcript
+
+                        transcript().info(
+                            "Agent terminated: model context window exceeded"
+                        )
+                        break
+
+                    # resolve tools calls (if any)
+                    if state.output.message.tool_calls:
+                        # call tool functions
+                        tool_results = await call_tools(
+                            state.output.message,
+                            state.tools,
+                            max_output=max_tool_output,
+                        )
+                        state.messages.extend(tool_results)
+
+                        # was an answer submitted?
+                        answer = submission(tool_results)
+                        if answer:
+                            # set the output to the answer for scoring
+                            state.output.completion = answer
+
+                            # exit if we are at max_attempts
+                            attempts += 1
+                            if attempts >= max_attempts:
+                                state.completed = True
+                                break
+
+                            # exit if the submission is successful
+                            answer_scores = await score(state)
+                            if score_value_fn(answer_scores[0].value) == 1.0:
+                                state.completed = True
+                                break
+
+                            # otherwise notify the model that it was incorrect and continue
+                            else:
+                                if is_callable_coroutine(incorrect_message):
+                                    response_message: str = await incorrect_message(
+                                        state, answer_scores
+                                    )  # type: ignore[misc,operator]
+                                elif callable(incorrect_message):
+                                    response_message = cast(
+                                        str, incorrect_message(state, answer_scores)
+                                    )
+                                else:
+                                    response_message = incorrect_message
+
+                                state.messages.append(
+                                    ChatMessageUser(content=response_message)
+                                )
+
+                    # no tool calls, urge the model to continue
+                    else:
+                        state.messages.append(ChatMessageUser(content=continue_message))
+
+            # propagate current state along with sample limit exceeded
+            except SampleLimitExceededError as ex:
+                raise ex.with_state(state)
 
             return state
 

inspect_ai/{util → solver}/_limit.py

@@ -1,5 +1,7 @@
 from typing import Literal
 
+from ._task_state import TaskState
+
 
 class SampleLimitExceededError(Exception):
     """Exception raised when a sample limit is exceeded.
@@ -18,9 +20,20 @@ class SampleLimitExceededError(Exception):
         value: int,
         limit: int,
         message: str | None = None,
+        state: TaskState | None = None,
     ) -> None:
         self.type = type
         self.value = value
         self.limit = limit
         self.message = f"Exceeded {type} limit: {limit:,}"
+        self.state = state
         super().__init__(message)
+
+    def with_state(self, state: TaskState) -> "SampleLimitExceededError":
+        return SampleLimitExceededError(
+            self.type,
+            value=self.value,
+            limit=self.limit,
+            message=self.message,
+            state=state,
+        )

inspect_ai/solver/_task_state.py

@@ -22,7 +22,6 @@ from inspect_ai.scorer._metric import Score
 from inspect_ai.scorer._target import Target
 from inspect_ai.tool import Tool, ToolChoice
 from inspect_ai.tool._tool_def import ToolDef
-from inspect_ai.util._limit import SampleLimitExceededError
 from inspect_ai.util._store import Store, store_jsonable
 from inspect_ai.util._store_model import SMT
 
@@ -173,7 +172,7 @@ class TaskState:
         self.metadata = metadata
         """Metadata from the `Sample` for this `TaskState`"""
 
-        self._messages: list[ChatMessage] = ChatMessageList(messages)
+        self._messages: list[ChatMessage] = ChatMessageList(messages, self)
         """
         Chat conversation history for sample.
 
@@ -272,7 +271,7 @@ class TaskState:
     @messages.setter
     def messages(self, messages: list[ChatMessage]) -> None:
         """Set messages in chat history."""
-        self._messages = ChatMessageList(messages)
+        self._messages = ChatMessageList(messages, self)
 
     @property
     def max_messages(self) -> int | None:
@@ -319,8 +318,32 @@ class TaskState:
 
     @property
     def completed(self) -> bool:
-        """Is the task completed."""
-        return self._completed
+        """Is the task completed.
+
+        Additionally, checks message and token limits and raises if they are exceeded.
+        """
+        from inspect_ai.log._samples import set_active_sample_total_messages
+
+        from ._limit import SampleLimitExceededError
+
+        # update messages
+        set_active_sample_total_messages(len(self.messages))
+
+        if self._completed:
+            return True
+        elif self.message_limit and len(self.messages) >= self.message_limit:
+            raise SampleLimitExceededError(
+                "message",
+                value=len(self.messages),
+                limit=self.message_limit,
+                state=self,
+            )
+        elif self.token_limit and self.token_usage >= self.token_limit:
+            raise SampleLimitExceededError(
+                "token", value=self.token_usage, limit=self.token_limit, state=self
+            )
+        else:
+            return self._completed
 
     @completed.setter
     def completed(self, completed: bool) -> None:
@@ -403,7 +426,8 @@ def sample_jsonable(sample: Sample) -> dict[str, Any]:
 
 
 class ChatMessageList(list[ChatMessage]):
-    def __init__(self, iterable: Iterable[ChatMessage]):
+    def __init__(self, iterable: Iterable[ChatMessage], parent_state: TaskState):
+        self.parent_state = parent_state
         items, length = self._iterable_length(iterable)
         self._check_size(length)
         super().__init__(items)
@@ -411,12 +435,18 @@ class ChatMessageList(list[ChatMessage]):
     def _check_size(self, additional_items: int = 1) -> None:
         from inspect_ai.log._samples import active_sample_message_limit
 
+        from ._limit import SampleLimitExceededError
+
         messages_limit = active_sample_message_limit()
         if messages_limit is not None:
             messages = len(self) + additional_items
             if messages > messages_limit:
                 raise SampleLimitExceededError(
-                    "message", value=messages, limit=messages_limit
+                    "message",
+                    value=messages,
+                    limit=messages_limit,
+                    message=None,
+                    state=self.parent_state,
                 )
 
     def append(self, item: ChatMessage) -> None:

inspect_ai/tool/_tools/_web_browser/_web_browser.py

@@ -345,7 +345,9 @@ async def web_browser_cmd(cmd: str, *args: str) -> str:
     if sandbox_env:
         store = store_as(WebBrowserStore)
         if not store.session_id:
-            result = await sandbox_env.exec(["python3", WEB_CLIENT_NEW_SESSION])
+            result = await sandbox_env.exec(
+                ["python3", WEB_CLIENT_NEW_SESSION], timeout=180
+            )
 
             if not result.success:
                 raise RuntimeError(

inspect_ai/tool/beta/_computer/_resources/Dockerfile

@@ -33,8 +33,6 @@ RUN apt-get update && \
 
 # Userland apt-get'able apps
 RUN apt-get install -y --no-install-recommends \
-      # A simple image viewer.
-      xpaint \
       # A calculator application.
       galculator && \
     apt-get clean
@@ -62,6 +60,10 @@ RUN apt-get install -y \
 # configure noVNC
 RUN ln -s /usr/share/novnc/vnc.html /usr/share/novnc/index.html
 
+# configure python alias
+RUN ln -s /usr/bin/python3 /usr/bin/python
+
+
 # We copy requirements.txt by itself so that changes to the scripts will be in a later layer
 # and we only pip install if requirements.txt changes
 COPY tool/requirements.txt /opt/inspect/tool/requirements.txt
@@ -78,7 +80,7 @@ RUN useradd -m -s /bin/bash -d $HOME $USERNAME
 RUN echo "${USERNAME} ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers
 USER ${USERNAME}
 WORKDIR $HOME
-COPY --chown=$USERNAME:$USERNAME image_home_dir/ $HOME
+ADD --chown=$USERNAME:$USERNAME image_home_dir/ $HOME
 
 # configure Firefox to skip all 'first run' UI
 RUN mkdir -p $HOME/.mozilla/firefox-esr/profile.default && \

inspect_ai/tool/beta/_computer/_resources/entrypoint/x11vnc_startup.sh

@@ -5,7 +5,7 @@ echo "starting vnc"
     -forever \
     -shared \
     -wait 50 \
-    -cursor most \
+    -multiptr \
     -cursor arrow \
     -rfbport 5900 \
     -nopw \

inspect_ai/tool/beta/_computer/_resources/image_home_dir/.config/Code/User/settings.json

@@ -0,0 +1,3 @@
+{
+  "security.workspace.trust.enabled": false
+}

inspect_ai/tool/beta/_computer/_resources/image_home_dir/.config/xfce4/xfconf/xfce-perchannel-xml/xfce4-panel.xml

@@ -0,0 +1,61 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<channel name="xfce4-panel" version="1.0">
+  <property name="configver" type="int" value="2"/>
+  <property name="panels" type="array">
+    <value type="int" value="1"/>
+    <property name="dark-mode" type="bool" value="true"/>
+    <property name="panel-1" type="empty">
+      <property name="position" type="string" value="p=6;x=0;y=0"/>
+      <property name="length" type="uint" value="100"/>
+      <property name="position-locked" type="bool" value="true"/>
+      <property name="icon-size" type="uint" value="16"/>
+      <property name="size" type="uint" value="26"/>
+      <property name="plugin-ids" type="array">
+        <value type="int" value="1"/>
+        <value type="int" value="2"/>
+        <value type="int" value="3"/>
+        <value type="int" value="4"/>
+        <value type="int" value="5"/>
+        <value type="int" value="6"/>
+        <value type="int" value="8"/>
+        <value type="int" value="10"/>
+        <value type="int" value="11"/>
+        <value type="int" value="12"/>
+        <value type="int" value="13"/>
+        <value type="int" value="14"/>
+      </property>
+    </property>
+  </property>
+  <property name="plugins" type="empty">
+    <property name="plugin-1" type="string" value="applicationsmenu"/>
+    <property name="plugin-2" type="string" value="tasklist">
+      <property name="grouping" type="uint" value="1"/>
+    </property>
+    <property name="plugin-3" type="string" value="separator">
+      <property name="expand" type="bool" value="true"/>
+      <property name="style" type="uint" value="0"/>
+    </property>
+    <property name="plugin-4" type="string" value="pager"/>
+    <property name="plugin-5" type="string" value="separator">
+      <property name="style" type="uint" value="0"/>
+    </property>
+    <property name="plugin-6" type="string" value="systray">
+      <property name="square-icons" type="bool" value="true"/>
+    </property>
+    <property name="plugin-8" type="string" value="pulseaudio">
+      <property name="enable-keyboard-shortcuts" type="bool" value="true"/>
+      <property name="show-notifications" type="bool" value="true"/>
+    </property>
+    <property name="plugin-9" type="string" value="power-manager-plugin"/>
+    <property name="plugin-10" type="string" value="notification-plugin"/>
+    <property name="plugin-11" type="string" value="separator">
+      <property name="style" type="uint" value="0"/>
+    </property>
+    <property name="plugin-12" type="string" value="clock"/>
+    <property name="plugin-13" type="string" value="separator">
+      <property name="style" type="uint" value="0"/>
+    </property>
+    <property name="plugin-14" type="string" value="actions"/>
+  </property>
+</channel>

inspect_ai/tool/beta/_computer/_resources/image_home_dir/.config/xfce4/xfconf/xfce-perchannel-xml/xfce4-screensaver.xml

@@ -0,0 +1,10 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<channel name="xfce4-screensaver" version="1.0">
+  <property name="saver" type="empty">
+    <property name="mode" type="int" value="0" />
+  </property>
+  <property name="lock" type="empty">
+    <property name="enabled" type="bool" value="false" />
+  </property>
+</channel>

inspect_ai/tool/beta/_computer/_resources/image_home_dir/Desktop/Terminal.desktop

@@ -0,0 +1,10 @@
+[Desktop Entry]
+Version=1.0
+Type=Application
+Name=Terminal
+Comment=Open Terminal
+Exec=/usr/bin/exo-open --launch TerminalEmulator
+Icon=utilities-terminal
+Path=
+Terminal=false
+StartupNotify=false

inspect_ai/util/__init__.py

@@ -3,7 +3,6 @@ from inspect_ai._util.trace import trace_action, trace_message
 from ._concurrency import concurrency
 from ._console import input_screen
 from ._display import DisplayType, display_type
-from ._limit import SampleLimitExceededError
 from ._panel import InputPanel, input_panel
 from ._resource import resource
 from ._sandbox import (
@@ -37,7 +36,6 @@ __all__ = [
     "input_panel",
     "input_screen",
     "OutputLimitExceededError",
-    "SampleLimitExceededError",
     "resource",
     "subprocess",
     "SandboxEnvironment",