inspect-ai 0.3.104__py3-none-any.whl → 0.3.106__py3-none-any.whl

inspect_ai/_view/www/src/client/api/client-api.ts

@@ -335,6 +335,9 @@ export const clientApi = (api: LogViewAPI, log_file?: string): ClientAPI => {
     ) => {
       return api.download_file(download_file, file_contents);
     },
+    log_message: (log_file: string, message: string) => {
+      return api.log_message(log_file, message);
+    },
     get_log_pending_samples: api.eval_pending_samples
       ? get_log_pending_samples
       : undefined,

inspect_ai/_view/www/src/client/api/jsonrpc.ts

@@ -41,6 +41,7 @@ export const kMethodEvalLogBytes = "eval_log_bytes";
 export const kMethodEvalLogHeaders = "eval_log_headers";
 export const kMethodPendingSamples = "eval_log_pending_samples";
 export const kMethodSampleData = "eval_log_sample_data";
+export const kMethodLogMessage = "log_message";
 
 export const kJsonRpcParseError = -32700;
 export const kJsonRpcInvalidRequest = -32600;

inspect_ai/_view/www/src/client/api/types.ts

@@ -115,6 +115,7 @@ export interface SampleSummary {
   scores: Scores1;
   error?: string;
   limit?: string;
+  metadata?: Record<string, any>;
   completed?: boolean;
   retries?: number;
 }
@@ -149,6 +150,7 @@ export interface LogViewAPI {
     end: number,
   ) => Promise<Uint8Array>;
   eval_log_headers: (log_files: string[]) => Promise<EvalLog[]>;
+  log_message: (log_file: string, message: string) => Promise<void>;
   download_file: (
     filename: string,
     filecontents: string | Blob | ArrayBuffer | ArrayBufferView,
@@ -177,6 +179,7 @@ export interface ClientAPI {
     id: string | number,
     epoch: number,
   ) => Promise<EvalSample | undefined>;
+  log_message?: (log_file: string, message: string) => Promise<void>;
   download_file: (
     file_name: string,
     file_contents: string | Blob | ArrayBuffer | ArrayBufferView,

inspect_ai/_view/www/src/state/samplePolling.ts

@@ -1,6 +1,7 @@
 import { Event } from "../app/types";
 import {
   AttachmentData,
+  ClientAPI,
   EventData,
   SampleData,
   SampleSummary,
@@ -183,6 +184,8 @@ export function createSamplePolling(
           const processedEvents = processEvents(
             sampleDataResponse.sampleData,
             pollingState,
+            api,
+            logFile,
           );
 
           // update max attachment id
@@ -268,7 +271,12 @@ function processAttachments(
   });
 }
 
-function processEvents(sampleData: SampleData, pollingState: PollingState) {
+function processEvents(
+  sampleData: SampleData,
+  pollingState: PollingState,
+  api: ClientAPI,
+  log_file: string,
+) {
   // Go through each event and resolve it, either appending or replacing
   log.debug(`Processing ${sampleData.events.length} events`);
   if (sampleData.events.length === 0) {
@@ -289,6 +297,14 @@ function processEvents(sampleData: SampleData, pollingState: PollingState) {
           attachmentId,
           available_attachments: Object.keys(pollingState.attachments),
         };
+
+        if (api.log_message) {
+          api.log_message(
+            log_file,
+            `Unable to resolve attachment ${attachmentId}\n` +
+              JSON.stringify(snapshot),
+          );
+        }
         console.warn(`Unable to resolve attachment ${attachmentId}`, snapshot);
       },
     );

inspect_ai/agent/_handoff.py

@@ -6,7 +6,7 @@ from inspect_ai._util.registry import (
     registry_unqualified_name,
     set_registry_info,
 )
-from inspect_ai.tool._tool import Tool, ToolResult, ToolSource
+from inspect_ai.tool._tool import TOOL_PARALLEL, Tool, ToolResult, ToolSource
 from inspect_ai.tool._tool_def import ToolDef
 from inspect_ai.tool._tool_description import ToolDescription, set_tool_description
 from inspect_ai.util._limit import Limit
@@ -61,7 +61,10 @@ def handoff(
         agent, tool_info.name, input_filter, output_filter, limits, **agent_kwargs
     )
     tool_name = tool_name or f"transfer_to_{tool_info.name}"
-    set_registry_info(agent_tool, RegistryInfo(type="tool", name=tool_name))
+    set_registry_info(
+        agent_tool,
+        RegistryInfo(type="tool", name=tool_name, metadata={TOOL_PARALLEL: False}),
+    )
     set_tool_description(
         agent_tool,
         ToolDescription(

inspect_ai/agent/_react.py

@@ -82,9 +82,8 @@ def react(
           the submit tool within the message. Alternatively, an async function
           to call to determine whether the loop should continue and what message
           to play back. Note that this function is called on _every_ iteration of
-          the loop. If you return a `str` with a custom message it will be
-          sent to the model as a user message only in the case that no tool
-          calls were made.
+          the loop so if you only want to send a message back when the model fails
+          to call tools you need to code that behavior explicitly.
        truncation: Truncate the conversation history in the event of a context
           window overflow. Defaults to "disabled" which does no truncation. Pass
           "auto" to use `trim_messages()` to reduce the context size. Pass a
@@ -246,13 +245,12 @@ def react(
                                 )
                             )
                     elif isinstance(do_continue, str):
-                        # if there were no tool calls we need to send back the user message
-                        if not state.output.message.tool_calls:
-                            state.messages.append(
-                                ChatMessageUser(
-                                    content=do_continue.format(submit=submit_tool.name)
-                                )
+                        # send back the user message
+                        state.messages.append(
+                            ChatMessageUser(
+                                content=do_continue.format(submit=submit_tool.name)
                             )
+                        )
                     else:  # do_continue is False
                         break
 
@@ -328,11 +326,14 @@ def react_no_submit(
                 if on_continue:
                     do_continue = await _call_on_continue(on_continue, state)
                     if do_continue is True:
-                        do_continue = DEFAULT_CONTINUE_PROMOT_NO_SUBMIT
-                    if do_continue:
-                        # send back user message if there are no tool calls
                         if not state.output.message.tool_calls:
-                            state.messages.append(ChatMessageUser(content=do_continue))
+                            state.messages.append(
+                                ChatMessageUser(
+                                    content=DEFAULT_CONTINUE_PROMOT_NO_SUBMIT
+                                )
+                            )
+                    elif isinstance(do_continue, str):
+                        state.messages.append(ChatMessageUser(content=do_continue))
                     else:
                         break
                 elif not state.output.message.tool_calls:
@@ -361,13 +362,13 @@ def _prompt_to_system_message(
                 and ("{submit}" not in prompt.assistant_prompt)
                 and prompt.submit_prompt
             ):
-                assistant_prompt = f"{prompt.assistant_prompt}\n{prompt.submit_prompt}"
+                assistant_prompt = f"{prompt.assistant_prompt}\n{prompt.submit_prompt.format(submit=submit_tool)}"
             else:
-                assistant_prompt = prompt.assistant_prompt
+                assistant_prompt = prompt.assistant_prompt.format(
+                    submit=submit_tool or "submit"
+                )
             prompt_lines.append(assistant_prompt)
-        prompt_content = "\n\n".join(prompt_lines).format(
-            submit=submit_tool or "submit"
-        )
+        prompt_content = "\n\n".join(prompt_lines)
         system_message: ChatMessage | None = ChatMessageSystem(content=prompt_content)
     else:
         system_message = None
@@ -471,12 +472,34 @@ def _remove_submit_tool(
 
         # remove submit tool from assistant messages
         if isinstance(message, ChatMessageAssistant) and message.tool_calls:
-            tools_calls = [
+            new_tools_calls = [
                 tool_call
                 for tool_call in message.tool_calls
                 if tool_call.function != submit_name
             ]
-            message = message.model_copy(update=dict(tool_calls=tools_calls))
+
+            # If a submit tool call was removed, we need to update the message
+            if len(new_tools_calls) < len(message.tool_calls):
+                message = message.model_copy(
+                    update=dict(
+                        tool_calls=new_tools_calls,
+                        # Some models (OpenAI) don't like to see the reasoning
+                        # content item that led to the submit tool call, so we
+                        # have to remove it too.
+                        content=(
+                            [
+                                content
+                                for content in message.content
+                                if (
+                                    isinstance(content, str)
+                                    or content.type != "reasoning"
+                                )
+                            ]
+                            if isinstance(message.content, list)
+                            else message.content
+                        ),
+                    )
+                )
 
         # always append message
         filtered.append(message)

inspect_ai/dataset/_dataset.py

@@ -308,7 +308,7 @@ class MemoryDataset(Dataset):
 
     @override
     def shuffle(self, seed: int | None = None) -> None:
-        if seed:
+        if seed is not None:
             random.Random(seed).shuffle(self.samples)
         else:
             random.shuffle(self.samples)

inspect_ai/log/_samples.py

@@ -3,6 +3,7 @@ from contextvars import ContextVar
 from datetime import datetime
 from typing import AsyncGenerator, Iterator, Literal
 
+from anyio.abc import TaskGroup
 from shortuuid import uuid
 
 from inspect_ai.dataset._dataset import Sample
@@ -28,6 +29,7 @@ class ActiveSample:
         fails_on_error: bool,
         transcript: Transcript,
         sandboxes: dict[str, SandboxConnection],
+        tg: TaskGroup,
     ) -> None:
         self.id = uuid()
         self.started: float | None = None
@@ -47,6 +49,7 @@ class ActiveSample:
         self.transcript = transcript
         self.sandboxes = sandboxes
         self._interrupt_action: Literal["score", "error"] | None = None
+        self.tg = tg
 
     @property
     def running_time(self) -> float:
@@ -86,6 +89,7 @@ async def active_sample(
     working_limit: int | None,
     fails_on_error: bool,
     transcript: Transcript,
+    tg: TaskGroup,
 ) -> AsyncGenerator[ActiveSample, None]:
     # create the sample
     active = ActiveSample(
@@ -101,6 +105,7 @@ async def active_sample(
         sandboxes=await sandbox_connections(),
         fails_on_error=fails_on_error,
         transcript=transcript,
+        tg=tg,
     )
 
     _active_samples.append(active)

inspect_ai/model/_call_tools.py

@@ -534,11 +534,11 @@ def prepend_agent_name(
         content = copy(message.content)
         for i in range(0, len(content)):
             if isinstance(content[i], ContentText):
-                content[i] = content[i].model_copy(
-                    update=dict(
-                        text=f"[{agent_name}] {cast(ContentText, content[i]).text}"
+                text = cast(ContentText, content[i]).text
+                if text:
+                    content[i] = content[i].model_copy(
+                        update=dict(text=f"[{agent_name}] {text}")
                     )
-                )
                 break
         return message.model_copy(update=dict(content=content))
 

inspect_ai/model/_providers/_openai_web_search.py

@@ -14,7 +14,7 @@ def maybe_web_search_tool(model_name: str, tool: ToolInfo) -> WebSearchToolParam
             tool.name == "web_search"
             and tool.options
             and "openai" in tool.options
-            and model_name in COMPATIBLE_MODELS
+            and any(model_name.startswith(model) for model in COMPATIBLE_MODELS)
         )
         else None
     )

inspect_ai/model/_providers/anthropic.py

@@ -41,6 +41,7 @@ from anthropic.types import (
 from anthropic.types.beta import (
     BetaToolComputerUse20250124Param,
     BetaToolTextEditor20241022Param,
+    BetaToolTextEditor20250429Param,
 )
 from pydantic import JsonValue
 from typing_extensions import override
@@ -397,6 +398,9 @@ class AnthropicAPI(ModelAPI):
     def is_claude_3_7(self) -> bool:
         return "claude-3-7-" in self.service_model_name()
 
+    def is_claude_4(self) -> bool:
+        return re.search(r"claude-4-[a-zA-Z]", self.service_model_name()) is not None
+
     @override
     def connection_key(self) -> str:
         return str(self.api_key)
@@ -627,7 +631,17 @@ class AnthropicAPI(ModelAPI):
 
     def text_editor_tool_param(
         self, tool: ToolInfo
-    ) -> ToolTextEditor20250124Param | BetaToolTextEditor20241022Param | None:
+    ) -> (
+        ToolTextEditor20250124Param
+        | BetaToolTextEditor20241022Param
+        | BetaToolTextEditor20250429Param
+        | None
+    ):
+        # See: https://docs.anthropic.com/en/docs/agents-and-tools/tool-use/text-editor-tool#before-using-the-text-editor-tool
+        # TODO: It would be great to enhance our `is_claude_xxx` functions to help here.
+        if self.model_name.startswith(("claude-3-5-haiku", "claude-3-opus")):
+            return None
+
         # check for compatible 'text editor' tool
         if tool.name == "text_editor" and (
             sorted(tool.parameters.properties.keys())
@@ -644,7 +658,11 @@ class AnthropicAPI(ModelAPI):
             )
         ):
             return (
-                BetaToolTextEditor20241022Param(
+                BetaToolTextEditor20250429Param(
+                    type="text_editor_20250429", name="str_replace_based_edit_tool"
+                )
+                if self.is_claude_4()
+                else BetaToolTextEditor20241022Param(
                     type="text_editor_20241022", name="str_replace_editor"
                 )
                 if self.is_claude_3_5()
@@ -706,6 +724,7 @@ ToolParamDef = (
     | BetaToolComputerUse20250124Param
     | ToolTextEditor20250124Param
     | BetaToolTextEditor20241022Param
+    | BetaToolTextEditor20250429Param
     | WebSearchTool20250305Param
 )
 
@@ -716,6 +735,7 @@ def add_cache_control(
     | BetaToolComputerUse20250124Param
     | ToolTextEditor20250124Param
     | BetaToolTextEditor20241022Param
+    | BetaToolTextEditor20250429Param
     | WebSearchTool20250305Param
     | dict[str, Any],
 ) -> None:
@@ -1008,6 +1028,7 @@ def _names_for_tool_call(
         (INTERNAL_COMPUTER_TOOL_NAME, "computer_20250124", "computer"),
         ("str_replace_editor", "text_editor_20241022", "text_editor"),
         ("str_replace_editor", "text_editor_20250124", "text_editor"),
+        ("str_replace_based_edit_tool", "text_editor_20250429", "text_editor"),
         ("bash", "bash_20250124", "bash_session"),
     )
 

inspect_ai/model/_providers/google.py

@@ -991,6 +991,10 @@ def _combine_text_parts(acc: list[Part], part: Part) -> list[Part]:
     """Combine adjacent text parts into a single part."""
     return (
         acc + [part]
-        if part.text is None or len(acc) == 0 or acc[-1].text is None
+        if part.text is None
+        or part.thought is True
+        or len(acc) == 0
+        or acc[-1].text is None
+        or acc[-1].thought is True
         else acc[:-1] + [Part(text=acc[-1].text + part.text)]
     )

inspect_ai/model/_providers/groq.py

@@ -156,6 +156,11 @@ class GroqAPI(ModelAPI):
                     "completion_time": completion.usage.completion_time,
                     "total_time": completion.usage.total_time,
                 }
+            if completion.choices[0].message.executed_tools:
+                metadata["executed_tools"] = [
+                    tool.model_dump()
+                    for tool in completion.choices[0].message.executed_tools
+                ]
 
             # extract output
             choices = self._chat_choices_from_response(completion, tools)

inspect_ai/model/_providers/perplexity.py

@@ -49,7 +49,33 @@ class PerplexityAPI(OpenAICompatibleAPI):
         tool_choice: "ToolChoice",
         config: GenerateConfig,
     ) -> tuple[ModelOutput | Exception, "ModelCall"]:
-        result = await super().generate(input, tools, tool_choice, config)
+        search_options: dict[str, Any] | None = None
+        for tool in tools:
+            if (
+                tool.name == "web_search"
+                and tool.options
+                and "perplexity" in tool.options
+            ):
+                maybe_opts = tool.options["perplexity"]
+                if maybe_opts is not None:
+                    if maybe_opts is True:
+                        search_options = {}
+                    elif isinstance(maybe_opts, dict):
+                        search_options = maybe_opts
+                    else:
+                        raise TypeError(
+                            f"Expected a dictionary or True for perplexity_options, got {type(maybe_opts)}"
+                        )
+            else:
+                raise ValueError(
+                    "Perplexity does not support tools other than web_search with perplexity options"
+                )
+
+        if search_options:
+            extra_body = {**(config.extra_body or {}), **search_options}
+            config = config.merge(GenerateConfig(extra_body=extra_body))
+
+        result = await super().generate(input, [], tool_choice, config)
         output, call = cast(tuple[ModelOutput, "ModelCall"], result)
 
         if self._response:

inspect_ai/model/_providers/providers.py

@@ -16,7 +16,7 @@ from .._registry import modelapi
 def groq() -> type[ModelAPI]:
     FEATURE = "Groq API"
     PACKAGE = "groq"
-    MIN_VERSION = "0.16.0"
+    MIN_VERSION = "0.28.0"
 
     # verify we have the package
     try:

inspect_ai/tool/_tools/_web_search/_web_search.py

@@ -18,7 +18,7 @@ from ._tavily import TavilyOptions, tavily_search_provider
 from ._web_search_provider import SearchProvider
 
 Provider: TypeAlias = Literal[
-    "gemini", "openai", "anthropic", "tavily", "google", "exa"
+    "gemini", "openai", "anthropic", "perplexity", "tavily", "google", "exa"
 ]
 valid_providers = set(get_args(Provider))
 
@@ -35,6 +35,7 @@ class Providers(TypedDict, total=False):
     openai: dict[str, Any] | Literal[True]
     anthropic: dict[str, Any] | Literal[True]
     gemini: dict[str, Any] | Literal[True]
+    perplexity: dict[str, Any] | Literal[True]
     tavily: dict[str, Any] | Literal[True]
     google: dict[str, Any] | Literal[True]
     exa: dict[str, Any] | Literal[True]
@@ -44,6 +45,7 @@ class _NormalizedProviders(TypedDict, total=False):
     openai: dict[str, Any]
     anthropic: dict[str, Any]
     gemini: dict[str, Any]
+    perplexity: dict[str, Any]
     tavily: dict[str, Any]
     google: dict[str, Any]
     exa: dict[str, Any]
@@ -67,7 +69,7 @@ def web_search(
     Web searches are executed using a provider. Providers are split
     into two categories:
 
-    - Internal providers: "openai", "anthropic" - these use the model's built-in
+    - Internal providers: "openai", "anthropic", "gemini", "perplexity" - these use the model's built-in
       search capability and do not require separate API keys. These work only for
       their respective model provider (e.g. the "openai" search provider
       works only for `openai/*` models).
@@ -84,7 +86,7 @@ def web_search(
 
     Args:
       providers: Configuration for the search providers to use. Currently supported
-        providers are "openai", "anthropic", "tavily", "google", and "exa". The
+        providers are "openai", "anthropic", "perplexity", "tavily", "google", and "exa". The
         `providers` parameter supports several formats based on either a `str`
         specifying a provider or a `dict` whose keys are the provider names and
         whose values are the provider-specific options. A single value or a list
@@ -121,6 +123,9 @@ def web_search(
         - anthropic: Supports Anthropic's web search parameters.
           See https://docs.anthropic.com/en/docs/agents-and-tools/tool-use/web-search-tool#tool-definition
 
+        - perplexity: Supports Perplexity's web search parameters.
+          See https://docs.perplexity.ai/api-reference/chat-completions-post
+
         - tavily: Supports options like `max_results`, `search_depth`, etc.
           See https://docs.tavily.com/documentation/api-reference/endpoint/search
 

inspect_ai/util/__init__.py

@@ -4,13 +4,16 @@ from inspect_ai.util._limit import (
     Limit,
     LimitExceededError,
     LimitScope,
+    SampleLimits,
     apply_limits,
     message_limit,
+    sample_limits,
     time_limit,
     token_limit,
     working_limit,
 )
 
+from ._background import background
 from ._collect import collect
 from ._concurrency import concurrency
 from ._console import input_screen
@@ -29,6 +32,7 @@ from ._sandbox import (
     SandboxEnvironmentType,
     sandbox,
     sandbox_default,
+    sandbox_service,
     sandbox_with,
     sandboxenv,
 )
@@ -44,6 +48,8 @@ from ._throttle import throttle
 
 __all__ = [
     "apply_limits",
+    "sample_limits",
+    "SampleLimits",
     "ExecResult",
     "concurrency",
     "DisplayType",
@@ -73,6 +79,7 @@ __all__ = [
     "sandbox",
     "sandbox_with",
     "sandbox_default",
+    "sandbox_service",
     "Store",
     "store",
     "StoreModel",
@@ -82,6 +89,7 @@ __all__ = [
     "Subtask",
     "subtask",
     "throttle",
+    "background",
     "token_limit",
     "time_limit",
     "working_limit",

inspect_ai/util/_background.py

@@ -0,0 +1,64 @@
+import sys
+from logging import getLogger
+from typing import Any, Awaitable, Callable
+
+if sys.version_info >= (3, 11):
+    from typing import TypeVarTuple
+else:
+    from typing_extensions import TypeVarTuple
+
+
+from typing_extensions import Unpack
+
+logger = getLogger(__name__)
+
+
+PosArgsT = TypeVarTuple("PosArgsT")
+
+
+def background(
+    func: Callable[[Unpack[PosArgsT]], Awaitable[Any]],
+    *args: Unpack[PosArgsT],
+) -> None:
+    """Run an async function in the background of the current sample.
+
+    Background functions must be run from an executing sample.
+    The function will run as long as the current sample is running.
+
+    When the sample terminates, an anyio cancelled error will be
+    raised in the background function. To catch this error and
+    cleanup:
+
+    ```python
+    import anyio
+
+    async def run():
+        try:
+            # background code
+        except anyio.get_cancelled_exc_class():
+            ...
+    ```
+
+    Args:
+       func: Async function to run
+       *args: Optional function arguments.
+    """
+    from inspect_ai.log._samples import sample_active
+
+    # get the active sample
+    sample = sample_active()
+    if sample is None:
+        raise RuntimeError(
+            "background() function must be called from a running sample."
+        )
+
+    # handle and log background exceptions
+    async def run() -> None:
+        try:
+            await func(*args)
+        except Exception as ex:
+            logger.error(f"Background worker error: {ex}")
+            raise
+
+    # kick it off
+    sample.tg.start_soon(run)