inspect-ai 0.3.57__py3-none-any.whl → 0.3.58__py3-none-any.whl

inspect_ai/approval/_human/util.py

@@ -5,7 +5,7 @@ from rich.text import Text
 
 from inspect_ai._util.transcript import transcript_markdown
 from inspect_ai.tool._tool_call import ToolCallContent, ToolCallView
-from inspect_ai.util._trace import trace_enabled
+from inspect_ai.util._display import display_type
 
 HUMAN_APPROVED = "Human operator approved tool call."
 HUMAN_REJECTED = "Human operator rejected the tool call."
@@ -18,7 +18,7 @@ def render_tool_approval(message: str, view: ToolCallView) -> list[RenderableTyp
     text_highlighter = ReprHighlighter()
 
     # ignore content if trace enabled
-    message = message.strip() if not trace_enabled() else ""
+    message = message.strip() if display_type() != "conversation" else ""
 
     def add_view_content(view_content: ToolCallContent) -> None:
         if view_content.title:

inspect_ai/dataset/_sources/csv.py

@@ -1,4 +1,5 @@
 import csv
+import os
 from io import TextIOWrapper
 from pathlib import Path
 from typing import Any
@@ -75,7 +76,7 @@ def csv_dataset(
         dataset = MemoryDataset(
             samples=data_to_samples(valid_data, data_to_sample, auto_id),
             name=name,
-            location=csv_file,
+            location=os.path.abspath(csv_file),
         )
 
         # resolve relative file paths

inspect_ai/dataset/_sources/json.py

@@ -1,4 +1,5 @@
 import json
+import os
 from io import TextIOWrapper
 from pathlib import Path
 from typing import Any, cast
@@ -75,7 +76,7 @@ def json_dataset(
         dataset = MemoryDataset(
             samples=data_to_samples(dataset_reader(f), data_to_sample, auto_id),
             name=name,
-            location=json_file,
+            location=os.path.abspath(json_file),
         )
 
         # resolve relative file paths

inspect_ai/dataset/_sources/util.py

@@ -1,6 +1,6 @@
 from typing import Callable
 
-from inspect_ai._util.content import Content, ContentImage
+from inspect_ai._util.content import Content, ContentAudio, ContentImage, ContentVideo
 from inspect_ai._util.file import filesystem
 from inspect_ai.model._chat_message import ChatMessage, ChatMessageUser
 from inspect_ai.util._sandbox.environment import SandboxEnvironmentSpec
@@ -44,24 +44,28 @@ def resolve_sample_files(dataset: Dataset) -> None:
             for path in sample.files.keys():
                 sample.files[path] = resolve_file(sample.files[path])
 
+        # check for setup script
+        if sample.setup is not None:
+            sample.setup = resolve_file(sample.setup)
+
         # check for image paths
         if not isinstance(sample.input, str):
-            sample.input = messages_with_resolved_images(sample.input, resolve_file)
+            sample.input = messages_with_resolved_content(sample.input, resolve_file)
 
 
-def messages_with_resolved_images(
+def messages_with_resolved_content(
     messages: list[ChatMessage], resolver: Callable[[str], str]
 ) -> list[ChatMessage]:
-    return [message_with_resolved_image(message, resolver) for message in messages]
+    return [message_with_resolved_content(message, resolver) for message in messages]
 
 
-def message_with_resolved_image(
+def message_with_resolved_content(
     message: ChatMessage, resolver: Callable[[str], str]
 ) -> ChatMessage:
     if isinstance(message, ChatMessageUser) and not isinstance(message.content, str):
         return ChatMessageUser(
             content=[
-                chat_content_with_resolved_image(content, resolver)
+                chat_content_with_resolved_content(content, resolver)
                 for content in message.content
             ],
             source=message.source,
@@ -70,7 +74,7 @@ def message_with_resolved_image(
         return message
 
 
-def chat_content_with_resolved_image(
+def chat_content_with_resolved_content(
     content: Content, resolver: Callable[[str], str]
 ) -> Content:
     if isinstance(content, ContentImage):
@@ -78,5 +82,9 @@ def chat_content_with_resolved_image(
             image=resolver(content.image),
             detail=content.detail,
         )
+    elif isinstance(content, ContentAudio):
+        return ContentAudio(audio=resolver(content.audio), format=content.format)
+    elif isinstance(content, ContentVideo):
+        return ContentVideo(video=resolver(content.video), format=content.format)
     else:
         return content

inspect_ai/log/_condense.py

@@ -6,7 +6,13 @@ from typing import (
 from pydantic import JsonValue
 
 from inspect_ai._util.constants import BASE_64_DATA_REMOVED
-from inspect_ai._util.content import Content, ContentImage, ContentText
+from inspect_ai._util.content import (
+    Content,
+    ContentAudio,
+    ContentImage,
+    ContentText,
+    ContentVideo,
+)
 from inspect_ai._util.hash import mm3_hash
 from inspect_ai._util.json import JsonChange
 from inspect_ai._util.url import is_data_uri
@@ -304,3 +310,7 @@ def walk_content(content: Content, content_fn: Callable[[str], str]) -> Content:
         return content.model_copy(update=dict(text=content_fn(content.text)))
     elif isinstance(content, ContentImage):
         return content.model_copy(update=dict(image=content_fn(content.image)))
+    elif isinstance(content, ContentAudio):
+        return content.model_copy(update=dict(audio=content_fn(content.audio)))
+    elif isinstance(content, ContentVideo):
+        return content.model_copy(update=dict(video=content_fn(content.video)))

inspect_ai/log/_log.py

@@ -48,9 +48,6 @@ class EvalConfig(BaseModel):
     epochs_reducer: list[str] | None = Field(default=None)
     """Reducers for aggregating per-sample scores."""
 
-    trace: bool | None = Field(default=None)
-    """Trace message interactions with evaluated model to terminal."""
-
     approval: ApprovalPolicyConfig | None = Field(default=None)
     """Approval policy for tool use."""
 
@@ -355,7 +352,7 @@ class EvalResults(BaseModel):
         """Scorer used to compute results (deprecated)."""
         warn_once(
             logger,
-            "The 'scorer' field is deprecated. Use 'scorers' instead.",
+            "The 'scorer' field is deprecated. Use 'scores' instead.",
         )
         return self.scores[0] if self.scores else None
 
@@ -364,7 +361,7 @@ class EvalResults(BaseModel):
         """Metrics computed (deprecated)."""
         warn_once(
             logger,
-            "The 'metrics' field is deprecated. Access metrics through 'scorers' instead.",
+            "The 'metrics' field is deprecated. Access metrics through 'scores' instead.",
         )
         return self.scores[0].metrics if self.scores else {}
 

inspect_ai/log/_recorders/eval.py

@@ -13,7 +13,12 @@ from pydantic_core import to_json
 from typing_extensions import override
 
 from inspect_ai._util.constants import LOG_SCHEMA_VERSION
-from inspect_ai._util.content import ContentImage, ContentText
+from inspect_ai._util.content import (
+    ContentAudio,
+    ContentImage,
+    ContentText,
+    ContentVideo,
+)
 from inspect_ai._util.error import EvalError
 from inspect_ai._util.file import FileSystem, async_fileystem, dirname, file, filesystem
 from inspect_ai._util.json import jsonable_python
@@ -90,9 +95,11 @@ class EvalRecorder(FileRecorder):
         self.data: dict[str, ZipLogFile] = {}
 
     @override
-    async def log_init(self, eval: EvalSpec, location: str | None = None) -> str:
+    async def log_init(
+        self, eval: EvalSpec, location: str | None = None, *, clean: bool = False
+    ) -> str:
         # if the file exists then read summaries
-        if location is not None and self.fs.exists(location):
+        if not clean and location is not None and self.fs.exists(location):
             with file(location, "rb") as f:
                 with ZipFile(f, "r") as zip:
                     log_start = _read_start(zip)
@@ -229,7 +236,7 @@ class EvalRecorder(FileRecorder):
     async def write_log(cls, location: str, log: EvalLog) -> None:
         # write using the recorder (so we get all of the extra streams)
         recorder = EvalRecorder(dirname(location))
-        await recorder.log_init(log.eval, location)
+        await recorder.log_init(log.eval, location, clean=True)
         await recorder.log_start(log.eval, log.plan)
         for sample in log.samples or []:
             await recorder.log_sample(log.eval, sample)
@@ -244,12 +251,16 @@ def text_inputs(inputs: str | list[ChatMessage]) -> str | list[ChatMessage]:
         input: list[ChatMessage] = []
         for message in inputs:
             if not isinstance(message.content, str):
-                filtered_content: list[ContentText | ContentImage] = []
+                filtered_content: list[
+                    ContentText | ContentImage | ContentAudio | ContentVideo
+                ] = []
                 for content in message.content:
-                    if content.type != "image":
+                    if content.type == "text":
                         filtered_content.append(content)
-                if len(filtered_content) == 0:
-                    filtered_content.append(ContentText(text="(Image)"))
+                    else:
+                        filtered_content.append(
+                            ContentText(text=f"({content.type.capitalize()})")
+                        )
                 message.content = filtered_content
                 input.append(message)
             else:

inspect_ai/log/_samples.py

@@ -29,7 +29,7 @@ class ActiveSample:
         sandboxes: dict[str, SandboxConnection],
     ) -> None:
         self.id = uuid()
-        self.started = datetime.now().timestamp()
+        self.started: float | None = None
         self.completed: float | None = None
         self.task = task
         self.model = model
@@ -48,10 +48,15 @@ class ActiveSample:
 
     @property
     def execution_time(self) -> float:
-        completed = (
-            self.completed if self.completed is not None else datetime.now().timestamp()
-        )
-        return completed - self.started
+        if self.started is not None:
+            completed = (
+                self.completed
+                if self.completed is not None
+                else datetime.now().timestamp()
+            )
+            return completed - self.started
+        else:
+            return 0
 
     def interrupt(self, action: Literal["score", "error"]) -> None:
         self._interrupt_action = action

inspect_ai/log/_transcript.py

@@ -1,3 +1,4 @@
+import asyncio
 import contextlib
 from contextvars import ContextVar
 from datetime import datetime
@@ -11,7 +12,7 @@ from typing import (
     Union,
 )
 
-from pydantic import BaseModel, Field, JsonValue, field_serializer
+from pydantic import BaseModel, ConfigDict, Field, JsonValue, field_serializer
 
 from inspect_ai._util.constants import SAMPLE_SUBTASK
 from inspect_ai._util.error import EvalError
@@ -176,6 +177,32 @@ class ToolEvent(BaseEvent):
         self.events = events
         self.pending = None
 
+    # mechanism for operator to cancel the tool call
+
+    def set_task(self, task: asyncio.Task[Any]) -> None:
+        """Set the tool task (for possible cancellation)"""
+        self._task = task
+
+    def cancel(self) -> None:
+        """Cancel the tool task."""
+        if self._task:
+            self._cancelled = True
+            self._task.cancel()
+
+    @property
+    def cancelled(self) -> bool:
+        """Was the task cancelled?"""
+        return self._cancelled is True
+
+    _cancelled: bool | None = None
+    """Was this tool call cancelled?"""
+
+    _task: asyncio.Task[Any] | None = None
+    """Handle to task (used for cancellation)"""
+
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+    """Required so that we can include '_task' as a member."""
+
 
 class ApprovalEvent(BaseEvent):
     """Tool approval."""

inspect_ai/model/__init__.py

@@ -1,6 +1,12 @@
 # ruff: noqa: F401 F403 F405
 
-from inspect_ai._util.content import Content, ContentImage, ContentText
+from inspect_ai._util.content import (
+    Content,
+    ContentAudio,
+    ContentImage,
+    ContentText,
+    ContentVideo,
+)
 from inspect_ai._util.deprecation import relocated_module_attribute
 
 from ._cache import (
@@ -42,8 +48,10 @@ __all__ = [
     "GenerateConfig",
     "GenerateConfigArgs",
     "CachePolicy",
-    "ContentText",
+    "ContentAudio",
     "ContentImage",
+    "ContentText",
+    "ContentVideo",
     "Content",
     "ChatMessage",
     "ChatMessageSystem",

inspect_ai/model/_call_tools.py

@@ -24,11 +24,17 @@ from typing import (
 from jsonschema import Draft7Validator
 from pydantic import BaseModel
 
-from inspect_ai._util.content import Content, ContentImage, ContentText
+from inspect_ai._util.content import (
+    Content,
+    ContentAudio,
+    ContentImage,
+    ContentText,
+    ContentVideo,
+)
 from inspect_ai._util.format import format_function_call
 from inspect_ai._util.text import truncate_string_to_bytes
 from inspect_ai._util.trace import trace_action
-from inspect_ai.model._trace import trace_tool_mesage
+from inspect_ai.model._conversation import conversation_tool_mesage
 from inspect_ai.tool import Tool, ToolCall, ToolError, ToolInfo
 from inspect_ai.tool._tool import ToolApprovalError, ToolParsingError
 from inspect_ai.tool._tool_call import ToolCallContent, ToolCallError
@@ -120,10 +126,14 @@ async def call_tools(
             # massage result, leave list[Content] alone, convert all other
             # types to string as that is what the model APIs accept
             truncated: tuple[int, int] | None = None
-            if isinstance(result, ContentText | ContentImage):
+            if isinstance(
+                result, ContentText | ContentImage | ContentAudio | ContentVideo
+            ):
                 content: str | list[Content] = [result]
             elif isinstance(result, list) and (
-                isinstance(result[0], ContentText | ContentImage)
+                isinstance(
+                    result[0], ContentText | ContentImage | ContentAudio | ContentVideo
+                )
             ):
                 content = result
             else:
@@ -163,6 +173,9 @@ async def call_tools(
         # call tools
         tool_messages: list[ChatMessageTool] = []
         for call in message.tool_calls:
+            # create the task
+            task = asyncio.create_task(call_tool_task(call))
+
             # create pending tool event and add it to the transcript
             event = ToolEvent(
                 id=call.id,
@@ -171,15 +184,44 @@ async def call_tools(
                 view=call.view,
                 pending=True,
             )
+            event.set_task(task)
             transcript()._event(event)
 
-            # execute the tool call
-            task = asyncio.create_task(call_tool_task(call))
-            tool_message, result_event = await task
+            # execute the tool call. if the operator cancelled the
+            # tool call then synthesize the appropriate message/event
+            try:
+                tool_message, result_event = await task
+            except asyncio.CancelledError:
+                if event.cancelled:
+                    tool_message = ChatMessageTool(
+                        content="",
+                        function=call.function,
+                        tool_call_id=call.id,
+                        error=ToolCallError(
+                            "timeout", "Command timed out before completing."
+                        ),
+                    )
+                    result_event = ToolEvent(
+                        id=call.id,
+                        function=call.function,
+                        arguments=call.arguments,
+                        result=tool_message.content,
+                        truncated=None,
+                        view=call.view,
+                        error=tool_message.error,
+                        events=[],
+                    )
+                    transcript().info(
+                        f"Tool call '{call.function}' was cancelled by operator."
+                    )
+                else:
+                    raise
+
+            # update return messages
             tool_messages.append(tool_message)
 
-            # trace if we are tracing
-            trace_tool_mesage(tool_message)
+            # print conversation if display is conversation
+            conversation_tool_mesage(tool_message)
 
             # update the event with the results
             event.set_result(
@@ -411,12 +453,13 @@ def truncate_tool_output(
     # truncate if required
     truncated = truncate_string_to_bytes(output, active_max_output)
     if truncated:
-        truncated_output = dedent(f"""
+        truncated_output = dedent("""
             The output of your call to {tool_name} was too long to be displayed.
             Here is a truncated version:
             <START_TOOL_OUTPUT>
-            {truncated.output}
-            <END_TOOL_OUTPUT>""")
+            {truncated_output}
+            <END_TOOL_OUTPUT>
+            """).format(tool_name=tool_name, truncated_output=truncated.output)
         return TruncatedToolOutput(
             truncated_output, truncated.original_bytes, active_max_output
         )

inspect_ai/model/_chat_message.py

@@ -59,10 +59,8 @@ class ChatMessageBase(BaseModel):
         if isinstance(self.content, str):
             self.content = text
         else:
-            all_images = [
-                content for content in self.content if content.type == "image"
-            ]
-            self.content = [ContentText(text=text)] + all_images
+            all_other = [content for content in self.content if content.type != "text"]
+            self.content = [ContentText(text=text)] + all_other
 
 
 class ChatMessageSystem(ChatMessageBase):

inspect_ai/model/{_trace.py → _conversation.py}

@@ -3,7 +3,8 @@ from rich.text import Text
 
 from inspect_ai._util.rich import lines_display
 from inspect_ai._util.transcript import transcript_markdown
-from inspect_ai.util._trace import trace_enabled, trace_panel
+from inspect_ai.util._conversation import conversation_panel
+from inspect_ai.util._display import display_type
 
 from ._chat_message import ChatMessage, ChatMessageAssistant, ChatMessageTool
 from ._render import messages_preceding_assistant, render_tool_calls
@@ -11,25 +12,25 @@ from ._render import messages_preceding_assistant, render_tool_calls
 MESSAGE_TITLE = "Message"
 
 
-def trace_tool_mesage(message: ChatMessageTool) -> None:
-    if trace_enabled():
+def conversation_tool_mesage(message: ChatMessageTool) -> None:
+    if display_type() == "conversation":
         # truncate output to 100 lines
         output = message.error.message if message.error else message.text.strip()
         content = lines_display(output, 100)
 
-        trace_panel(
+        conversation_panel(
             title=f"Tool Output: {message.function}",
             content=content,
         )
 
 
-def trace_assistant_message(
+def conversation_assistant_message(
     input: list[ChatMessage], message: ChatMessageAssistant
 ) -> None:
-    if trace_enabled():
+    if display_type() == "conversation":
         # print precding messages that aren't tool or assistant
         for m in messages_preceding_assistant(input):
-            trace_panel(
+            conversation_panel(
                 title=m.role.capitalize(),
                 content=transcript_markdown(m.text, escape=True),
             )
@@ -45,4 +46,4 @@ def trace_assistant_message(
             content.extend(render_tool_calls(message.tool_calls))
 
         # print the assistant message
-        trace_panel(title="Assistant", content=content)
+        conversation_panel(title="Assistant", content=content)

inspect_ai/model/_model.py

@@ -43,6 +43,7 @@ from ._chat_message import (
     ChatMessageTool,
     ChatMessageUser,
 )
+from ._conversation import conversation_assistant_message
 from ._generate_config import (
     GenerateConfig,
     active_generate_config,
@@ -50,7 +51,6 @@ from ._generate_config import (
 )
 from ._model_call import ModelCall
 from ._model_output import ModelOutput, ModelUsage
-from ._trace import trace_assistant_message
 
 logger = logging.getLogger(__name__)
 
@@ -487,7 +487,7 @@ class Model:
             updated_output: ModelOutput, updated_call: ModelCall | None
         ) -> None:
             # trace
-            trace_assistant_message(input, updated_output.choices[0].message)
+            conversation_assistant_message(input, updated_output.choices[0].message)
 
             # update event
             event.output = updated_output

inspect_ai/model/_providers/anthropic.py

@@ -28,11 +28,11 @@ from pydantic import JsonValue
 from typing_extensions import override
 
 from inspect_ai._util.constants import BASE_64_DATA_REMOVED, DEFAULT_MAX_RETRIES
-from inspect_ai._util.content import Content, ContentText
+from inspect_ai._util.content import Content, ContentImage, ContentText
 from inspect_ai._util.error import exception_message
-from inspect_ai._util.images import image_as_data_uri
+from inspect_ai._util.images import file_as_data_uri
 from inspect_ai._util.logger import warn_once
-from inspect_ai._util.url import data_uri_mime_type, data_uri_to_base64, is_data_uri
+from inspect_ai._util.url import data_uri_mime_type, data_uri_to_base64
 from inspect_ai.tool import ToolCall, ToolChoice, ToolFunction, ToolInfo
 
 from .._chat_message import (
@@ -584,11 +584,9 @@ async def message_param_content(
 ) -> TextBlockParam | ImageBlockParam:
     if isinstance(content, ContentText):
         return TextBlockParam(type="text", text=content.text or NO_CONTENT)
-    else:
+    elif isinstance(content, ContentImage):
         # resolve to url
-        image = content.image
-        if not is_data_uri(image):
-            image = await image_as_data_uri(image)
+        image = await file_as_data_uri(content.image)
 
         # resolve mime type and base64 content
         media_type = data_uri_mime_type(image) or "image/png"
@@ -601,6 +599,10 @@ async def message_param_content(
             type="image",
             source=dict(type="base64", media_type=cast(Any, media_type), data=image),
         )
+    else:
+        raise RuntimeError(
+            "Anthropic models do not currently support audio or video inputs."
+        )
 
 
 def model_call_filter(key: JsonValue | None, value: JsonValue) -> JsonValue:

inspect_ai/model/_providers/azureai.py

@@ -31,8 +31,8 @@ from azure.core.exceptions import AzureError, HttpResponseError
 from typing_extensions import override
 
 from inspect_ai._util.constants import DEFAULT_MAX_TOKENS
-from inspect_ai._util.content import Content, ContentText
-from inspect_ai._util.images import image_as_data_uri
+from inspect_ai._util.content import Content, ContentImage, ContentText
+from inspect_ai._util.images import file_as_data_uri
 from inspect_ai.tool import ToolChoice, ToolInfo
 from inspect_ai.tool._tool_call import ToolCall
 from inspect_ai.tool._tool_choice import ToolFunction
@@ -312,12 +312,14 @@ async def chat_request_message(
 async def chat_content_item(content: Content) -> ContentItem:
     if isinstance(content, ContentText):
         return TextContentItem(text=content.text)
-    else:
+    elif isinstance(content, ContentImage):
         return ImageContentItem(
             image_url=ImageUrl(
-                url=await image_as_data_uri(content.image), detail=content.detail
+                url=await file_as_data_uri(content.image), detail=content.detail
             )
         )
+    else:
+        raise RuntimeError("Azure AI models do not support audio or video inputs.")
 
 
 def chat_tool_call(tool_call: ToolCall) -> ChatCompletionsToolCall:

inspect_ai/model/_providers/bedrock.py

@@ -11,7 +11,7 @@ from inspect_ai._util.constants import (
 )
 from inspect_ai._util.content import Content, ContentImage, ContentText
 from inspect_ai._util.error import pip_dependency_error
-from inspect_ai._util.images import image_as_data
+from inspect_ai._util.images import file_as_data
 from inspect_ai._util.version import verify_required_version
 from inspect_ai.tool import ToolChoice, ToolInfo
 from inspect_ai.tool._tool_call import ToolCall
@@ -430,7 +430,9 @@ def model_output_from_response(
             content.append(ContentText(type="text", text=c.text))
         elif c.image is not None:
             base64_image = base64.b64encode(c.image.source.bytes).decode("utf-8")
-            content.append(ContentImage(image=base64_image))
+            content.append(
+                ContentImage(image=f"data:image/{c.image.format};base64,{base64_image}")
+            )
         elif c.toolUse is not None:
             tool_calls.append(
                 ToolCall(
@@ -565,7 +567,7 @@ async def converse_chat_message(
                 if c.type == "text":
                     tool_result_content.append(ConverseToolResultContent(text=c.text))
                 elif c.type == "image":
-                    image_data, image_type = await image_as_data(c.image)
+                    image_data, image_type = await file_as_data(c.image)
                     tool_result_content.append(
                         ConverseToolResultContent(
                             image=ConverseImage(
@@ -604,7 +606,7 @@ async def converse_contents(
         result: list[ConverseMessageContent] = []
         for c in content:
             if c.type == "image":
-                image_data, image_type = await image_as_data(c.image)
+                image_data, image_type = await file_as_data(c.image)
                 result.append(
                     ConverseMessageContent(
                         image=ConverseImage(