inspect-ai 0.3.55__py3-none-any.whl → 0.3.56__py3-none-any.whl

inspect_ai/_view/www/src/types/log.d.ts

@@ -29,6 +29,7 @@ export type SandboxEnvironmentSpec = [unknown] | [unknown, unknown];
 export type Model = string;
 export type ModelBaseUrl = string | null;
 export type Limit = number | [unknown, unknown] | null;
+export type SampleId = string | number | (string | number)[] | null;
 export type Epochs = number | null;
 export type EpochsReducer = string[] | null;
 export type Trace = boolean | null;
@@ -42,10 +43,12 @@ export type TimeLimit = number | null;
 export type MaxSamples = number | null;
 export type MaxTasks = number | null;
 export type MaxSubprocesses = number | null;
+export type MaxSandboxes = number | null;
 export type SandboxCleanup = boolean | null;
 export type LogSamples = boolean | null;
 export type LogImages = boolean | null;
 export type LogBuffer = number | null;
+export type ScoreDisplay = boolean | null;
 export type Type = "git";
 export type Origin = string;
 export type Commit = string;
@@ -76,6 +79,7 @@ export type TopLogprobs = number | null;
 export type ParallelToolCalls = boolean | null;
 export type MaxToolOutput = number | null;
 export type CachePrompt = "auto" | boolean | null;
+export type ReasoningEffort = ("low" | "medium" | "high") | null;
 export type TotalSamples = number;
 export type CompletedSamples = number;
 export type Name3 = string;
@@ -119,6 +123,7 @@ export type Role = "system";
 export type Content1 = string | (ContentText | ContentImage)[];
 export type Source1 = ("input" | "generate") | null;
 export type Role1 = "user";
+export type ToolCallId = string | null;
 export type Content2 = string | (ContentText | ContentImage)[];
 export type Source2 = ("input" | "generate") | null;
 export type Role2 = "assistant";
@@ -133,7 +138,7 @@ export type Content3 = string;
 export type Content4 = string | (ContentText | ContentImage)[];
 export type Source3 = ("input" | "generate") | null;
 export type Role3 = "tool";
-export type ToolCallId = string | null;
+export type ToolCallId1 = string | null;
 export type Function1 = string | null;
 export type Type4 =
   | "parsing"
@@ -241,14 +246,10 @@ export type Name5 = string;
 export type Description = string;
 export type Type6 = "object";
 export type Type7 =
-  | "string"
-  | "integer"
-  | "number"
-  | "boolean"
-  | "array"
-  | "object"
-  | "null";
+  | ("string" | "integer" | "number" | "boolean" | "array" | "object" | "null")
+  | null;
 export type Description1 = string | null;
+export type Enum = unknown[] | null;
 export type Properties1 = {
   [k: string]: ToolParam;
 } | null;
@@ -267,7 +268,13 @@ export type Event5 = "tool";
 export type Type8 = "function";
 export type Id3 = string;
 export type Function2 = string;
-export type Result = string | number | boolean | (ContentText | ContentImage)[];
+export type Result =
+  | string
+  | number
+  | boolean
+  | ContentText
+  | ContentImage
+  | (ContentText | ContentImage)[];
 export type Truncated = [unknown, unknown] | null;
 export type Timestamp6 = string;
 export type Pending6 = boolean | null;
@@ -388,7 +395,7 @@ export type Value2 =
 export type Answer1 = string | null;
 export type Explanation2 = string | null;
 export type Metadata8 = {} | null;
-export type SampleId = string | number | null;
+export type SampleId1 = string | number | null;
 export type Samples2 = SampleScore[];
 export type Location1 = string;
 
@@ -438,6 +445,7 @@ export interface EvalDataset {
 export interface ModelArgs {}
 export interface EvalConfig {
   limit: Limit;
+  sample_id: SampleId;
   epochs: Epochs;
   epochs_reducer: EpochsReducer;
   trace: Trace;
@@ -449,10 +457,12 @@ export interface EvalConfig {
   max_samples: MaxSamples;
   max_tasks: MaxTasks;
   max_subprocesses: MaxSubprocesses;
+  max_sandboxes: MaxSandboxes;
   sandbox_cleanup: SandboxCleanup;
   log_samples: LogSamples;
   log_images: LogImages;
   log_buffer: LogBuffer;
+  score_display: ScoreDisplay;
 }
 export interface ApprovalPolicyConfig {
   approvers: Approvers;
@@ -523,6 +533,7 @@ export interface GenerateConfig {
   parallel_tool_calls: ParallelToolCalls;
   max_tool_output: MaxToolOutput;
   cache_prompt: CachePrompt;
+  reasoning_effort: ReasoningEffort;
 }
 export interface EvalResults {
   total_samples: TotalSamples;
@@ -607,6 +618,7 @@ export interface ChatMessageUser {
   content: Content1;
   source: Source1;
   role: Role1;
+  tool_call_id: ToolCallId;
 }
 export interface ChatMessageAssistant {
   content: Content2;
@@ -635,7 +647,7 @@ export interface ChatMessageTool {
   content: Content4;
   source: Source3;
   role: Role3;
-  tool_call_id: ToolCallId;
+  tool_call_id: ToolCallId1;
   function: Function1;
   error: ToolCallError | null;
 }
@@ -825,6 +837,7 @@ export interface ToolParam {
   type: Type7;
   description: Description1;
   default: Default;
+  enum: Enum;
   items: ToolParam | null;
   properties: Properties1;
   additionalProperties: Additionalproperties;
@@ -862,6 +875,7 @@ export interface GenerateConfig1 {
   parallel_tool_calls: ParallelToolCalls;
   max_tool_output: MaxToolOutput;
   cache_prompt: CachePrompt;
+  reasoning_effort: ReasoningEffort;
 }
 /**
  * Model call (raw request/response data).
@@ -1031,5 +1045,5 @@ export interface SampleScore {
   answer: Answer1;
   explanation: Explanation2;
   metadata: Metadata8;
-  sample_id: SampleId;
+  sample_id: SampleId1;
 }

inspect_ai/_view/www/yarn.lock

@@ -131,6 +131,13 @@
   dependencies:
     "@babel/types" "^7.25.2"
 
+"@babel/runtime@^7.21.0":
+  version "7.26.0"
+  resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.26.0.tgz#8600c2f595f277c60815256418b85356a65173c1"
+  integrity sha512-FDSOghenHTiToteC/QRlv2q3DhPZ/oOXTBoirfWNx1Cx3TMVcGWQtMMmQcSvb/JjpNeGzx8Pq/b4fKEJuWm1sw==
+  dependencies:
+    regenerator-runtime "^0.14.0"
+
 "@babel/template@^7.25.0":
   version "7.25.0"
   resolved "https://registry.yarnpkg.com/@babel/template/-/template-7.25.0.tgz#e733dc3134b4fede528c15bc95e89cb98c52592a"
@@ -525,6 +532,14 @@ argparse@^2.0.1:
   resolved "https://registry.yarnpkg.com/argparse/-/argparse-2.0.1.tgz#246f50f3ca78a3240f6c997e8a9bd1eac49e4b38"
   integrity sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==
 
+asciinema-player@^3.8.1:
+  version "3.8.1"
+  resolved "https://registry.yarnpkg.com/asciinema-player/-/asciinema-player-3.8.1.tgz#d56ccc04a85570559900b2297cf44c2a7453d118"
+  integrity sha512-NkpbFg81Y6iJFpDRndakLCQ0G26XSpvuT3vJTFjMRgHb26lqHgRNY9gun54e5MehZ4fEDNYkMZv+z6MfZ8c2aA==
+  dependencies:
+    "@babel/runtime" "^7.21.0"
+    solid-js "^1.3.0"
+
 babel-plugin-prismjs@^2.1.0:
   version "2.1.0"
   resolved "https://registry.yarnpkg.com/babel-plugin-prismjs/-/babel-plugin-prismjs-2.1.0.tgz#ade627896106326ad04d6d77fba92877618de571"
@@ -647,6 +662,11 @@ cross-spawn@^7.0.2:
     shebang-command "^2.0.0"
     which "^2.0.1"
 
+csstype@^3.1.0:
+  version "3.1.3"
+  resolved "https://registry.yarnpkg.com/csstype/-/csstype-3.1.3.tgz#d80ff294d114fb0e6ac500fbf85b60137d7eff81"
+  integrity sha512-M1uQkMl8rQK/szD0LNhtqxIPLpimGm8sOBwU7lLnCpSbTyY3yeU1Vc7l4KT5zT4s/yOxHH5O7tIuuLOCnLADRw==
+
 cuint@^0.2.2:
   version "0.2.2"
   resolved "https://registry.yarnpkg.com/cuint/-/cuint-0.2.2.tgz#408086d409550c2631155619e9fa7bcadc3b991b"
@@ -1242,6 +1262,11 @@ queue-microtask@^1.2.2:
   resolved "https://registry.yarnpkg.com/queue-microtask/-/queue-microtask-1.2.3.tgz#4929228bbc724dfac43e0efb058caf7b6cfb6243"
   integrity sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==
 
+regenerator-runtime@^0.14.0:
+  version "0.14.1"
+  resolved "https://registry.yarnpkg.com/regenerator-runtime/-/regenerator-runtime-0.14.1.tgz#356ade10263f685dda125100cd862c1db895327f"
+  integrity sha512-dYnhHh0nJoMfnkZs6GmmhFknAGRrLznOu5nc9ML+EJxGvrx6H7teuevqVqCuPcPK//3eDrrjQhehXVx9cnkGdw==
+
 resolve-from@^4.0.0:
   version "4.0.0"
   resolved "https://registry.yarnpkg.com/resolve-from/-/resolve-from-4.0.0.tgz#4abcd852ad32dd7baabfe9b40e00a36db5f392e6"
@@ -1294,6 +1319,16 @@ semver@^6.0.0, semver@^6.3.1:
   resolved "https://registry.yarnpkg.com/semver/-/semver-6.3.1.tgz#556d2ef8689146e46dcea4bfdd095f3434dffcb4"
   integrity sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==
 
+seroval-plugins@^1.1.0:
+  version "1.1.1"
+  resolved "https://registry.yarnpkg.com/seroval-plugins/-/seroval-plugins-1.1.1.tgz#1e0c175e13bb4c620d4ce5916fbbb63de70c31f9"
+  integrity sha512-qNSy1+nUj7hsCOon7AO4wdAIo9P0jrzAMp18XhiOzA6/uO5TKtP7ScozVJ8T293oRIvi5wyCHSM4TrJo/c/GJA==
+
+seroval@^1.1.0:
+  version "1.1.1"
+  resolved "https://registry.yarnpkg.com/seroval/-/seroval-1.1.1.tgz#7630e0c17a3efa6be43f17ad6bcf9f966a61b443"
+  integrity sha512-rqEO6FZk8mv7Hyv4UCj3FD3b6Waqft605TLfsCe/BiaylRpyyMC0b+uA5TJKawX3KzMrdi3wsLbCaLplrQmBvQ==
+
 shebang-command@^2.0.0:
   version "2.0.0"
   resolved "https://registry.yarnpkg.com/shebang-command/-/shebang-command-2.0.0.tgz#ccd0af4f8835fbdc265b82461aaf0c36663f34ea"
@@ -1306,6 +1341,15 @@ shebang-regex@^3.0.0:
   resolved "https://registry.yarnpkg.com/shebang-regex/-/shebang-regex-3.0.0.tgz#ae16f1644d873ecad843b0307b143362d4c42172"
   integrity sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==
 
+solid-js@^1.3.0:
+  version "1.9.3"
+  resolved "https://registry.yarnpkg.com/solid-js/-/solid-js-1.9.3.tgz#078f026fe32f6b9b48e8e0557be150f0c2d610a9"
+  integrity sha512-5ba3taPoZGt9GY3YlsCB24kCg0Lv/rie/HTD4kG6h4daZZz7+yK02xn8Vx8dLYBc9i6Ps5JwAbEiqjmKaLB3Ag==
+  dependencies:
+    csstype "^3.1.0"
+    seroval "^1.1.0"
+    seroval-plugins "^1.1.0"
+
 source-map-js@^1.2.0:
   version "1.2.0"
   resolved "https://registry.yarnpkg.com/source-map-js/-/source-map-js-1.2.0.tgz#16b809c162517b5b8c3e7dcd315a2a5c2612b2af"

inspect_ai/approval/_apply.py

@@ -75,4 +75,8 @@ def init_tool_approval(approval: list[ApprovalPolicy] | None) -> None:
         _tool_approver.set(None)
 
 
+def have_tool_approval() -> bool:
+    return _tool_approver.get(None) is not None
+
+
 _tool_approver: ContextVar[Approver | None] = ContextVar("tool_approver", default=None)

inspect_ai/approval/_human/panel.py

@@ -24,8 +24,6 @@ from .util import (
     render_tool_approval,
 )
 
-PANEL_TITLE = "Approvals"
-
 
 async def panel_approval(
     message: str,
@@ -35,7 +33,7 @@ async def panel_approval(
     choices: list[ApprovalDecision],
 ) -> Approval:
     # ensure the approvals panel is shown
-    await input_panel(PANEL_TITLE, ApprovalInputPanel)
+    await input_panel(ApprovalInputPanel)
 
     # submit to human approval manager (will be picked up by panel)
     approvals = human_approval_manager()
@@ -52,11 +50,10 @@ async def panel_approval(
 
 
 class ApprovalInputPanel(InputPanel):
+    DEFAULT_TITLE = "Approval"
+
     DEFAULT_CSS = """
     ApprovalInputPanel {
-        width: 1fr;
-        height: 1fr;
-        padding: 0 1 1 1;
         layout: grid;
         grid-size: 1 3;
         grid-rows: auto 1fr auto;
@@ -88,7 +85,7 @@ class ApprovalInputPanel(InputPanel):
         self._approvals = human_approval_manager().approval_requests()
         if len(self._approvals) > 0:
             approval_id, approval_request = self._approvals[0]
-            self.title = f"{PANEL_TITLE} ({len(self._approvals):,})"
+            self.title = f"{self.DEFAULT_TITLE} ({len(self._approvals):,})"
             heading.request = approval_request
             content.approval = approval_request.request
             actions.approval_request = approval_id, approval_request
@@ -97,7 +94,7 @@ class ApprovalInputPanel(InputPanel):
                 actions.activate()
             self.visible = True
         else:
-            self.title = PANEL_TITLE
+            self.title = self.DEFAULT_TITLE
             heading.request = None
             content.approval = None
             actions.approval_request = None

inspect_ai/dataset/_dataset.py

@@ -1,16 +1,19 @@
 import abc
 import random
+from dataclasses import dataclass, field
 from typing import (
     TYPE_CHECKING,
     Any,
     Callable,
     Iterator,
     Sequence,
+    Type,
+    TypeVar,
     Union,
     overload,
 )
 
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, Field, ValidationError
 from typing_extensions import override
 
 from inspect_ai.model import ChatMessage
@@ -20,6 +23,8 @@ from inspect_ai.util._sandbox.environment import resolve_sandbox_environment
 if TYPE_CHECKING:
     from _typeshed import SupportsRichComparison
 
+MT = TypeVar("MT", bound=BaseModel)
+
 
 class Sample(BaseModel):
     def __init__(
@@ -76,6 +81,20 @@ class Sample(BaseModel):
     metadata: dict[str, Any] | None = Field(default=None)
     """Arbitrary metadata associated with the sample."""
 
+    def metadata_as(self, metadata_cls: Type[MT]) -> MT:
+        """Metadata as a Pydantic model.
+
+        Args:
+           metadata_cls: BaseModel derived class.
+
+        Returns:
+           BaseModel: Instance of metadata_cls.
+        """
+        if self.metadata is None:
+            raise ValueError("Sample does not have metadata")
+
+        return metadata_as(self.metadata, metadata_cls)
+
     sandbox: SandboxEnvironmentSpec | None = Field(default=None)
     """Sandbox environment type and optional config file."""
 
@@ -177,7 +196,8 @@ class Dataset(Sequence[Sample], abc.ABC):
         """
 
 
-class FieldSpec(BaseModel):
+@dataclass
+class FieldSpec:
     r"""Specification for mapping data source fields to sample fields.
 
     Args:
@@ -191,28 +211,28 @@ class FieldSpec(BaseModel):
         setup (str): Optional. Setup script to run for sample .
     """
 
-    input: str = Field(default="input")
+    input: str = field(default="input")
     """Name of the field containing the sample input."""
 
-    target: str = Field(default="target")
+    target: str = field(default="target")
     """Name of the field containing the sample target."""
 
-    choices: str = Field(default="choices")
+    choices: str = field(default="choices")
     """Name of field containing the list of answer choices."""
 
-    id: str = Field(default="id")
+    id: str = field(default="id")
     """ Unique identifier for the sample."""
 
-    metadata: list[str] | None = Field(default=None)
+    metadata: list[str] | Type[BaseModel] | None = field(default=None)
     """List of additional field names that should be read as metadata."""
 
-    sandbox: str = Field(default="sandbox")
+    sandbox: str = field(default="sandbox")
     """Sandbox type along with optional config file."""
 
-    files: str = Field(default="files")
+    files: str = field(default="files")
     """Files that go along wtih the sample."""
 
-    setup: str = Field(default="setup")
+    setup: str = field(default="setup")
     """Setup script to run for sample (run within default SandboxEnvironment)."""
 
 
@@ -313,3 +333,24 @@ class MemoryDataset(Dataset):
             samples=[sample for sample in self if predicate(sample)],
             shuffled=self.shuffled,
         )
+
+
+def metadata_as(metadata: dict[str, Any], metadata_cls: Type[MT]) -> MT:
+    # validate that metadata_cls is frozen
+    if not metadata_cls.model_config.get("frozen", False):
+        raise ValueError(
+            f"Metadata model {metadata_cls.__name__} must have frozen=True"
+        )
+
+    # filter to only fields in the model
+    model_fields = {
+        k: v
+        for k, v in metadata.items()
+        if k in metadata_cls.__pydantic_fields__.keys()
+    }
+
+    # parse and return model instance
+    try:
+        return metadata_cls(**model_fields)
+    except ValidationError as ex:
+        raise ValueError(f"Could not parse metadata into {metadata_cls.__name__}: {ex}")

inspect_ai/dataset/_util.py

@@ -1,6 +1,8 @@
 import json
 from typing import Any, Iterable, cast
 
+from pydantic import ValidationError
+
 from inspect_ai.model import (
     ChatMessage,
     ChatMessageAssistant,
@@ -33,9 +35,35 @@ def record_to_sample_fn(
             # collect metadata if specified
             metadata: dict[str, Any] | None = None
             if sample_fields.metadata:
-                metadata = {}
-                for name in sample_fields.metadata:
-                    metadata[name] = record.get(name)
+                if isinstance(sample_fields.metadata, list):
+                    metadata = {}
+                    for name in sample_fields.metadata:
+                        metadata[name] = record.get(name)
+                else:
+                    # must be frozen
+                    if not sample_fields.metadata.model_config.get("frozen", False):
+                        raise ValueError(
+                            f"Metadata model {sample_fields.metadata.__name__} must have frozen=True"
+                        )
+
+                    # filter to only fields in the model
+                    model_fields = record.get("metadata", None)
+                    if isinstance(model_fields, str):
+                        model_fields = json.loads(model_fields)
+                    elif model_fields is None:
+                        model_fields = {
+                            k: v
+                            for k, v in record.items()
+                            if k in sample_fields.metadata.__pydantic_fields__.keys()
+                        }
+
+                    # parse and return metadata
+                    try:
+                        metadata = sample_fields.metadata(**model_fields).model_dump()
+                    except ValidationError as ex:
+                        raise ValueError(
+                            f"Could not parse metadata into {sample_fields.metadata.__name__}: {ex}"
+                        )
             elif "metadata" in record:
                 metadata_field = record.get("metadata")
                 if isinstance(metadata_field, str):

inspect_ai/log/__init__.py

@@ -23,6 +23,7 @@ from ._log import (
     EvalRevision,
     EvalSample,
     EvalSampleReductions,
+    EvalSampleScore,
     EvalScore,
     EvalSpec,
     EvalStats,
@@ -60,6 +61,7 @@ __all__ = [
     "EvalResults",
     "EvalRevision",
     "EvalSample",
+    "EvalSampleScore",
     "EvalSampleReductions",
     "EvalScore",
     "EvalSpec",

inspect_ai/log/_log.py

@@ -23,7 +23,6 @@ from inspect_ai.model import (
     ModelUsage,
 )
 from inspect_ai.scorer import Score
-from inspect_ai.scorer._metric import SampleScore
 from inspect_ai.util._sandbox.environment import SandboxEnvironmentSpec
 
 from ._transcript import Event
@@ -301,6 +300,10 @@ class EvalScore(BaseModel):
     """Additional scorer metadata."""
 
 
+class EvalSampleScore(Score):
+    sample_id: str | int | None = Field(default=None)
+
+
 class EvalSampleReductions(BaseModel):
     scorer: str
     """Name the of scorer"""
@@ -308,7 +311,7 @@ class EvalSampleReductions(BaseModel):
     reducer: str | None = Field(default=None)
     """Name the of reducer"""
 
-    samples: list[SampleScore]
+    samples: list[EvalSampleScore]
     """List of reduced scores"""
 
 

inspect_ai/model/_call_tools.py

@@ -118,10 +118,12 @@ async def call_tools(
             # massage result, leave list[Content] alone, convert all other
             # types to string as that is what the model APIs accept
             truncated: tuple[int, int] | None = None
-            if isinstance(result, list) and (
+            if isinstance(result, ContentText | ContentImage):
+                content: str | list[Content] = [result]
+            elif isinstance(result, list) and (
                 isinstance(result[0], ContentText | ContentImage)
             ):
-                content: str | list[Content] = result
+                content = result
             else:
                 content = str(result)
 

inspect_ai/model/_chat_message.py

@@ -74,6 +74,9 @@ class ChatMessageUser(ChatMessageBase):
     role: Literal["user"] = Field(default="user")
     """Conversation role."""
 
+    tool_call_id: str | None = Field(default=None)
+    """ID of tool call this message has the content payload for."""
+
 
 class ChatMessageAssistant(ChatMessageBase):
     role: Literal["assistant"] = Field(default="assistant")

inspect_ai/model/_model.py

@@ -19,7 +19,7 @@ from tenacity import (
 )
 
 from inspect_ai._util.constants import DEFAULT_MAX_CONNECTIONS
-from inspect_ai._util.content import ContentText
+from inspect_ai._util.content import Content, ContentImage, ContentText
 from inspect_ai._util.hooks import init_hooks, override_api_key, send_telemetry
 from inspect_ai._util.platform import platform_init
 from inspect_ai._util.registry import (
@@ -40,6 +40,7 @@ from ._chat_message import (
     ChatMessage,
     ChatMessageAssistant,
     ChatMessageSystem,
+    ChatMessageTool,
     ChatMessageUser,
 )
 from ._generate_config import (
@@ -163,6 +164,10 @@ class ModelAPI(abc.ABC):
         """Any tool use in a message stream means that tools must be passed."""
         return False
 
+    def tool_result_images(self) -> bool:
+        """Tool results can containe images"""
+        return False
+
 
 class Model:
     """Model interface."""
@@ -291,6 +296,11 @@ class Model:
                 tools = []
             tool_choice = "none"
 
+        # break tool image content out into user messages if the model doesn't
+        # support tools returning images
+        if not self.api.tool_result_images():
+            input = tool_result_images_as_user_message(input)
+
         # optionally collapse *consecutive* messages into one -
         # (some apis e.g. anthropic require this)
         if self.api.collapse_user_messages():
@@ -693,6 +703,37 @@ def simple_input_messages(
     return messages
 
 
+def tool_result_images_as_user_message(
+    messages: list[ChatMessage],
+) -> list[ChatMessage]:
+    return functools.reduce(tool_result_images_reducer, messages, [])
+
+
+def tool_result_images_reducer(
+    messages: list[ChatMessage],
+    message: ChatMessage,
+) -> list[ChatMessage]:
+    # append the message
+    messages.append(message)
+
+    # if there are tool result images, pull them out into a ChatUserMessage
+    if isinstance(message, ChatMessageTool) and isinstance(message.content, list):
+        user_content: list[Content] = []
+        for i in range(0, len(message.content)):
+            if isinstance(message.content[i], ContentImage):
+                user_content.append(message.content[i])
+                message.content[i] = ContentText(
+                    text="Image content is in the message below."
+                )
+        if len(user_content) > 0:
+            messages.append(
+                ChatMessageUser(content=user_content, tool_call_id=message.tool_call_id)
+            )
+
+    # return messages
+    return messages
+
+
 # Functions to reduce consecutive user messages to a single user message -> required for some models
 def collapse_consecutive_user_messages(
     messages: list[ChatMessage],

inspect_ai/model/_providers/anthropic.py

@@ -229,6 +229,10 @@ class AnthropicAPI(ModelAPI):
     def tools_required(self) -> bool:
         return True
 
+    @override
+    def tool_result_images(self) -> bool:
+        return True
+
     # convert some common BadRequestError states into 'refusal' model output
     def handle_bad_request(self, ex: BadRequestError) -> ModelOutput | None:
         error = exception_message(ex).lower()

inspect_ai/model/_render.py

@@ -3,13 +3,20 @@ from rich.console import RenderableType
 from inspect_ai.tool._tool_call import ToolCall
 from inspect_ai.tool._tool_transcript import transcript_tool_call
 
-from ._chat_message import ChatMessage, ChatMessageAssistant, ChatMessageTool
+from ._chat_message import (
+    ChatMessage,
+    ChatMessageAssistant,
+    ChatMessageTool,
+    ChatMessageUser,
+)
 
 
 def messages_preceding_assistant(messages: list[ChatMessage]) -> list[ChatMessage]:
     preceding: list[ChatMessage] = []
     for m in reversed(messages):
-        if not isinstance(m, ChatMessageTool | ChatMessageAssistant):
+        if not isinstance(m, ChatMessageTool | ChatMessageAssistant) and not (
+            isinstance(m, ChatMessageUser) and m.tool_call_id
+        ):
             preceding.append(m)
         else:
             break

inspect_ai/scorer/_metric.py

@@ -90,6 +90,13 @@ class Score(BaseModel):
         """Read the score as a boolean."""
         return bool(self._as_scalar())
 
+    def as_list(self) -> list[str | int | float | bool]:
+        """Read the score as a list."""
+        if isinstance(self.value, list):
+            return self.value
+        else:
+            raise ValueError("This score is not a list")
+
     def as_dict(self) -> dict[str, str | int | float | bool | None]:
         """Read the score as a dictionary."""
         if isinstance(self.value, dict):
@@ -104,13 +111,17 @@ class Score(BaseModel):
             raise ValueError("This score is not a scalar")
 
 
-class SampleScore(Score):
+class SampleScore(BaseModel):
     """Score for a Sample
 
     Args:
+       score: Score
        sample_id: (str | int | None) Unique id of a sample
     """
 
+    score: Score
+    """A score"""
+
     sample_id: str | int | None = Field(default=None)
     """A sample id"""