inspect-ai 0.3.57__py3-none-any.whl → 0.3.59__py3-none-any.whl

inspect_ai/_eval/task/images.py

@@ -1,66 +1,69 @@
 import asyncio
 
 from inspect_ai._util.constants import BASE_64_DATA_REMOVED
-from inspect_ai._util.images import image_as_data_uri
+from inspect_ai._util.content import Content, ContentAudio, ContentImage, ContentVideo
+from inspect_ai._util.images import file_as_data_uri
 from inspect_ai._util.url import is_data_uri
 from inspect_ai.dataset import Sample
-from inspect_ai.model import ChatMessage, ChatMessageUser, Content, ContentImage
+from inspect_ai.model import ChatMessage, ChatMessageUser
 from inspect_ai.solver import TaskState
 
 
-async def states_with_base64_images(states: list[TaskState]) -> list[TaskState]:
-    return await asyncio.gather(*[state_with_base64_images(state) for state in states])
+async def states_with_base64_content(states: list[TaskState]) -> list[TaskState]:
+    return await asyncio.gather(*[state_with_base64_content(state) for state in states])
 
 
-async def state_with_base64_images(state: TaskState) -> TaskState:
-    state.messages = await messages_with_base64_images(state.messages)
+async def state_with_base64_content(state: TaskState) -> TaskState:
+    state.messages = await messages_with_base64_content(state.messages)
     return state
 
 
-def state_without_base64_images(state: TaskState) -> TaskState:
-    state.messages = messages_without_base64_images(state.messages)
+def state_without_base64_content(state: TaskState) -> TaskState:
+    state.messages = messages_without_base64_content(state.messages)
     return state
 
 
-async def samples_with_base64_images(samples: list[Sample]) -> list[Sample]:
+async def samples_with_base64_content(samples: list[Sample]) -> list[Sample]:
     return await asyncio.gather(
-        *[sample_with_base64_images(sample) for sample in samples]
+        *[sample_with_base64_content(sample) for sample in samples]
     )
 
 
-async def sample_with_base64_images(sample: Sample) -> Sample:
+async def sample_with_base64_content(sample: Sample) -> Sample:
     if isinstance(sample.input, list):
         return sample.model_copy(
-            update={"input": await messages_with_base64_images(sample.input)}
+            update={"input": await messages_with_base64_content(sample.input)}
         )
     else:
         return sample
 
 
-def sample_without_base64_images(sample: Sample) -> Sample:
+def sample_without_base64_content(sample: Sample) -> Sample:
     if isinstance(sample.input, list):
         return sample.model_copy(
-            update={"input": messages_without_base64_images(sample.input)}
+            update={"input": messages_without_base64_content(sample.input)}
         )
     else:
         return sample
 
 
-async def messages_with_base64_images(messages: list[ChatMessage]) -> list[ChatMessage]:
+async def messages_with_base64_content(
+    messages: list[ChatMessage],
+) -> list[ChatMessage]:
     return await asyncio.gather(
-        *[message_with_base64_image(message) for message in messages]
+        *[message_with_base64_content(message) for message in messages]
     )
 
 
-def messages_without_base64_images(messages: list[ChatMessage]) -> list[ChatMessage]:
-    return [message_without_base64_image(message) for message in messages]
+def messages_without_base64_content(messages: list[ChatMessage]) -> list[ChatMessage]:
+    return [message_without_base64_content(message) for message in messages]
 
 
-async def message_with_base64_image(message: ChatMessage) -> ChatMessage:
+async def message_with_base64_content(message: ChatMessage) -> ChatMessage:
     if isinstance(message, ChatMessageUser) and not isinstance(message.content, str):
         return ChatMessageUser(
             content=[
-                await chat_content_with_base64_image(content)
+                await chat_content_with_base64_content(content)
                 for content in message.content
             ],
             source=message.source,
@@ -69,11 +72,11 @@ async def message_with_base64_image(message: ChatMessage) -> ChatMessage:
         return message
 
 
-def message_without_base64_image(message: ChatMessage) -> ChatMessage:
+def message_without_base64_content(message: ChatMessage) -> ChatMessage:
     if isinstance(message, ChatMessageUser) and not isinstance(message.content, str):
         return ChatMessageUser(
             content=[
-                chat_content_without_base64_image(content)
+                chat_content_without_base64_content(content)
                 for content in message.content
             ],
             source=message.source,
@@ -82,18 +85,30 @@ def message_without_base64_image(message: ChatMessage) -> ChatMessage:
         return message
 
 
-async def chat_content_with_base64_image(content: Content) -> Content:
+async def chat_content_with_base64_content(content: Content) -> Content:
     if isinstance(content, ContentImage):
         return ContentImage(
-            image=await image_as_data_uri(content.image),
+            image=await file_as_data_uri(content.image),
             detail=content.detail,
         )
+    elif isinstance(content, ContentAudio):
+        return ContentAudio(
+            audio=await file_as_data_uri(content.audio), format=content.format
+        )
+    elif isinstance(content, ContentVideo):
+        return ContentVideo(
+            video=await file_as_data_uri(content.video), format=content.format
+        )
     else:
         return content
 
 
-def chat_content_without_base64_image(content: Content) -> Content:
+def chat_content_without_base64_content(content: Content) -> Content:
     if isinstance(content, ContentImage) and is_data_uri(content.image):
         return ContentImage(image=BASE_64_DATA_REMOVED, detail=content.detail)
+    elif isinstance(content, ContentAudio) and is_data_uri(content.audio):
+        return ContentAudio(audio=BASE_64_DATA_REMOVED, format="mp3")
+    elif isinstance(content, ContentVideo) and is_data_uri(content.video):
+        return ContentVideo(video=BASE_64_DATA_REMOVED, format="mp4")
     else:
         return content

inspect_ai/_eval/task/results.py

@@ -2,6 +2,7 @@ import fnmatch
 import re
 from collections import defaultdict
 from copy import deepcopy
+from dataclasses import dataclass, field
 from typing import Any, Tuple, cast
 
 from inspect_ai._util.registry import (
@@ -19,6 +20,8 @@ from inspect_ai.log import (
 from inspect_ai.log._log import EvalSampleReductions
 from inspect_ai.scorer import Metric, Score, Scorer
 from inspect_ai.scorer._metric import SampleScore
+from inspect_ai.scorer._metrics.accuracy import accuracy
+from inspect_ai.scorer._metrics.std import stderr
 from inspect_ai.scorer._reducer import ScoreReducer, mean_score, reducer_log_name
 from inspect_ai.scorer._scorer import (
     SCORER_METRICS,
@@ -27,6 +30,27 @@ from inspect_ai.scorer._scorer import (
 )
 
 
+@dataclass
+class ScorerInfo:
+    name: str
+    metrics: list[Metric | dict[str, list[Metric]]] | dict[str, list[Metric]]
+    params: dict[str, Any] = field(default_factory=dict)
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+    @staticmethod
+    def from_scorer(scorer: Scorer) -> "ScorerInfo":
+        name = registry_unqualified_name(scorer)
+        metrics = scorer_metrics(scorer)
+        metadata = deepcopy(registry_info(scorer).metadata)
+        del metadata[SCORER_METRICS]
+        params = registry_params(scorer)
+        return ScorerInfo(name=name, metrics=metrics, params=params, metadata=metadata)
+
+    @staticmethod
+    def from_name(name: str) -> "ScorerInfo":
+        return ScorerInfo(name=name, metrics=[accuracy(), stderr()])
+
+
 def eval_results(
     samples: int,
     scores: list[dict[str, SampleScore]],
@@ -38,18 +62,23 @@ def eval_results(
     results = EvalResults(total_samples=samples, completed_samples=len(scores))
     reductions = None
 
+    # extract scorers info from scorers then create scorers info for any
+    # scores not already accounted for by a scorer name
+    scorers_info = [ScorerInfo.from_scorer(scorer) for scorer in (scorers or [])]
+    scorer_names = [info.name for info in scorers_info]
+    for name in set(key for sample_scores in scores for key in sample_scores):
+        if name not in scorer_names:
+            scorers_info.append(ScorerInfo.from_name(name))
+            scorer_names.append(name)
+
     # record scorer
-    if scorers:
+    if len(scorers_info) > 0:
         result_scores: list[EvalScore] = []
         sample_reductions: list[EvalSampleReductions] = []
-        for scorer in scorers:
-            # extract non-metrics metadata
-            metadata = deepcopy(registry_info(scorer).metadata)
-            del metadata[SCORER_METRICS]
-
+        for scorer_info in scorers_info:
             # this scorer
             scorer_name = unique_scorer_name(
-                scorer, [eval_score.name for eval_score in result_scores]
+                scorer_info.name, [eval_score.name for eval_score in result_scores]
             )
 
             # scores for this scorer
@@ -75,7 +104,7 @@ def eval_results(
 
                 # Compute metrics for this scorer
                 simple_scores = cast(list[Score], reduced_scores)
-                targets = metrics if metrics is not None else scorer_metrics(scorer)
+                targets = metrics if metrics is not None else scorer_info.metrics
                 if isinstance(targets, list):
                     ## split the metrics into the simple metrics and any dictionary
                     ## metrics, to be processed independently
@@ -88,8 +117,7 @@ def eval_results(
                     result_scores.extend(
                         scorer_for_metrics(
                             scorer_name=scorer_name,
-                            scorer=scorer,
-                            metadata=metadata,
+                            scorer_info=scorer_info,
                             scores=simple_scores,
                             metrics=simple_metrics,
                             reducer_name=reducer_display_nm,
@@ -99,8 +127,7 @@ def eval_results(
                         result_scores.extend(
                             scorers_from_metric_dict(
                                 scorer_name=scorer_name,
-                                scorer=scorer,
-                                metadata=metadata,
+                                scorer_info=scorer_info,
                                 scores=simple_scores,
                                 metrics=dict_metric,
                                 reducer_name=reducer_display_nm,
@@ -116,8 +143,7 @@ def eval_results(
                     result_scores.extend(
                         scorers_from_metric_dict(
                             scorer_name=scorer_name,
-                            scorer=scorer,
-                            metadata=metadata,
+                            scorer_info=scorer_info,
                             scores=simple_scores,
                             metrics=targets,
                             reducer_name=reducer_display_nm,
@@ -156,8 +182,7 @@ def split_metrics(
 
 def scorer_for_metrics(
     scorer_name: str,
-    scorer: Scorer,
-    metadata: dict[str, Any],
+    scorer_info: ScorerInfo,
     scores: list[Score],
     metrics: list[Metric],
     reducer_name: str | None = None,
@@ -218,8 +243,10 @@ def scorer_for_metrics(
             scorer=scorer_name,
             reducer=reducer_name,
             name=scorer_name,
-            params=registry_params(scorer),
-            metadata=metadata if len(metadata.keys()) > 0 else None,
+            params=scorer_info.params,
+            metadata=scorer_info.metadata
+            if len(scorer_info.metadata.keys()) > 0
+            else None,
             metrics=list_metrics,
         )
     )
@@ -228,8 +255,7 @@ def scorer_for_metrics(
 
 def scorers_from_metric_dict(
     scorer_name: str,
-    scorer: Scorer,
-    metadata: dict[str, Any],
+    scorer_info: ScorerInfo,
     scores: list[Score],
     metrics: dict[str, list[Metric]],
     reducer_name: str | None = None,
@@ -299,8 +325,10 @@ def scorers_from_metric_dict(
                 scorer=scorer_name,
                 reducer=reducer_name,
                 name=metric_key,
-                params=registry_params(scorer),
-                metadata=metadata if len(metadata.keys()) > 0 else None,
+                params=scorer_info.params,
+                metadata=scorer_info.metadata
+                if len(scorer_info.metadata.keys()) > 0
+                else None,
                 metrics=result_metrics,
             )
         )