inspect-ai 0.3.91__py3-none-any.whl → 0.3.93__py3-none-any.whl

inspect_ai/log/_file.py

@@ -16,6 +16,7 @@ from inspect_ai._util.file import (
 )
 from inspect_ai._util.json import jsonable_python
 from inspect_ai.log._condense import resolve_sample_attachments
+from inspect_ai.log._log import EvalSampleSummary
 
 from ._log import EvalLog, EvalSample
 from ._recorders import recorder_type_for_format, recorder_type_for_location
@@ -393,6 +394,61 @@ async def read_eval_log_sample_async(
     return sample
 
 
+def read_eval_log_sample_summaries(
+    log_file: str | Path | EvalLogInfo,
+    format: Literal["eval", "json", "auto"] = "auto",
+) -> list[EvalSampleSummary]:
+    """Read sample summaries from an eval log.
+
+    Args:
+       log_file (str | FileInfo): Log file to read.
+       format (Literal["eval", "json", "auto"]): Read from format
+          (defaults to 'auto' based on `log_file` extension)
+
+    Returns:
+       Sample summaries for eval log.
+    """
+    # don't mix trio and asyncio
+    if current_async_backend() == "trio":
+        raise RuntimeError(
+            "read_eval_log_sample_summaries cannot be called from a trio async context (please use read_eval_log_sample_summaries_asymc instead)"
+        )
+
+    # will use s3fs and is not called from main inspect solver/scorer/tool/sandbox
+    # flow, so force the use of asyncio
+    return run_coroutine(read_eval_log_sample_summaries_async(log_file, format))
+
+
+async def read_eval_log_sample_summaries_async(
+    log_file: str | Path | EvalLogInfo,
+    format: Literal["eval", "json", "auto"] = "auto",
+) -> list[EvalSampleSummary]:
+    """Read sample summaries from an eval log.
+
+    Args:
+       log_file (str | FileInfo): Log file to read.
+       format (Literal["eval", "json", "auto"]): Read from format
+          (defaults to 'auto' based on `log_file` extension)
+
+    Returns:
+       Sample summaries for eval log.
+    """
+    # resolve to file path
+    log_file = (
+        log_file
+        if isinstance(log_file, str)
+        else log_file.as_posix()
+        if isinstance(log_file, Path)
+        else log_file.name
+    )
+
+    if format == "auto":
+        recorder_type = recorder_type_for_location(log_file)
+    else:
+        recorder_type = recorder_type_for_format(format)
+    return await recorder_type.read_log_sample_summaries(log_file)
+
+
 def read_eval_log_samples(
     log_file: str | Path | EvalLogInfo,
     all_samples_required: bool = True,

inspect_ai/log/_log.py

@@ -30,6 +30,7 @@ from inspect_ai.util._store import Store
 from inspect_ai.util._store_model import SMT
 
 from ._transcript import Event
+from ._util import text_input_only, thin_metadata
 
 logger = getLogger(__name__)
 
@@ -42,6 +43,7 @@ class EvalConfigDefaults(TypedDict):
     fail_on_error: bool
     sandbox_cleanup: bool
     log_samples: bool
+    log_realtime: bool
     log_images: bool
     score_display: bool
 
@@ -53,6 +55,7 @@ def eval_config_defaults() -> EvalConfigDefaults:
         "fail_on_error": True,
         "sandbox_cleanup": True,
         "log_samples": True,
+        "log_realtime": True,
         "log_images": True,
         "score_display": True,
     }
@@ -120,6 +123,9 @@ class EvalConfig(BaseModel):
     log_samples: bool | None = Field(default=None)
     """Log detailed information on each sample."""
 
+    log_realtime: bool | None = Field(default=None)
+    """Log events in realtime (enables live viewing of samples in inspect view)."""
+
     log_images: bool | None = Field(default=None)
     """Log base64 encoded versions of images."""
 
@@ -161,6 +167,70 @@ class EvalSampleLimit(BaseModel):
     """The limit value"""
 
 
+class EvalSampleSummary(BaseModel):
+    """Summary information (including scoring) for a sample."""
+
+    id: int | str
+    """Unique id for sample."""
+
+    epoch: int
+    """Epoch number for sample."""
+
+    input: str | list[ChatMessage]
+    """Sample input (text inputs only)."""
+
+    target: str | list[str]
+    """Sample target value(s)"""
+
+    metadata: dict[str, Any] = Field(default_factory=dict)
+    """Sample metadata (scalar types only, strings truncated to 1k)."""
+
+    scores: dict[str, Score] | None = Field(default=None)
+    """Scores for sample (score values only, no answers, explanations, or metadata)."""
+
+    model_usage: dict[str, ModelUsage] = Field(default_factory=dict)
+    """Model token usage for sample."""
+
+    total_time: float | None = Field(default=None)
+    """Total time that the sample was running."""
+
+    working_time: float | None = Field(default=None)
+    """Time spent working (model generation, sandbox calls, etc.)"""
+
+    uuid: str | None = Field(default=None)
+    """Globally unique identifier for sample run (exists for samples created in Inspect >= 0.3.70)"""
+
+    error: str | None = Field(default=None)
+    """Error that halted sample."""
+
+    limit: str | None = Field(default=None)
+    """Limit that halted the sample"""
+
+    retries: int | None = Field(default=None)
+    """Number of retries for the sample."""
+
+    completed: bool = Field(default=False)
+    """Is the sample complete."""
+
+    @model_validator(mode="after")
+    def thin_data(self) -> "EvalSampleSummary":
+        # thin input
+        self.input = text_input_only(self.input)
+
+        # thin metadata
+        self.metadata = thin_metadata(self.metadata)
+
+        # thin score explanations and metadata
+        if self.scores is not None:
+            self.scores = {
+                key: Score(value=score.value) for key, score in self.scores.items()
+            }
+        return self
+
+    # allow field model_usage
+    model_config = ConfigDict(protected_namespaces=())
+
+
 class EvalSample(BaseModel):
     """Sample from evaluation task."""
 
@@ -271,6 +341,35 @@ class EvalSample(BaseModel):
     limit: EvalSampleLimit | None = Field(default=None)
     """The limit that halted the sample"""
 
+    def summary(self) -> EvalSampleSummary:
+        """Summary of sample.
+
+        The summary excludes potentially large fields like messages, output,
+        events, store, and metadata so that it is always fast to load.
+
+        If there are images, audio, or video in the input, they are
+        replaced with a placeholder.
+
+        Returns:
+           Summary of sample.
+        """
+        return EvalSampleSummary(
+            id=self.id,
+            epoch=self.epoch,
+            input=self.input,
+            target=self.target,
+            metadata=self.metadata,
+            scores=self.scores,
+            model_usage=self.model_usage,
+            total_time=self.total_time,
+            working_time=self.working_time,
+            uuid=self.uuid,
+            error=self.error.message if self.error is not None else None,
+            limit=f"{self.limit.type}" if self.limit is not None else None,
+            retries=len(self.error_retries) if self.error_retries is not None else None,
+            completed=True,
+        )
+
     # deprecated properties
 
     @property

inspect_ai/log/_recorders/__init__.py

@@ -1,3 +1,4 @@
+from .._log import EvalSampleSummary
 from .create import (
     create_recorder_for_format,
     create_recorder_for_location,
@@ -7,6 +8,7 @@ from .create import (
 from .recorder import Recorder
 
 __all__ = [
+    "EvalSampleSummary",
     "Recorder",
     "create_recorder_for_format",
     "create_recorder_for_location",

inspect_ai/log/_recorders/buffer/database.py

@@ -26,7 +26,8 @@ from ..._condense import (
     walk_input,
     walk_json_dict,
 )
-from ..types import SampleEvent, SampleSummary
+from ..._log import EvalSampleSummary
+from ..types import SampleEvent
 from .filestore import (
     Manifest,
     SampleBufferFilestore,
@@ -141,7 +142,7 @@ class SampleBufferDatabase(SampleBuffer):
         )
         self._sync_time = time.monotonic()
 
-    def start_sample(self, sample: SampleSummary) -> None:
+    def start_sample(self, sample: EvalSampleSummary) -> None:
         with self._get_connection(write=True) as conn:
             sample = self._consense_sample(conn, sample)
             conn.execute(
@@ -177,7 +178,7 @@ class SampleBufferDatabase(SampleBuffer):
             # Insert all rows
             conn.execute(sql, values)
 
-    def complete_sample(self, summary: SampleSummary) -> None:
+    def complete_sample(self, summary: EvalSampleSummary) -> None:
         with self._get_connection(write=True) as conn:
             summary = self._consense_sample(conn, summary)
             conn.execute(
@@ -307,9 +308,9 @@ class SampleBufferDatabase(SampleBuffer):
             conn.execute("PRAGMA foreign_keys = ON")
 
             # concurrency setup
-            conn.execute("PRAGMA journal_mode=WAL")
+            conn.execute("PRAGMA journal_mode=MEMORY")
             conn.execute("PRAGMA busy_timeout=10000")
-            conn.execute("PRAGMA synchronous=NORMAL")
+            conn.execute("PRAGMA synchronous=OFF")
 
             # do work
             yield conn
@@ -359,7 +360,7 @@ class SampleBufferDatabase(SampleBuffer):
 
     def _get_samples(
         self, conn: Connection, resolve_attachments: bool = False
-    ) -> Iterator[SampleSummary]:
+    ) -> Iterator[EvalSampleSummary]:
         cursor = conn.execute(
             """
             SELECT s.data as sample_data
@@ -369,7 +370,7 @@ class SampleBufferDatabase(SampleBuffer):
         )
 
         for row in cursor:
-            summary = SampleSummary.model_validate_json(row["sample_data"])
+            summary = EvalSampleSummary.model_validate_json(row["sample_data"])
             if resolve_attachments:
                 summary = self._resolve_sample_attachments(conn, summary)
             yield summary
@@ -437,8 +438,8 @@ class SampleBufferDatabase(SampleBuffer):
             )
 
     def _consense_sample(
-        self, conn: Connection, sample: SampleSummary
-    ) -> SampleSummary:
+        self, conn: Connection, sample: EvalSampleSummary
+    ) -> EvalSampleSummary:
         # alias attachments
         attachments: dict[str, str] = {}
         sample = sample.model_copy(
@@ -456,8 +457,8 @@ class SampleBufferDatabase(SampleBuffer):
         return sample
 
     def _resolve_sample_attachments(
-        self, conn: Connection, sample: SampleSummary
-    ) -> SampleSummary:
+        self, conn: Connection, sample: EvalSampleSummary
+    ) -> EvalSampleSummary:
         return sample.model_copy(
             update={
                 "input": walk_input(

inspect_ai/log/_recorders/buffer/filestore.py

@@ -14,7 +14,7 @@ from inspect_ai._util.file import FileSystem, basename, dirname, file, filesyste
 from inspect_ai._util.json import to_json_safe, to_json_str_safe
 from inspect_ai.log._file import read_eval_log
 
-from ..types import SampleSummary
+from ..._log import EvalSampleSummary
 from .types import SampleBuffer, SampleData, Samples
 
 logger = getLogger(__name__)
@@ -33,7 +33,7 @@ class SegmentFile(BaseModel):
 
 
 class SampleManifest(BaseModel):
-    summary: SampleSummary
+    summary: EvalSampleSummary
     segments: list[int] = Field(default_factory=list)
 
 

inspect_ai/log/_recorders/buffer/types.py

@@ -5,13 +5,13 @@ from pydantic import BaseModel, JsonValue
 
 from inspect_ai._display.core.display import TaskDisplayMetric
 
-from ..types import SampleSummary
+from ..._log import EvalSampleSummary
 
 JsonData: TypeAlias = dict[str, JsonValue]
 
 
 class Samples(BaseModel):
-    samples: list[SampleSummary]
+    samples: list[EvalSampleSummary]
     metrics: list[TaskDisplayMetric]
     refresh: int
     etag: str

inspect_ai/log/_recorders/eval.py

@@ -11,18 +11,10 @@ from pydantic_core import to_json
 from typing_extensions import override
 
 from inspect_ai._util.constants import DESERIALIZING_CONTEXT, LOG_SCHEMA_VERSION
-from inspect_ai._util.content import (
-    ContentAudio,
-    ContentImage,
-    ContentReasoning,
-    ContentText,
-    ContentVideo,
-)
 from inspect_ai._util.error import EvalError
 from inspect_ai._util.file import FileSystem, dirname, file, filesystem
 from inspect_ai._util.json import jsonable_python
 from inspect_ai._util.trace import trace_action
-from inspect_ai.model._chat_message import ChatMessage
 
 from .._log import (
     EvalLog,
@@ -30,12 +22,12 @@ from .._log import (
     EvalResults,
     EvalSample,
     EvalSampleReductions,
+    EvalSampleSummary,
     EvalSpec,
     EvalStats,
     sort_samples,
 )
 from .file import FileRecorder
-from .types import SampleSummary
 
 logger = getLogger(__name__)
 
@@ -222,6 +214,15 @@ class EvalRecorder(FileRecorder):
                         f"Sample id {id} for epoch {epoch} not found in log {location}"
                     )
 
+    @classmethod
+    @override
+    async def read_log_sample_summaries(cls, location: str) -> list[EvalSampleSummary]:
+        with file(location, "rb") as z:
+            with ZipFile(z, mode="r") as zip:
+                summary_counter = _read_summary_counter(zip)
+                summaries = _read_all_summaries(zip, summary_counter)
+                return summaries
+
     @classmethod
     @override
     async def write_log(cls, location: str, log: EvalLog) -> None:
@@ -236,36 +237,6 @@ class EvalRecorder(FileRecorder):
         )
 
 
-def text_inputs(inputs: str | list[ChatMessage]) -> str | list[ChatMessage]:
-    # Clean the input of any images
-    if isinstance(inputs, list):
-        input: list[ChatMessage] = []
-        for message in inputs:
-            if not isinstance(message.content, str):
-                filtered_content: list[
-                    ContentText
-                    | ContentReasoning
-                    | ContentImage
-                    | ContentAudio
-                    | ContentVideo
-                ] = []
-                for content in message.content:
-                    if content.type == "text":
-                        filtered_content.append(content)
-                    else:
-                        filtered_content.append(
-                            ContentText(text=f"({content.type.capitalize()})")
-                        )
-                message.content = filtered_content
-                input.append(message)
-            else:
-                input.append(message)
-
-        return input
-    else:
-        return inputs
-
-
 class ZipLogFile:
     _zip: ZipFile | None
     _temp_file: BinaryIO
@@ -273,19 +244,20 @@ class ZipLogFile:
 
     def __init__(self, file: str) -> None:
         self._file = file
+        self._zip = None
         self._fs = filesystem(file)
         self._lock = anyio.Lock()
         self._temp_file = tempfile.TemporaryFile()
         self._samples: list[EvalSample] = []
         self._summary_counter = 0
-        self._summaries: list[SampleSummary] = []
+        self._summaries: list[EvalSampleSummary] = []
         self._log_start: LogStart | None = None
 
     async def init(
         self,
         log_start: LogStart | None,
         summary_counter: int,
-        summaries: list[SampleSummary],
+        summaries: list[EvalSampleSummary],
     ) -> None:
         async with self._lock:
             self._open()
@@ -309,31 +281,14 @@ class ZipLogFile:
     async def write_buffered_samples(self) -> None:
         async with self._lock:
             # Write the buffered samples
-            summaries: list[SampleSummary] = []
+            summaries: list[EvalSampleSummary] = []
             for sample in self._samples:
                 # Write the sample
                 self._zip_writestr(_sample_filename(sample.id, sample.epoch), sample)
 
                 # Capture the summary
-                summaries.append(
-                    SampleSummary(
-                        id=sample.id,
-                        epoch=sample.epoch,
-                        input=text_inputs(sample.input),
-                        target=sample.target,
-                        completed=True,
-                        scores=sample.scores,
-                        error=sample.error.message
-                        if sample.error is not None
-                        else None,
-                        limit=f"{sample.limit.type}"
-                        if sample.limit is not None
-                        else None,
-                        retries=len(sample.error_retries)
-                        if sample.error_retries is not None
-                        else None,
-                    )
-                )
+                summaries.append(sample.summary())
+
             self._samples.clear()
 
             # write intermediary summaries and add to master list
@@ -451,12 +406,12 @@ def _read_summary_counter(zip: ZipFile) -> int:
     return current_count
 
 
-def _read_all_summaries(zip: ZipFile, count: int) -> list[SampleSummary]:
+def _read_all_summaries(zip: ZipFile, count: int) -> list[EvalSampleSummary]:
     if SUMMARIES_JSON in zip.namelist():
         summaries_raw = _read_json(zip, SUMMARIES_JSON)
         if isinstance(summaries_raw, list):
             return [
-                SampleSummary.model_validate(value, context=DESERIALIZING_CONTEXT)
+                EvalSampleSummary.model_validate(value, context=DESERIALIZING_CONTEXT)
                 for value in summaries_raw
             ]
         else:
@@ -464,7 +419,7 @@ def _read_all_summaries(zip: ZipFile, count: int) -> list[SampleSummary]:
                 f"Expected a list of summaries when reading {SUMMARIES_JSON}"
             )
     else:
-        summaries: list[SampleSummary] = []
+        summaries: list[EvalSampleSummary] = []
         for i in range(1, count):
             summary_file = _journal_summary_file(i)
             summary_path = _journal_summary_path(summary_file)
@@ -472,7 +427,7 @@ def _read_all_summaries(zip: ZipFile, count: int) -> list[SampleSummary]:
             if isinstance(summary, list):
                 summaries.extend(
                     [
-                        SampleSummary.model_validate(
+                        EvalSampleSummary.model_validate(
                             value, context=DESERIALIZING_CONTEXT
                         )
                         for value in summary

inspect_ai/log/_recorders/file.py

@@ -8,7 +8,7 @@ from inspect_ai._util.constants import MODEL_NONE
 from inspect_ai._util.file import filesystem
 from inspect_ai._util.registry import registry_unqualified_name
 
-from .._log import EvalLog, EvalSample, EvalSpec
+from .._log import EvalLog, EvalSample, EvalSampleSummary, EvalSpec
 from .recorder import Recorder
 
 logger = getLogger(__name__)
@@ -40,11 +40,7 @@ class FileRecorder(Recorder):
         cls, location: str, id: str | int, epoch: int = 1
     ) -> EvalSample:
         # establish the log to read from (might be cached)
-        if cls.__last_read_sample_log and (cls.__last_read_sample_log[0] == "location"):
-            eval_log = cls.__last_read_sample_log[1]
-        else:
-            eval_log = await cls.read_log(location)
-            cls.__last_read_sample_log = (location, eval_log)
+        eval_log = await cls._log_file_maybe_cached(location)
 
         # throw if no samples
         if not eval_log.samples:
@@ -66,6 +62,32 @@ class FileRecorder(Recorder):
         else:
             return eval_sample
 
+    @classmethod
+    @override
+    async def read_log_sample_summaries(cls, location: str) -> list[EvalSampleSummary]:
+        # establish the log to read from (might be cached)
+        eval_log = await cls._log_file_maybe_cached(location)
+
+        # throw if no samples
+        if not eval_log.samples:
+            raise IndexError(f"No samples found in log {location}")
+
+        summaries: list[EvalSampleSummary] = []
+        for sample in eval_log.samples:
+            summaries.append(sample.summary())
+
+        return summaries
+
+    @classmethod
+    async def _log_file_maybe_cached(cls, location: str) -> EvalLog:
+        # establish the log to read from (might be cached)
+        if cls.__last_read_sample_log and (cls.__last_read_sample_log[0] == "location"):
+            eval_log = cls.__last_read_sample_log[1]
+        else:
+            eval_log = await cls.read_log(location)
+            cls.__last_read_sample_log = (location, eval_log)
+        return eval_log
+
     def _log_file_key(self, eval: EvalSpec) -> str:
         # clean underscores, slashes, and : from the log file key (so we can reliably parse it
         # later without worrying about underscores)

inspect_ai/log/_recorders/recorder.py

@@ -8,6 +8,7 @@ from inspect_ai.log._log import (
     EvalResults,
     EvalSample,
     EvalSampleReductions,
+    EvalSampleSummary,
     EvalSpec,
     EvalStats,
 )
@@ -57,6 +58,12 @@ class Recorder(abc.ABC):
         cls, location: str, id: str | int, epoch: int = 1
     ) -> EvalSample: ...
 
+    @classmethod
+    @abc.abstractmethod
+    async def read_log_sample_summaries(
+        cls, location: str
+    ) -> list[EvalSampleSummary]: ...
+
     @classmethod
     @abc.abstractmethod
     async def write_log(cls, location: str, log: EvalLog) -> None: ...

inspect_ai/log/_recorders/types.py

@@ -1,31 +1,9 @@
-from pydantic import BaseModel, Field, model_validator
+from pydantic import BaseModel
 
 from inspect_ai.log._transcript import Event
-from inspect_ai.model._chat_message import ChatMessage
-from inspect_ai.scorer._metric import Score
 
 
 class SampleEvent(BaseModel):
     id: str | int
     epoch: int
     event: Event
-
-
-class SampleSummary(BaseModel):
-    id: int | str
-    epoch: int
-    input: str | list[ChatMessage]
-    target: str | list[str]
-    completed: bool = Field(default=False)
-    scores: dict[str, Score] | None = Field(default=None)
-    error: str | None = Field(default=None)
-    limit: str | None = Field(default=None)
-    retries: int | None = Field(default=None)
-
-    @model_validator(mode="after")
-    def thin_scores(self) -> "SampleSummary":
-        if self.scores is not None:
-            self.scores = {
-                key: Score(value=score.value) for key, score in self.scores.items()
-            }
-        return self

inspect_ai/log/_samples.py

@@ -119,14 +119,6 @@ def sample_active() -> ActiveSample | None:
     return _sample_active.get(None)
 
 
-def active_sample_token_limit() -> int | None:
-    active = sample_active()
-    if active:
-        return active.token_limit
-    else:
-        return None
-
-
 def set_active_sample_token_limit(token_limit: int | None) -> None:
     active = sample_active()
     if active:

inspect_ai/log/_transcript.py

@@ -14,7 +14,13 @@ from typing import (
     Union,
 )
 
-from pydantic import BaseModel, ConfigDict, Field, JsonValue, field_serializer
+from pydantic import (
+    BaseModel,
+    ConfigDict,
+    Field,
+    JsonValue,
+    field_serializer,
+)
 from shortuuid import uuid
 
 from inspect_ai._util.constants import SAMPLE_SUBTASK