inspect-ai 0.3.92__py3-none-any.whl → 0.3.94__py3-none-any.whl

inspect_ai/log/_recorders/buffer/filestore.py

@@ -14,7 +14,7 @@ from inspect_ai._util.file import FileSystem, basename, dirname, file, filesyste
 from inspect_ai._util.json import to_json_safe, to_json_str_safe
 from inspect_ai.log._file import read_eval_log
 
-from ..types import SampleSummary
+from ..._log import EvalSampleSummary
 from .types import SampleBuffer, SampleData, Samples
 
 logger = getLogger(__name__)
@@ -33,7 +33,7 @@ class SegmentFile(BaseModel):
 
 
 class SampleManifest(BaseModel):
-    summary: SampleSummary
+    summary: EvalSampleSummary
     segments: list[int] = Field(default_factory=list)
 
 

inspect_ai/log/_recorders/buffer/types.py

@@ -5,13 +5,13 @@ from pydantic import BaseModel, JsonValue
 
 from inspect_ai._display.core.display import TaskDisplayMetric
 
-from ..types import SampleSummary
+from ..._log import EvalSampleSummary
 
 JsonData: TypeAlias = dict[str, JsonValue]
 
 
 class Samples(BaseModel):
-    samples: list[SampleSummary]
+    samples: list[EvalSampleSummary]
     metrics: list[TaskDisplayMetric]
     refresh: int
     etag: str

inspect_ai/log/_recorders/eval.py

@@ -11,18 +11,10 @@ from pydantic_core import to_json
 from typing_extensions import override
 
 from inspect_ai._util.constants import DESERIALIZING_CONTEXT, LOG_SCHEMA_VERSION
-from inspect_ai._util.content import (
-    ContentAudio,
-    ContentImage,
-    ContentReasoning,
-    ContentText,
-    ContentVideo,
-)
 from inspect_ai._util.error import EvalError
 from inspect_ai._util.file import FileSystem, dirname, file, filesystem
 from inspect_ai._util.json import jsonable_python
 from inspect_ai._util.trace import trace_action
-from inspect_ai.model._chat_message import ChatMessage
 
 from .._log import (
     EvalLog,
@@ -30,12 +22,12 @@ from .._log import (
     EvalResults,
     EvalSample,
     EvalSampleReductions,
+    EvalSampleSummary,
     EvalSpec,
     EvalStats,
     sort_samples,
 )
 from .file import FileRecorder
-from .types import SampleSummary
 
 logger = getLogger(__name__)
 
@@ -222,6 +214,15 @@ class EvalRecorder(FileRecorder):
                         f"Sample id {id} for epoch {epoch} not found in log {location}"
                     )
 
+    @classmethod
+    @override
+    async def read_log_sample_summaries(cls, location: str) -> list[EvalSampleSummary]:
+        with file(location, "rb") as z:
+            with ZipFile(z, mode="r") as zip:
+                summary_counter = _read_summary_counter(zip)
+                summaries = _read_all_summaries(zip, summary_counter)
+                return summaries
+
     @classmethod
     @override
     async def write_log(cls, location: str, log: EvalLog) -> None:
@@ -236,36 +237,6 @@ class EvalRecorder(FileRecorder):
         )
 
 
-def text_inputs(inputs: str | list[ChatMessage]) -> str | list[ChatMessage]:
-    # Clean the input of any images
-    if isinstance(inputs, list):
-        input: list[ChatMessage] = []
-        for message in inputs:
-            if not isinstance(message.content, str):
-                filtered_content: list[
-                    ContentText
-                    | ContentReasoning
-                    | ContentImage
-                    | ContentAudio
-                    | ContentVideo
-                ] = []
-                for content in message.content:
-                    if content.type == "text":
-                        filtered_content.append(content)
-                    else:
-                        filtered_content.append(
-                            ContentText(text=f"({content.type.capitalize()})")
-                        )
-                message.content = filtered_content
-                input.append(message)
-            else:
-                input.append(message)
-
-        return input
-    else:
-        return inputs
-
-
 class ZipLogFile:
     _zip: ZipFile | None
     _temp_file: BinaryIO
@@ -273,19 +244,20 @@ class ZipLogFile:
 
     def __init__(self, file: str) -> None:
         self._file = file
+        self._zip = None
         self._fs = filesystem(file)
         self._lock = anyio.Lock()
         self._temp_file = tempfile.TemporaryFile()
         self._samples: list[EvalSample] = []
         self._summary_counter = 0
-        self._summaries: list[SampleSummary] = []
+        self._summaries: list[EvalSampleSummary] = []
         self._log_start: LogStart | None = None
 
     async def init(
         self,
         log_start: LogStart | None,
         summary_counter: int,
-        summaries: list[SampleSummary],
+        summaries: list[EvalSampleSummary],
     ) -> None:
         async with self._lock:
             self._open()
@@ -309,31 +281,14 @@ class ZipLogFile:
     async def write_buffered_samples(self) -> None:
         async with self._lock:
             # Write the buffered samples
-            summaries: list[SampleSummary] = []
+            summaries: list[EvalSampleSummary] = []
             for sample in self._samples:
                 # Write the sample
                 self._zip_writestr(_sample_filename(sample.id, sample.epoch), sample)
 
                 # Capture the summary
-                summaries.append(
-                    SampleSummary(
-                        id=sample.id,
-                        epoch=sample.epoch,
-                        input=text_inputs(sample.input),
-                        target=sample.target,
-                        completed=True,
-                        scores=sample.scores,
-                        error=sample.error.message
-                        if sample.error is not None
-                        else None,
-                        limit=f"{sample.limit.type}"
-                        if sample.limit is not None
-                        else None,
-                        retries=len(sample.error_retries)
-                        if sample.error_retries is not None
-                        else None,
-                    )
-                )
+                summaries.append(sample.summary())
+
             self._samples.clear()
 
             # write intermediary summaries and add to master list
@@ -451,12 +406,12 @@ def _read_summary_counter(zip: ZipFile) -> int:
     return current_count
 
 
-def _read_all_summaries(zip: ZipFile, count: int) -> list[SampleSummary]:
+def _read_all_summaries(zip: ZipFile, count: int) -> list[EvalSampleSummary]:
     if SUMMARIES_JSON in zip.namelist():
         summaries_raw = _read_json(zip, SUMMARIES_JSON)
         if isinstance(summaries_raw, list):
             return [
-                SampleSummary.model_validate(value, context=DESERIALIZING_CONTEXT)
+                EvalSampleSummary.model_validate(value, context=DESERIALIZING_CONTEXT)
                 for value in summaries_raw
             ]
         else:
@@ -464,7 +419,7 @@ def _read_all_summaries(zip: ZipFile, count: int) -> list[SampleSummary]:
                 f"Expected a list of summaries when reading {SUMMARIES_JSON}"
             )
     else:
-        summaries: list[SampleSummary] = []
+        summaries: list[EvalSampleSummary] = []
         for i in range(1, count):
             summary_file = _journal_summary_file(i)
             summary_path = _journal_summary_path(summary_file)
@@ -472,7 +427,7 @@ def _read_all_summaries(zip: ZipFile, count: int) -> list[SampleSummary]:
             if isinstance(summary, list):
                 summaries.extend(
                     [
-                        SampleSummary.model_validate(
+                        EvalSampleSummary.model_validate(
                             value, context=DESERIALIZING_CONTEXT
                         )
                         for value in summary

inspect_ai/log/_recorders/file.py

@@ -8,7 +8,7 @@ from inspect_ai._util.constants import MODEL_NONE
 from inspect_ai._util.file import filesystem
 from inspect_ai._util.registry import registry_unqualified_name
 
-from .._log import EvalLog, EvalSample, EvalSpec
+from .._log import EvalLog, EvalSample, EvalSampleSummary, EvalSpec
 from .recorder import Recorder
 
 logger = getLogger(__name__)
@@ -40,11 +40,7 @@ class FileRecorder(Recorder):
         cls, location: str, id: str | int, epoch: int = 1
     ) -> EvalSample:
         # establish the log to read from (might be cached)
-        if cls.__last_read_sample_log and (cls.__last_read_sample_log[0] == "location"):
-            eval_log = cls.__last_read_sample_log[1]
-        else:
-            eval_log = await cls.read_log(location)
-            cls.__last_read_sample_log = (location, eval_log)
+        eval_log = await cls._log_file_maybe_cached(location)
 
         # throw if no samples
         if not eval_log.samples:
@@ -66,6 +62,32 @@ class FileRecorder(Recorder):
         else:
             return eval_sample
 
+    @classmethod
+    @override
+    async def read_log_sample_summaries(cls, location: str) -> list[EvalSampleSummary]:
+        # establish the log to read from (might be cached)
+        eval_log = await cls._log_file_maybe_cached(location)
+
+        # throw if no samples
+        if not eval_log.samples:
+            raise IndexError(f"No samples found in log {location}")
+
+        summaries: list[EvalSampleSummary] = []
+        for sample in eval_log.samples:
+            summaries.append(sample.summary())
+
+        return summaries
+
+    @classmethod
+    async def _log_file_maybe_cached(cls, location: str) -> EvalLog:
+        # establish the log to read from (might be cached)
+        if cls.__last_read_sample_log and (cls.__last_read_sample_log[0] == "location"):
+            eval_log = cls.__last_read_sample_log[1]
+        else:
+            eval_log = await cls.read_log(location)
+            cls.__last_read_sample_log = (location, eval_log)
+        return eval_log
+
     def _log_file_key(self, eval: EvalSpec) -> str:
         # clean underscores, slashes, and : from the log file key (so we can reliably parse it
         # later without worrying about underscores)

inspect_ai/log/_recorders/recorder.py

@@ -8,6 +8,7 @@ from inspect_ai.log._log import (
     EvalResults,
     EvalSample,
     EvalSampleReductions,
+    EvalSampleSummary,
     EvalSpec,
     EvalStats,
 )
@@ -57,6 +58,12 @@ class Recorder(abc.ABC):
         cls, location: str, id: str | int, epoch: int = 1
     ) -> EvalSample: ...
 
+    @classmethod
+    @abc.abstractmethod
+    async def read_log_sample_summaries(
+        cls, location: str
+    ) -> list[EvalSampleSummary]: ...
+
     @classmethod
     @abc.abstractmethod
     async def write_log(cls, location: str, log: EvalLog) -> None: ...

inspect_ai/log/_recorders/types.py

@@ -1,31 +1,9 @@
-from pydantic import BaseModel, Field, model_validator
+from pydantic import BaseModel
 
 from inspect_ai.log._transcript import Event
-from inspect_ai.model._chat_message import ChatMessage
-from inspect_ai.scorer._metric import Score
 
 
 class SampleEvent(BaseModel):
     id: str | int
     epoch: int
     event: Event
-
-
-class SampleSummary(BaseModel):
-    id: int | str
-    epoch: int
-    input: str | list[ChatMessage]
-    target: str | list[str]
-    completed: bool = Field(default=False)
-    scores: dict[str, Score] | None = Field(default=None)
-    error: str | None = Field(default=None)
-    limit: str | None = Field(default=None)
-    retries: int | None = Field(default=None)
-
-    @model_validator(mode="after")
-    def thin_scores(self) -> "SampleSummary":
-        if self.scores is not None:
-            self.scores = {
-                key: Score(value=score.value) for key, score in self.scores.items()
-            }
-        return self

inspect_ai/log/_samples.py

@@ -5,12 +5,11 @@ from typing import AsyncGenerator, Iterator, Literal
 
 from shortuuid import uuid
 
-from inspect_ai._util.constants import SAMPLE_SUBTASK
 from inspect_ai.dataset._dataset import Sample
 from inspect_ai.util._sandbox import SandboxConnection
 from inspect_ai.util._sandbox.context import sandbox_connections
 
-from ._transcript import Transcript, transcript
+from ._transcript import ModelEvent, Transcript
 
 
 class ActiveSample:
@@ -47,7 +46,6 @@ class ActiveSample:
         self.total_tokens = 0
         self.transcript = transcript
         self.sandboxes = sandboxes
-        self.retry_count = 0
         self._interrupt_action: Literal["score", "error"] | None = None
 
     @property
@@ -119,14 +117,6 @@ def sample_active() -> ActiveSample | None:
     return _sample_active.get(None)
 
 
-def active_sample_token_limit() -> int | None:
-    active = sample_active()
-    if active:
-        return active.token_limit
-    else:
-        return None
-
-
 def set_active_sample_token_limit(token_limit: int | None) -> None:
     active = sample_active()
     if active:
@@ -159,27 +149,26 @@ def set_active_sample_total_messages(total_messages: int) -> None:
         active.total_messages = total_messages
 
 
+_active_model_event: ContextVar[ModelEvent | None] = ContextVar(
+    "_active_model_event", default=None
+)
+
+
 @contextlib.contextmanager
-def track_active_sample_retries() -> Iterator[None]:
-    reset_active_sample_retries()
+def track_active_model_event(event: ModelEvent) -> Iterator[None]:
+    token = _active_model_event.set(event)
     try:
         yield
     finally:
-        reset_active_sample_retries()
-
-
-def reset_active_sample_retries() -> None:
-    active = sample_active()
-    if active:
-        active.retry_count = 0
+        _active_model_event.reset(token)
 
 
 def report_active_sample_retry() -> None:
-    active = sample_active()
-    if active:
-        # only do this for the top level subtask
-        if transcript().name == SAMPLE_SUBTASK:
-            active.retry_count = active.retry_count + 1
+    model_event = _active_model_event.get()
+    if model_event is not None:
+        if model_event.retries is None:
+            model_event.retries = 0
+        model_event.retries = model_event.retries + 1
 
 
 _sample_active: ContextVar[ActiveSample | None] = ContextVar(

inspect_ai/log/_transcript.py

@@ -14,12 +14,19 @@ from typing import (
     Union,
 )
 
-from pydantic import BaseModel, ConfigDict, Field, JsonValue, field_serializer
+from pydantic import (
+    BaseModel,
+    ConfigDict,
+    Field,
+    JsonValue,
+    field_serializer,
+)
 from shortuuid import uuid
 
-from inspect_ai._util.constants import SAMPLE_SUBTASK
+from inspect_ai._util.constants import DESERIALIZING
 from inspect_ai._util.error import EvalError
-from inspect_ai._util.json import JsonChange, json_changes
+from inspect_ai._util.json import JsonChange
+from inspect_ai._util.logger import warn_once
 from inspect_ai._util.working import sample_working_time
 from inspect_ai.dataset._dataset import Sample
 from inspect_ai.log._message import LoggingMessage
@@ -28,7 +35,6 @@ from inspect_ai.model._generate_config import GenerateConfig
 from inspect_ai.model._model_call import ModelCall
 from inspect_ai.model._model_output import ModelOutput
 from inspect_ai.scorer._metric import Score
-from inspect_ai.solver._task_state import state_jsonable
 from inspect_ai.tool._tool import ToolResult
 from inspect_ai.tool._tool_call import (
     ToolCall,
@@ -38,6 +44,7 @@ from inspect_ai.tool._tool_call import (
 )
 from inspect_ai.tool._tool_choice import ToolChoice
 from inspect_ai.tool._tool_info import ToolInfo
+from inspect_ai.util._span import current_span_id
 from inspect_ai.util._store import store, store_changes, store_jsonable
 
 logger = getLogger(__name__)
@@ -51,6 +58,9 @@ class BaseEvent(BaseModel):
     }
     id_: str = Field(default_factory=lambda: str(uuid()), exclude=True)
 
+    span_id: str | None = Field(default=None)
+    """Span the event occurred within."""
+
     timestamp: datetime = Field(default_factory=datetime.now)
     """Clock time at which event occurred."""
 
@@ -60,6 +70,17 @@ class BaseEvent(BaseModel):
     pending: bool | None = Field(default=None)
     """Is this event pending?"""
 
+    def model_post_init(self, __context: Any) -> None:
+        # check if deserializing
+        is_deserializing = isinstance(__context, dict) and __context.get(
+            DESERIALIZING, False
+        )
+
+        # Generate context id fields if not deserializing
+        if not is_deserializing:
+            if self.span_id is None:
+                self.span_id = current_span_id()
+
     @field_serializer("timestamp")
     def serialize_timestamp(self, dt: datetime) -> str:
         return dt.astimezone().isoformat()
@@ -141,6 +162,9 @@ class ModelEvent(BaseEvent):
     output: ModelOutput
     """Output from model."""
 
+    retries: int | None = Field(default=None)
+    """Retries for the model API request."""
+
     error: str | None = Field(default=None)
     """Error which occurred during model call."""
 
@@ -197,7 +221,13 @@ class ToolEvent(BaseEvent):
     """Error that occurred during tool call."""
 
     events: list["Event"] = Field(default_factory=list)
-    """Transcript of events for tool."""
+    """Transcript of events for tool.
+
+    Note that events are no longer recorded separately within
+    tool events but rather all events are recorded in the main
+    transcript. This field is deprecated and here for backwards
+    compatibility with transcripts that have sub-events.
+    """
 
     completed: datetime | None = Field(default=None)
     """Time that tool call completed (see `timestamp` for started)"""
@@ -216,7 +246,6 @@ class ToolEvent(BaseEvent):
         result: ToolResult,
         truncated: tuple[int, int] | None,
         error: ToolCallError | None,
-        events: list["Event"],
         waiting_time: float,
         agent: str | None,
         failed: bool | None,
@@ -224,7 +253,6 @@ class ToolEvent(BaseEvent):
         self.result = result
         self.truncated = truncated
         self.error = error
-        self.events = events
         self.pending = None
         completed = datetime.now()
         self.completed = completed
@@ -396,6 +424,35 @@ class ScoreEvent(BaseEvent):
     """Was this an intermediate scoring?"""
 
 
+class SpanBeginEvent(BaseEvent):
+    """Mark the beginning of a transcript span."""
+
+    event: Literal["span_begin"] = Field(default="span_begin")
+    """Event type."""
+
+    id: str
+    """Unique identifier for span."""
+
+    parent_id: str | None = Field(default=None)
+    """Identifier for parent span."""
+
+    type: str | None = Field(default=None)
+    """Optional 'type' field for span."""
+
+    name: str
+    """Span name."""
+
+
+class SpanEndEvent(BaseEvent):
+    """Mark the end of a transcript span."""
+
+    event: Literal["span_end"] = Field(default="span_end")
+    """Event type."""
+
+    id: str
+    """Unique identifier for span."""
+
+
 class StepEvent(BaseEvent):
     """Step within current sample or subtask."""
 
@@ -431,7 +488,13 @@ class SubtaskEvent(BaseEvent):
     """Subtask function result."""
 
     events: list["Event"] = Field(default_factory=list)
-    """Transcript of events for subtask."""
+    """Transcript of events for subtask.
+
+    Note that events are no longer recorded separately within
+    subtasks but rather all events are recorded in the main
+    transcript. This field is deprecated and here for backwards
+    compatibility with transcripts that have sub-events.
+    """
 
     completed: datetime | None = Field(default=None)
     """Time that subtask completed (see `timestamp` for started)"""
@@ -461,6 +524,8 @@ Event: TypeAlias = Union[
     | ErrorEvent
     | LoggerEvent
     | InfoEvent
+    | SpanBeginEvent
+    | SpanEndEvent
     | StepEvent
     | SubtaskEvent,
 ]
@@ -474,8 +539,7 @@ class Transcript:
 
     _event_logger: Callable[[Event], None] | None
 
-    def __init__(self, name: str = "") -> None:
-        self.name = name
+    def __init__(self) -> None:
         self._event_logger = None
         self._events: list[Event] = []
 
@@ -492,19 +556,20 @@ class Transcript:
     def step(self, name: str, type: str | None = None) -> Iterator[None]:
         """Context manager for recording StepEvent.
 
+        The `step()` context manager is deprecated and will be removed in a future version.
+        Please use the `span()` context manager instead.
+
         Args:
             name (str): Step name.
             type (str | None): Optional step type.
         """
-        # step event
-        self._event(StepEvent(action="begin", name=name, type=type))
-
-        # run the step (tracking state/store changes)
-        with track_state_changes(type), track_store_changes():
-            yield
-
-        # end step event
-        self._event(StepEvent(action="end", name=name, type=type))
+        warn_once(
+            logger,
+            "The `transcript().step()` context manager is deprecated and will "
+            + "be removed in a future version. Please replace the call to step() "
+            + "with a call to span().",
+        )
+        yield
 
     @property
     def events(self) -> Sequence[Event]:
@@ -545,23 +610,6 @@ def track_store_changes() -> Iterator[None]:
         transcript()._event(StoreEvent(changes=changes))
 
 
-@contextlib.contextmanager
-def track_state_changes(type: str | None = None) -> Iterator[None]:
-    # we only want to track for step() inside the the sample
-    # (solver level tracking is handled already and there are
-    # no state changes in subtasks)
-    if transcript().name == SAMPLE_SUBTASK and type != "solver":
-        before = state_jsonable()
-        yield
-        after = state_jsonable()
-
-        changes = json_changes(before, after)
-        if changes:
-            transcript()._event(StateEvent(changes=changes))
-    else:
-        yield
-
-
 def init_transcript(transcript: Transcript) -> None:
     _transcript.set(transcript)