inspect-ai 0.3.81__py3-none-any.whl → 0.3.83__py3-none-any.whl

inspect_ai/log/_condense.py

@@ -25,6 +25,7 @@ from inspect_ai.model._model_output import ModelOutput
 from ._log import EvalSample
 from ._transcript import (
     Event,
+    InfoEvent,
     ModelEvent,
     SampleInitEvent,
     StateEvent,
@@ -133,6 +134,25 @@ def resolve_sample_attachments(sample: EvalSample) -> EvalSample:
     )
 
 
+def attachments_content_fn(
+    log_images: bool, max_length: int, attachments: dict[str, str]
+) -> Callable[[str], str]:
+    def create_attachment(text: str) -> str:
+        hash = mm3_hash(text)
+        attachments[hash] = text
+        return f"{ATTACHMENT_PROTOCOL}{hash}"
+
+    def content_fn(text: str) -> str:
+        if not log_images and is_data_uri(text):
+            return BASE_64_DATA_REMOVED
+        elif len(text) > max_length:
+            return create_attachment(text)
+        else:
+            return text
+
+    return content_fn
+
+
 def walk_events(events: list[Event], content_fn: Callable[[str], str]) -> list[Event]:
     return [walk_event(event, content_fn) for event in events]
 
@@ -150,6 +170,8 @@ def walk_event(event: Event, content_fn: Callable[[str], str]) -> Event:
         return walk_subtask_event(event, content_fn)
     elif isinstance(event, ToolEvent):
         return walk_tool_event(event, content_fn)
+    elif isinstance(event, InfoEvent):
+        return walk_info_event(event, content_fn)
     else:
         return event
 
@@ -164,6 +186,10 @@ def walk_tool_event(event: ToolEvent, content_fn: Callable[[str], str]) -> ToolE
     return event.model_copy(update=dict(events=walk_events(event.events, content_fn)))
 
 
+def walk_info_event(event: InfoEvent, content_fn: Callable[[str], str]) -> InfoEvent:
+    return event.model_copy(update=dict(data=walk_json_value(event.data, content_fn)))
+
+
 def walk_sample_init_event(
     event: SampleInitEvent, content_fn: Callable[[str], str]
 ) -> SampleInitEvent:

inspect_ai/log/_log.py

@@ -121,6 +121,9 @@ class EvalConfig(BaseModel):
     log_buffer: int | None = Field(default=None)
     """Number of samples to buffer before writing log file."""
 
+    log_shared: int | None = Field(default=None)
+    """Interval (in seconds) for syncing sample events to log directory."""
+
     score_display: bool | None = Field(default=None)
     """Display scoring metrics realtime."""
 
@@ -180,16 +183,16 @@ class EvalSample(BaseModel):
     setup: str | None = Field(default=None)
     """Setup script to run for sample (run within default SandboxEnvironment)."""
 
-    messages: list[ChatMessage]
+    messages: list[ChatMessage] = Field(default_factory=list)
     """Chat conversation history for sample."""
 
-    output: ModelOutput
+    output: ModelOutput = Field(default_factory=ModelOutput)
     """Model output from sample."""
 
     scores: dict[str, Score] | None = Field(default=None)
     """Scores for sample."""
 
-    metadata: dict[str, Any]
+    metadata: dict[str, Any] = Field(default_factory=dict)
     """Additional sample metadata."""
 
     def metadata_as(self, metadata_cls: Type[MT]) -> MT:
@@ -206,14 +209,16 @@ class EvalSample(BaseModel):
     store: dict[str, Any] = Field(default_factory=dict)
     """State at end of sample execution."""
 
-    def store_as(self, model_cls: Type[SMT]) -> SMT:
+    def store_as(self, model_cls: Type[SMT], instance: str | None = None) -> SMT:
         """Pydantic model interface to the store.
 
         Args:
           model_cls: Pydantic model type (must derive from StoreModel)
+          instance: Optional instances name for store (enables multiple instances
+            of a given StoreModel type within a single sample)
 
         Returns:
-          StoreModel: Instance of model_cls bound to sample store data.
+          StoreModel: model_cls bound to sample store data.
         """
         # un-namespace names for creation
         data = {
@@ -223,6 +228,10 @@ class EvalSample(BaseModel):
         # since we are reading from the log provide a fully detached store
         data["store"] = Store()
 
+        # provide instance if specified
+        if instance is not None:
+            data["instance"] = instance
+
         # create the model
         return model_cls.model_validate(data)
 
@@ -563,6 +572,9 @@ class EvalSpec(BaseModel):
     task_file: str | None = Field(default=None)
     """Task source file."""
 
+    task_registry_name: str | None = Field(default=None)
+    """Task registry name."""
+
     task_attribs: dict[str, Any] = Field(default_factory=dict)
     """Attributes of the @task decorator."""
 

inspect_ai/log/_recorders/buffer/__init__.py

@@ -0,0 +1,14 @@
+from .buffer import cleanup_sample_buffers, sample_buffer
+from .database import SampleBufferDatabase
+from .types import AttachmentData, EventData, SampleBuffer, SampleData, Samples
+
+__all__ = [
+    "AttachmentData",
+    "EventData",
+    "SampleData",
+    "Samples",
+    "SampleBuffer",
+    "SampleBufferDatabase",
+    "sample_buffer",
+    "cleanup_sample_buffers",
+]

inspect_ai/log/_recorders/buffer/buffer.py

@@ -0,0 +1,30 @@
+from logging import getLogger
+
+from .database import SampleBufferDatabase, cleanup_sample_buffer_databases
+from .filestore import SampleBufferFilestore, cleanup_sample_buffer_filestores
+from .types import SampleBuffer
+
+logger = getLogger(__name__)
+
+
+def sample_buffer(location: str) -> SampleBuffer:
+    try:
+        return SampleBufferDatabase(location, create=False)
+    except FileNotFoundError:
+        return SampleBufferFilestore(location, create=False)
+
+
+def running_tasks(log_dir: str) -> list[str]:
+    tasks = SampleBufferDatabase.running_tasks(log_dir)
+    if tasks is not None:
+        return tasks
+    else:
+        return SampleBufferFilestore.running_tasks(log_dir) or []
+
+
+def cleanup_sample_buffers(log_dir: str) -> None:
+    try:
+        cleanup_sample_buffer_databases()
+        cleanup_sample_buffer_filestores(log_dir)
+    except Exception as ex:
+        logger.warning(f"Unexpected error cleaning up sample buffers: {ex}")