inspect-ai 0.3.62__py3-none-any.whl → 0.3.64__py3-none-any.whl

inspect_ai/log/_log.py

@@ -4,11 +4,17 @@ import sys
 import traceback
 from logging import getLogger
 from types import TracebackType
-from typing import Any, Literal, Type
+from typing import Any, Literal, Type, TypedDict
 
 import click
 import tenacity
-from pydantic import BaseModel, ConfigDict, Field, PrivateAttr, model_validator
+from pydantic import (
+    BaseModel,
+    ConfigDict,
+    Field,
+    PrivateAttr,
+    model_validator,
+)
 from rich.console import Console, RenderableType
 from rich.traceback import Traceback
 
@@ -17,12 +23,7 @@ from inspect_ai._util.error import EvalError, exception_message
 from inspect_ai._util.logger import warn_once
 from inspect_ai.approval._policy import ApprovalPolicyConfig
 from inspect_ai.dataset._dataset import MT, metadata_as
-from inspect_ai.model import (
-    ChatMessage,
-    GenerateConfig,
-    ModelOutput,
-    ModelUsage,
-)
+from inspect_ai.model import ChatMessage, GenerateConfig, ModelOutput, ModelUsage
 from inspect_ai.scorer import Score
 from inspect_ai.util._sandbox.environment import SandboxEnvironmentSpec
 from inspect_ai.util._store import Store
@@ -35,7 +36,31 @@ logger = getLogger(__name__)
 SCORER_PLACEHOLDER = "88F74D2C"
 
 
+class EvalConfigDefaults(TypedDict):
+    epochs: int
+    epochs_reducer: list[str]
+    fail_on_error: bool
+    sandbox_cleanup: bool
+    log_samples: bool
+    log_images: bool
+    score_display: bool
+
+
+def eval_config_defaults() -> EvalConfigDefaults:
+    return {
+        "epochs": 1,
+        "epochs_reducer": ["mean"],
+        "fail_on_error": True,
+        "sandbox_cleanup": True,
+        "log_samples": True,
+        "log_images": True,
+        "score_display": True,
+    }
+
+
 class EvalConfig(BaseModel):
+    """Configuration used for evaluation."""
+
     limit: int | tuple[int, int] | None = Field(default=None)
     """Sample limit (number of samples or range of samples)."""
 
@@ -114,6 +139,8 @@ class EvalConfig(BaseModel):
 
 
 class EvalSampleLimit(BaseModel):
+    """Limit encontered by sample."""
+
     type: Literal["context", "time", "message", "token", "operator", "custom"]
     """The type of limit"""
 
@@ -122,6 +149,8 @@ class EvalSampleLimit(BaseModel):
 
 
 class EvalSample(BaseModel):
+    """Sample from evaluation task."""
+
     id: int | str
     """Unique id for sample."""
 
@@ -196,7 +225,7 @@ class EvalSample(BaseModel):
     """Attachments referenced from messages and events.
 
     Resolve attachments for a sample (replacing attachment://* references with
-    attachment content) with the resolve_sample_attachments() function.
+    attachment content) by passing `resolve_attachments=True` to log reading functions.
     """
 
     limit: EvalSampleLimit | None = Field(default=None)
@@ -267,6 +296,8 @@ class EvalEvents(BaseModel):
 
 
 class EvalPlanStep(BaseModel):
+    """Solver step."""
+
     solver: str
     """Name of solver."""
 
@@ -275,6 +306,8 @@ class EvalPlanStep(BaseModel):
 
 
 class EvalPlan(BaseModel):
+    """Plan (solvers) used in evaluation."""
+
     name: str = Field(default="plan")
     """Plan name."""
 
@@ -289,20 +322,24 @@ class EvalPlan(BaseModel):
 
 
 class EvalMetric(BaseModel):
+    """Metric for evaluation score."""
+
     name: str
     """Metric name."""
 
     value: int | float
     """Metric value."""
 
-    options: dict[str, Any] = Field(default_factory=dict)
-    """Options specified when creating metric."""
+    params: dict[str, Any] = Field(default_factory=dict)
+    """Params specified when creating metric."""
 
     metadata: dict[str, Any] | None = Field(default=None)
     """Additional metadata associated with metric."""
 
 
 class EvalScore(BaseModel):
+    """Score for evaluation task."""
+
     name: str
     """Score name."""
 
@@ -323,10 +360,15 @@ class EvalScore(BaseModel):
 
 
 class EvalSampleScore(Score):
+    """Score and sample_id scored."""
+
     sample_id: str | int | None = Field(default=None)
+    """Sample ID."""
 
 
 class EvalSampleReductions(BaseModel):
+    """Score reductions."""
+
     scorer: str
     """Name the of scorer"""
 
@@ -338,6 +380,8 @@ class EvalSampleReductions(BaseModel):
 
 
 class EvalResults(BaseModel):
+    """Scoring results from evaluation."""
+
     total_samples: int = Field(default=0)
     """Total samples in eval (dataset samples * epochs)"""
 
@@ -404,6 +448,8 @@ class EvalResults(BaseModel):
             if "metrics" in values:
                 metrics = values["metrics"]
                 del values["metrics"]
+            else:
+                metrics = None
             # Convert the scorer to the new schema
             score = values["scorer"]
             if metrics:
@@ -418,6 +464,8 @@ class EvalResults(BaseModel):
 
 
 class EvalDataset(BaseModel):
+    """Dataset used for evaluation."""
+
     name: str | None = Field(default=None)
     """Dataset name."""
 
@@ -434,7 +482,33 @@ class EvalDataset(BaseModel):
     """Was the dataset shuffled after reading."""
 
 
+class EvalMetricDefinition(BaseModel):
+    name: str
+    """Metric name"""
+
+    options: dict[str, Any] | None = Field(default=None)
+
+
+class EvalScorer(BaseModel):
+    name: str
+    """Scorer name"""
+
+    options: dict[str, Any] | None = Field(default=None)
+    """Scorer arguments"""
+
+    metrics: (
+        list[EvalMetricDefinition | dict[str, list[EvalMetricDefinition]]]
+        | dict[str, list[EvalMetricDefinition]]
+        | None
+    ) = Field(default=None)
+
+    metadata: dict[str, Any] | None = Field(default=None)
+    """Scorer metadata"""
+
+
 class EvalRevision(BaseModel):
+    """Git revision for evaluation."""
+
     type: Literal["git"]
     """Type of revision (currently only "git")"""
 
@@ -446,6 +520,8 @@ class EvalRevision(BaseModel):
 
 
 class EvalSpec(BaseModel):
+    """Eval target and configuration."""
+
     run_id: str = Field(default_factory=str)
     """Unique run id"""
 
@@ -506,6 +582,14 @@ class EvalSpec(BaseModel):
     metadata: dict[str, Any] | None = Field(default=None)
     """Additional eval metadata."""
 
+    scorers: list[EvalScorer] | None = Field(default=None)
+    """Scorers and args for this eval"""
+
+    metrics: (
+        list[EvalMetricDefinition] | dict[str, list[EvalMetricDefinition]] | None
+    ) = Field(default=None)
+    """metrics and args for this eval"""
+
     # allow field model_args
     model_config = ConfigDict(protected_namespaces=())
 
@@ -549,6 +633,8 @@ def rich_traceback(
 
 
 class EvalStats(BaseModel):
+    """Timing and usage statistics."""
+
     started_at: str = Field(default_factory=str)
     """Evaluation start time."""
 
@@ -563,6 +649,8 @@ class EvalStats(BaseModel):
 
 
 class EvalLog(BaseModel):
+    """Evaluation log."""
+
     # WARNING: The order of these fields is important for the log file format.
     # Do not change the order of these fields without incrementing the version number,
     # updating the log file read/write functionality (such as read_eval_log),
@@ -578,13 +666,13 @@ class EvalLog(BaseModel):
     eval: EvalSpec
     """Eval identity and configuration."""
 
-    plan: EvalPlan = Field(default=EvalPlan())
+    plan: EvalPlan = Field(default_factory=EvalPlan)
     """Eval plan (solvers and config)"""
 
     results: EvalResults | None = None
     """Eval results (scores and metrics)."""
 
-    stats: EvalStats = Field(default=EvalStats())
+    stats: EvalStats = Field(default_factory=EvalStats)
     """Eval stats (runtime, model usage)"""
 
     error: EvalError | None = Field(default=None)

inspect_ai/log/_message.py

@@ -5,12 +5,14 @@ from typing import Any, Literal, Type, cast
 from pydantic import BaseModel, Field, model_validator
 
 LoggingLevel = Literal[
-    "debug", "http", "sandbox", "info", "warning", "error", "critical"
+    "debug", "trace", "http", "sandbox", "info", "warning", "error", "critical"
 ]
 """Logging level."""
 
 
 class LoggingMessage(BaseModel):
+    """Message written to Python log."""
+
     name: str | None = Field(default=None)
     """Logger name (e.g. 'httpx')"""
 
@@ -33,7 +35,7 @@ class LoggingMessage(BaseModel):
     """Logged from line number."""
 
     @staticmethod
-    def from_log_record(record: LogRecord) -> "LoggingMessage":
+    def _from_log_record(record: LogRecord) -> "LoggingMessage":
         """Create a LoggingMesssage from a LogRecord.
 
         Args:

inspect_ai/log/_recorders/file.py

@@ -28,6 +28,10 @@ class FileRecorder(Recorder):
     def is_local(self) -> bool:
         return self.fs.is_local()
 
+    @override
+    def is_writeable(self) -> bool:
+        return self.fs.is_writeable(self.log_dir)
+
     @override
     @classmethod
     async def read_log_sample(

inspect_ai/log/_recorders/json.py

@@ -9,12 +9,7 @@ from typing_extensions import override
 
 from inspect_ai._util.constants import LOG_SCHEMA_VERSION
 from inspect_ai._util.error import EvalError
-from inspect_ai._util.file import (
-    absolute_file_path,
-    async_fileystem,
-    file,
-    filesystem,
-)
+from inspect_ai._util.file import absolute_file_path, async_fileystem, file, filesystem
 from inspect_ai._util.trace import trace_action
 
 from .._log import (
@@ -236,12 +231,13 @@ def _read_header_streaming(log_file: str) -> EvalLog:
         f.seek(0)
 
         # Parse the log file, stopping before parsing samples
+        status: Literal["started", "success", "cancelled", "error"] | None = None
         for k, v in ijson.kvitems(f, ""):
             if k == "status":
                 assert v in get_args(
                     Literal["started", "success", "cancelled", "error"]
                 )
-                status: Literal["started", "success", "cancelled", "error"] = v
+                status = v
             if k == "eval":
                 eval = EvalSpec(**v)
             elif k == "plan":
@@ -257,6 +253,8 @@ def _read_header_streaming(log_file: str) -> EvalLog:
                 error = EvalError(**v)
                 break
 
+    assert status, "Must encounter a 'status'"
+
     return EvalLog(
         eval=eval,
         plan=plan,

inspect_ai/log/_recorders/recorder.py

@@ -21,6 +21,9 @@ class Recorder(abc.ABC):
     @abc.abstractmethod
     def default_log_buffer(self) -> int: ...
 
+    @abc.abstractmethod
+    def is_writeable(self) -> bool: ...
+
     @abc.abstractmethod
     async def log_init(self, eval: EvalSpec, location: str | None = None) -> str: ...
 

inspect_ai/log/_transcript.py

@@ -167,7 +167,7 @@ class ToolEvent(BaseEvent):
     events: list["Event"] = Field(default_factory=list)
     """Transcript of events for tool."""
 
-    def set_result(
+    def _set_result(
         self,
         result: ToolResult,
         truncated: tuple[int, int] | None,
@@ -182,11 +182,11 @@ class ToolEvent(BaseEvent):
 
     # mechanism for operator to cancel the tool call
 
-    def set_task(self, task: asyncio.Task[Any]) -> None:
+    def _set_task(self, task: asyncio.Task[Any]) -> None:
         """Set the tool task (for possible cancellation)"""
         self._task = task
 
-    def cancel(self) -> None:
+    def _cancel(self) -> None:
         """Cancel the tool task."""
         if self._task:
             self._cancelled = True
@@ -264,6 +264,9 @@ class InfoEvent(BaseEvent):
     event: Literal["info"] = Field(default="info")
     """Event type."""
 
+    source: str | None = Field(default=None)
+    """Optional source for info event."""
+
     data: JsonValue
     """Data provided with event."""
 
@@ -279,17 +282,24 @@ class ErrorEvent(BaseEvent):
 
 
 class ScoreEvent(BaseEvent):
-    """Event with sample score."""
+    """Event with score.
+
+    Can be the final score for a `Sample`, or can be an intermediate score
+    resulting from a call to `score`.
+    """
 
     event: Literal["score"] = Field(default="score")
     """Event type."""
 
     score: Score
-    """Sample score."""
+    """Score value."""
 
     target: str | list[str] | None = Field(default=None)
     """"Sample target."""
 
+    intermediate: bool = Field(default=False)
+    """Was this an intermediate scoring?"""
+
 
 class StepEvent(BaseEvent):
     """Step within current sample or subtask."""
@@ -355,13 +365,14 @@ class Transcript:
         self.name = name
         self._events: list[Event] = []
 
-    def info(self, data: JsonValue) -> None:
+    def info(self, data: JsonValue, *, source: str | None = None) -> None:
         """Add an `InfoEvent` to the transcript.
 
         Args:
-           data (JsonValue): Data associated with the event.
+           data: Data associated with the event.
+           source: Optional event source.
         """
-        self._event(InfoEvent(data=data))
+        self._event(InfoEvent(source=source, data=data))
 
     @contextlib.contextmanager
     def step(self, name: str, type: str | None = None) -> Iterator[None]:

inspect_ai/model/__init__.py

@@ -21,6 +21,7 @@ from ._call_tools import call_tools
 from ._chat_message import (
     ChatMessage,
     ChatMessageAssistant,
+    ChatMessageBase,
     ChatMessageSystem,
     ChatMessageTool,
     ChatMessageUser,
@@ -54,6 +55,7 @@ __all__ = [
     "ContentVideo",
     "Content",
     "ChatMessage",
+    "ChatMessageBase",
     "ChatMessageSystem",
     "ChatMessageUser",
     "ChatMessageAssistant",

inspect_ai/model/_cache.py

@@ -58,22 +58,23 @@ def _parse_expiry(period: str) -> int:
 class CachePolicy:
     """The `CachePolicy` is used to define various criteria that impact how model calls are cached.
 
-    Attributes:
-        expiry(str | None): Default "24h". The expiry time for the cache entry.
-          This is a string of the format "12h" for 12 hours or "1W" for a week,
-          etc. This is how long we will keep the cache entry, if we access it
-          after this point we'll clear it. Setting to `None` will cache
-          indefinitely.
-        per_epoch(bool): Default True. By default we cache responses separately
-          for different epochs. The general use case is that if there are
-          multiple epochs, we should cache each response separately because
-          scorers will aggregate across epochs. However, sometimes a response
-          can be cached regardless of epoch if the call being made isn't under
-          test as part of the evaluation. If False, this option allows you to
-          bypass that and cache independently of the epoch.
-        scopes(dict[str, str]): A dictionary of additional metadata that should
-          be included in the cache key. This allows for more fine-grained
-          control over the cache key generation.
+    `expiry`: Default "24h". The expiry time for the cache entry.
+    This is a string of the format "12h" for 12 hours or "1W" for a week,
+    etc. This is how long we will keep the cache entry, if we access it
+    after this point we'll clear it. Setting to `None` will cache
+    indefinitely.
+
+    `per_epoch`: Default True. By default we cache responses separately
+    for different epochs. The general use case is that if there are
+    multiple epochs, we should cache each response separately because
+    scorers will aggregate across epochs. However, sometimes a response
+    can be cached regardless of epoch if the call being made isn't under
+    test as part of the evaluation. If False, this option allows you to
+    bypass that and cache independently of the epoch.
+
+    `scopes`: A dictionary of additional metadata that should
+    be included in the cache key. This allows for more fine-grained
+    control over the cache key generation.
     """
 
     def __init__(
@@ -82,6 +83,14 @@ class CachePolicy:
         per_epoch: bool = True,
         scopes: dict[str, str] = {},
     ) -> None:
+        """Create a CachePolicy.
+
+        Args:
+           expiry: Expiry.
+           per_epoch: Per epoch
+           scopes: Scopes
+
+        """
         self.per_epoch = per_epoch
         self.scopes = scopes
 
@@ -236,7 +245,11 @@ def cache_fetch(entry: CacheEntry) -> ModelOutput | None:
 
 
 def cache_clear(model: str = "") -> bool:
-    """Clear the cache directory."""
+    """Clear the cache directory.
+
+    Args:
+       model: Model to clear cache for.
+    """
     try:
         path = cache_path(model)
 
@@ -252,6 +265,11 @@ def cache_clear(model: str = "") -> bool:
 
 
 def cache_path(model: str = "") -> Path:
+    """Path to cache directory.
+
+    Args:
+       model: Path to cache directory for specific model.
+    """
     env_cache_dir = os.environ.get("INSPECT_CACHE_DIR", None)
     if env_cache_dir:
         generate_cache = Path(env_cache_dir) / "generate"
@@ -320,9 +338,9 @@ def cache_size(
     will be calculated.
 
     Args:
-        subdirs(list[str]): List of folders to filter by, which are generally
+        subdirs: List of folders to filter by, which are generally
             model names. Empty directories will be ignored.
-        files(list[str]): List of files to filter by explicitly. Note that
+        files: List of files to filter by explicitly. Note that
             return value group these up by their parent directory
 
     Returns:
@@ -344,7 +362,7 @@ def cache_list_expired(filter_by: list[str] = []) -> list[Path]:
     """Returns a list of all the cached files that have passed their expiry time.
 
     Args:
-        filter_by(list[str]): Default []. List of model names to filter by. If
+        filter_by: Default []. List of model names to filter by. If
             an empty list, this will search the entire cache.
     """
     expired_cache_entries = []
@@ -384,7 +402,7 @@ def cache_prune(files: list[Path] = []) -> None:
     """Delete all expired cache entries.
 
     Args:
-        files(list[Path]): Default []. List of files to prune. If empty, this
+        files: List of files to prune. If empty, this
             will search the entire cache.
     """
     if not files:

inspect_ai/model/_call_tools.py

@@ -133,7 +133,8 @@ async def call_tools(
             ):
                 content: str | list[Content] = [result]
             elif isinstance(result, list) and (
-                isinstance(
+                len(result) == 0
+                or isinstance(
                     result[0], ContentText | ContentImage | ContentAudio | ContentVideo
                 )
             ):
@@ -186,7 +187,7 @@ async def call_tools(
                 view=call.view,
                 pending=True,
             )
-            event.set_task(task)
+            event._set_task(task)
             transcript()._event(event)
 
             # execute the tool call. if the operator cancelled the
@@ -226,7 +227,7 @@ async def call_tools(
             conversation_tool_mesage(tool_message)
 
             # update the event with the results
-            event.set_result(
+            event._set_result(
                 result=result_event.result,
                 truncated=result_event.truncated,
                 error=result_event.error,

inspect_ai/model/_chat_message.py

@@ -13,8 +13,13 @@ logger = getLogger(__name__)
 
 
 class ChatMessageBase(BaseModel):
+    """Base class for chat messages."""
+
+    role: Literal["system", "user", "assistant", "tool"]
+    """Conversation role"""
+
     content: str | list[Content]
-    """Content (simple string or list of string|image content)"""
+    """Content (simple string or list of content objects)"""
 
     source: Literal["input", "generate"] | None = Field(default=None)
     """Source of message."""
@@ -31,9 +36,6 @@ class ChatMessageBase(BaseModel):
         property returns either the plain str content, or if the
         content is a list of text and images, the text items
         concatenated together (separated by newline)
-
-        Returns: Text content of `ChatMessage` If this message does
-          not have text content then "" is returned.
         """
         if isinstance(self.content, str):
             return self.content
@@ -66,11 +68,15 @@ class ChatMessageBase(BaseModel):
 
 
 class ChatMessageSystem(ChatMessageBase):
+    """System chat message."""
+
     role: Literal["system"] = Field(default="system")
     """Conversation role."""
 
 
 class ChatMessageUser(ChatMessageBase):
+    """User chat message."""
+
     role: Literal["user"] = Field(default="user")
     """Conversation role."""
 
@@ -79,6 +85,8 @@ class ChatMessageUser(ChatMessageBase):
 
 
 class ChatMessageAssistant(ChatMessageBase):
+    """Assistant chat message."""
+
     role: Literal["assistant"] = Field(default="assistant")
     """Conversation role."""
 
@@ -112,6 +120,8 @@ class ChatMessageAssistant(ChatMessageBase):
 
 
 class ChatMessageTool(ChatMessageBase):
+    """Tool chat message."""
+
     role: Literal["tool"] = Field(default="tool")
     """Conversation role."""
 

inspect_ai/model/_generate_config.py

@@ -80,7 +80,7 @@ class GenerateConfigArgs(TypedDict, total=False):
 
 
 class GenerateConfig(BaseModel):
-    """Base class for model generation configs."""
+    """Model generation options."""
 
     max_retries: int | None = Field(default=None)
     """Maximum number of times to retry request (defaults to 5)."""