inspect-ai 0.3.95__py3-none-any.whl → 0.3.97__py3-none-any.whl

inspect_ai/analysis/beta/_dataframe/events/columns.py

@@ -1,3 +1,4 @@
+from datetime import datetime
 from typing import Any, Callable, Mapping, Type
 
 from jsonpath_ng import JSONPath  # type: ignore
@@ -7,6 +8,12 @@ from typing_extensions import override
 from inspect_ai.log._transcript import Event
 
 from ..columns import Column, ColumnType
+from .extract import (
+    completion_as_str,
+    model_event_input_as_str,
+    tool_choice_as_str,
+    tool_view_as_str,
+)
 
 
 class EventColumn(Column):
@@ -35,3 +42,46 @@ class EventColumn(Column):
     @override
     def path_schema(self) -> Mapping[str, Any] | None:
         return None
+
+
+EventInfo: list[Column] = [
+    EventColumn("event", path="event"),
+    EventColumn("span_id", path="span_id"),
+]
+"""Event basic information columns."""
+
+EventTiming: list[Column] = [
+    EventColumn("timestamp", path="timestamp", type=datetime),
+    EventColumn("completed", path="completed", type=datetime),
+    EventColumn("working_start", path="working_start"),
+    EventColumn("working_time", path="working_time"),
+]
+"""Event timing columns."""
+
+ModelEventColumns: list[Column] = [
+    EventColumn("model_event_model", path="model"),
+    EventColumn("model_event_role", path="role"),
+    EventColumn("model_event_input", path=model_event_input_as_str),
+    EventColumn("model_event_tools", path="tools"),
+    EventColumn("model_event_tool_choice", path=tool_choice_as_str),
+    EventColumn("model_event_config", path="config"),
+    EventColumn("model_event_usage", path="output.usage"),
+    EventColumn("model_event_time", path="output.time"),
+    EventColumn("model_event_completion", path=completion_as_str),
+    EventColumn("model_event_retries", path="retries"),
+    EventColumn("model_event_error", path="error"),
+    EventColumn("model_event_cache", path="cache"),
+    EventColumn("model_event_call", path="call"),
+]
+"""Model event columns."""
+
+ToolEventColumns: list[Column] = [
+    EventColumn("tool_event_function", path="function"),
+    EventColumn("tool_event_arguments", path="arguments"),
+    EventColumn("tool_event_view", path=tool_view_as_str),
+    EventColumn("tool_event_result", path="result"),
+    EventColumn("tool_event_truncated", path="truncated"),
+    EventColumn("tool_event_error_type", path="error.type"),
+    EventColumn("tool_event_error_message", path="error.message"),
+]
+"""Tool event columns."""

inspect_ai/analysis/beta/_dataframe/events/extract.py

@@ -0,0 +1,26 @@
+from inspect_ai.log._transcript import ModelEvent, ToolEvent
+
+from ..extract import messages_as_str
+
+
+def model_event_input_as_str(event: ModelEvent) -> str:
+    return messages_as_str(event.input)
+
+
+def tool_choice_as_str(event: ModelEvent) -> str:
+    if isinstance(event.tool_choice, str):
+        return event.tool_choice
+    else:
+        return event.tool_choice.name
+
+
+def completion_as_str(event: ModelEvent) -> str:
+    return event.output.completion
+
+
+def tool_view_as_str(event: ToolEvent) -> str | None:
+    if event.view is not None:
+        title = f"{event.view.title}\n\n" if event.view.title is not None else ""
+        return f"{title}{event.view.content}"
+    else:
+        return None

inspect_ai/analysis/beta/_dataframe/events/table.py

@@ -1,14 +1,88 @@
 from __future__ import annotations
 
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Callable, Literal, Sequence, TypeAlias
+
+from inspect_ai.analysis.beta._dataframe.events.columns import EventInfo
+from inspect_ai.log._file import list_eval_logs
+from inspect_ai.log._transcript import Event
 
 if TYPE_CHECKING:
     import pandas as pd
 
+from typing_extensions import overload
+
+from ..columns import Column, ColumnError
+from ..samples.table import EventsDetail, _read_samples_df
 from ..util import LogPaths, verify_prerequisites
 
+EventFilter: TypeAlias = Callable[[Event], bool]
+"""Filter for `events_df()` rows."""
+
+
+@overload
+def events_df(
+    logs: LogPaths = list_eval_logs(),
+    columns: Sequence[Column] = EventInfo,
+    filter: EventFilter | None = None,
+    strict: Literal[True] = True,
+    parallel: bool | int = False,
+    quiet: bool = False,
+) -> "pd.DataFrame": ...
+
 
-def events_df(logs: LogPaths, recursive: bool = True) -> "pd.DataFrame":
+@overload
+def events_df(
+    logs: LogPaths = list_eval_logs(),
+    columns: Sequence[Column] = EventInfo,
+    filter: EventFilter | None = None,
+    strict: Literal[False] = False,
+    parallel: bool | int = False,
+    quiet: bool = False,
+) -> tuple["pd.DataFrame", list[ColumnError]]: ...
+
+
+def events_df(
+    logs: LogPaths = list_eval_logs(),
+    columns: Sequence[Column] = EventInfo,
+    filter: EventFilter | None = None,
+    strict: bool = True,
+    parallel: bool | int = False,
+    quiet: bool = False,
+) -> "pd.DataFrame" | tuple["pd.DataFrame", list[ColumnError]]:
+    """Read a dataframe containing events from a set of evals.
+
+    Args:
+       logs: One or more paths to log files or log directories.
+          Defaults to the contents of the currently active log directory
+          (e.g. ./logs or INSPECT_LOG_DIR).
+       columns: Specification for what columns to read from log files.
+       filter: Callable that filters event types.
+       strict: Raise import errors immediately. Defaults to `True`.
+          If `False` then a tuple of `DataFrame` and errors is returned.
+       parallel: If `True`, use `ProcessPoolExecutor` to read logs in parallel
+          (with workers based on `mp.cpu_count()`, capped at 8). If `int`, read
+          in parallel with the specified number of workers. If `False` (the default)
+          do not read in parallel.
+       quiet: If `True` do not print any output or progress (defaults to `False`).
+
+    Returns:
+       For `strict`, a Pandas `DataFrame` with information for the specified logs.
+       For `strict=False`, a tuple of Pandas `DataFrame` and a dictionary of errors
+       encountered (by log file) during import.
+    """
     verify_prerequisites()
 
-    raise NotImplementedError("events_df has not been implemented yet.")
+    # resolve filter/detail
+    if callable(filter):
+        detail = EventsDetail(filter=filter)
+    else:
+        detail = EventsDetail()
+
+    return _read_samples_df(
+        logs=logs,
+        columns=columns,
+        strict=strict,
+        detail=detail,
+        progress=not quiet,
+        parallel=parallel,
+    )

inspect_ai/analysis/beta/_dataframe/extract.py

@@ -5,11 +5,16 @@ from typing import Any, cast
 import shortuuid
 from pydantic import BaseModel, JsonValue
 
-from inspect_ai._util.json import jsonable_python
+from inspect_ai.model._chat_message import (
+    ChatMessage,
+    ChatMessageAssistant,
+    ChatMessageTool,
+    ChatMessageUser,
+)
 
 
 def model_to_record(model: BaseModel) -> dict[str, JsonValue]:
-    return cast(dict[str, JsonValue], jsonable_python(model))
+    return cast(dict[str, JsonValue], model.model_dump(mode="json", exclude_none=True))
 
 
 def list_as_str(x: JsonValue) -> str:
@@ -21,34 +26,48 @@ def score_values(x: JsonValue) -> dict[str, JsonValue]:
     return {k: v["value"] for k, v in scores.items()}
 
 
-def input_as_str(x: JsonValue) -> str:
-    if isinstance(x, str):
-        return x
-    else:
-        return messages_as_str(x)
+def auto_id(base: str, index: str) -> str:
+    seed = f"{base}_{index}"
+    hash_bytes = hashlib.md5(seed.encode("utf-8")).digest()
+    long_uuid = uuid.UUID(bytes=hash_bytes)
+    return shortuuid.encode(long_uuid)
 
 
-def messages_as_str(x: JsonValue) -> str:
-    if isinstance(x, list):
-        messages = cast(list[dict[str, Any]], x)
-        return "\n\n".join([message_as_str(message) for message in messages])
-    else:
-        raise ValueError(f"Unexpected type for messages: {type(x)}")
+def messages_as_str(messages: str | list[ChatMessage]) -> str:
+    if isinstance(messages, str):
+        messages = [ChatMessageUser(content=messages)]
+    return "\n\n".join([message_as_str(message) for message in messages])
 
 
-def message_as_str(message: dict[str, Any]) -> str:
-    return f"{message['role']}:\n{content_as_str(message['content'])}"
+def message_as_str(message: ChatMessage) -> str:
+    transcript: list[str] = []
+    role = message.role
+    content = message.text.strip() if message.text else ""
 
+    # assistant messages with tool calls
+    if isinstance(message, ChatMessageAssistant) and message.tool_calls is not None:
+        entry = f"{role}:\n{content}\n"
 
-def content_as_str(content: str | list[dict[str, Any]]) -> str:
-    if isinstance(content, str):
-        return content
-    else:
-        return "\n".join([c["text"] if c["type"] == "text" else "" for c in content])
+        for tool in message.tool_calls:
+            func_name = tool.function
+            args = tool.arguments
 
+            if isinstance(args, dict):
+                args_text = "\n".join(f"{k}: {v}" for k, v in args.items())
+                entry += f"\nTool Call: {func_name}\nArguments:\n{args_text}"
+            else:
+                entry += f"\nTool Call: {func_name}\nArguments: {args}"
 
-def auto_id(base: str, index: str) -> str:
-    seed = f"{base}_{index}"
-    hash_bytes = hashlib.md5(seed.encode("utf-8")).digest()
-    long_uuid = uuid.UUID(bytes=hash_bytes)
-    return shortuuid.encode(long_uuid)
+        transcript.append(entry)
+
+    # tool responses with errors
+    elif isinstance(message, ChatMessageTool) and message.error is not None:
+        func_name = message.function or "unknown"
+        entry = f"{role}:\n{content}\n\nError in tool call '{func_name}':\n{message.error.message}\n"
+        transcript.append(entry)
+
+    # normal messages
+    else:
+        transcript.append(f"{role}:\n{content}\n")
+
+    return "\n".join(transcript)

inspect_ai/analysis/beta/_dataframe/messages/columns.py

@@ -43,8 +43,8 @@ class MessageColumn(Column):
 
 MessageContent: list[Column] = [
     MessageColumn("role", path="role", required=True),
-    MessageColumn("content", path=message_text),
     MessageColumn("source", path="source"),
+    MessageColumn("content", path=message_text),
 ]
 """Message content columns."""
 

inspect_ai/analysis/beta/_dataframe/messages/table.py

@@ -1,7 +1,8 @@
 from __future__ import annotations
 
-from typing import TYPE_CHECKING, Callable, Literal, TypeAlias
+from typing import TYPE_CHECKING, Callable, Literal, Sequence, TypeAlias
 
+from inspect_ai.log._file import list_eval_logs
 from inspect_ai.model._chat_message import ChatMessage
 
 if TYPE_CHECKING:
@@ -9,58 +10,60 @@ if TYPE_CHECKING:
 
 from typing_extensions import overload
 
-from ..columns import Column, ColumnErrors
+from ..columns import Column, ColumnError
 from ..samples.table import MessagesDetail, _read_samples_df
 from ..util import LogPaths, verify_prerequisites
 from .columns import MessageColumns
 
-MessageFilter: TypeAlias = (
-    list[Literal["system", "user", "assistant", "tool"]] | Callable[[ChatMessage], bool]
-)
+MessageFilter: TypeAlias = Callable[[ChatMessage], bool]
 """Filter for `messages_df()` rows."""
 
 
 @overload
 def messages_df(
-    logs: LogPaths,
-    columns: list[Column] = MessageColumns,
+    logs: LogPaths = list_eval_logs(),
+    columns: Sequence[Column] = MessageColumns,
     filter: MessageFilter | None = None,
-    recursive: bool = True,
-    reverse: bool = False,
     strict: Literal[True] = True,
+    parallel: bool | int = False,
+    quiet: bool = False,
 ) -> "pd.DataFrame": ...
 
 
 @overload
 def messages_df(
-    logs: LogPaths,
-    columns: list[Column] = MessageColumns,
+    logs: LogPaths = list_eval_logs(),
+    columns: Sequence[Column] = MessageColumns,
     filter: MessageFilter | None = None,
-    recursive: bool = True,
-    reverse: bool = False,
     strict: Literal[False] = False,
-) -> tuple["pd.DataFrame", ColumnErrors]: ...
+    parallel: bool | int = False,
+    quiet: bool = False,
+) -> tuple["pd.DataFrame", list[ColumnError]]: ...
 
 
 def messages_df(
-    logs: LogPaths,
-    columns: list[Column] = MessageColumns,
+    logs: LogPaths = list_eval_logs(),
+    columns: Sequence[Column] = MessageColumns,
     filter: MessageFilter | None = None,
-    recursive: bool = True,
-    reverse: bool = False,
     strict: bool = True,
-) -> "pd.DataFrame" | tuple["pd.DataFrame", ColumnErrors]:
+    parallel: bool | int = False,
+    quiet: bool = False,
+) -> "pd.DataFrame" | tuple["pd.DataFrame", list[ColumnError]]:
     """Read a dataframe containing messages from a set of evals.
 
     Args:
        logs: One or more paths to log files or log directories.
+          Defaults to the contents of the currently active log directory
+          (e.g. ./logs or INSPECT_LOG_DIR).
        columns: Specification for what columns to read from log files.
-       filter: List of message role types to include or callable that performs the filter.
-       recursive: Include recursive contents of directories (defaults to `True`)
-       reverse: Reverse the order of the dataframe (by default, items
-          are ordered from oldest to newest).
+       filter: Callable that filters messages
        strict: Raise import errors immediately. Defaults to `True`.
           If `False` then a tuple of `DataFrame` and errors is returned.
+       parallel: If `True`, use `ProcessPoolExecutor` to read logs in parallel
+          (with workers based on `mp.cpu_count()`, capped at 8). If `int`, read
+          in parallel with the specified number of workers. If `False` (the default)
+          do not read in parallel.
+       quiet: If `True` do not print any output or progress (defaults to `False`).
 
     Returns:
        For `strict`, a Pandas `DataFrame` with information for the specified logs.
@@ -70,18 +73,16 @@ def messages_df(
     verify_prerequisites()
 
     # resolve filter/detail
-    if filter is None:
-        detail = MessagesDetail(filter=lambda m: True)
-    elif callable(filter):
+    if callable(filter):
         detail = MessagesDetail(filter=filter)
     else:
-        detail = MessagesDetail(filter=lambda m: m.role in filter)
+        detail = MessagesDetail()
 
     return _read_samples_df(
         logs=logs,
         columns=columns,
-        recursive=recursive,
-        reverse=reverse,
         strict=strict,
         detail=detail,
+        parallel=parallel,
+        progress=not quiet,
     )

inspect_ai/analysis/beta/_dataframe/progress.py

@@ -0,0 +1,56 @@
+from contextlib import contextmanager
+from typing import Iterator, Protocol
+
+from rich.progress import (
+    BarColumn,
+    Progress,
+    TaskID,
+    TaskProgressColumn,
+    TextColumn,
+    TimeElapsedColumn,
+)
+
+
+class ImportProgress(Protocol):
+    def update(self) -> None: ...
+    def reset(self, description: str, completed: int, total: int) -> None: ...
+
+
+class NoProgress(ImportProgress):
+    def update(self) -> None:
+        pass
+
+    def reset(self, description: str, completed: int, total: int) -> None:
+        pass
+
+
+class RichImportProgress(ImportProgress):
+    def __init__(self, progress: Progress, task_id: TaskID) -> None:
+        self._progress = progress
+        self._task_id = task_id
+
+    def update(self) -> None:
+        self._progress.update(self._task_id, advance=1)
+
+    def reset(self, description: str, completed: int, total: int) -> None:
+        self._progress.reset(
+            self._task_id, description=description, completed=completed, total=total
+        )
+
+
+@contextmanager
+def no_progress() -> Iterator[ImportProgress]:
+    yield NoProgress()
+
+
+@contextmanager
+def import_progress(description: str, total: float | None) -> Iterator[ImportProgress]:
+    with Progress(
+        TextColumn("[progress.description]{task.description:<18}"),
+        BarColumn(),
+        TaskProgressColumn(),
+        TimeElapsedColumn(),
+        transient=True,
+    ) as progress:
+        task_id = progress.add_task(description, total=total)
+        yield RichImportProgress(progress, task_id)

inspect_ai/analysis/beta/_dataframe/record.py

@@ -1,6 +1,6 @@
 import json
 from datetime import date, datetime, time, timezone
-from typing import Any, Callable, Literal, Type, cast, overload
+from typing import Any, Callable, Literal, Sequence, Type, cast, overload
 
 import yaml
 from jsonpath_ng import JSONPath  # type: ignore
@@ -20,38 +20,41 @@ from .extract import model_to_record
 
 @overload
 def import_record(
+    log: EvalLog,
     record: EvalLog
     | EvalSampleSummary
     | EvalSample
     | ChatMessage
     | Event
     | dict[str, JsonValue],
-    columns: list[Column],
+    columns: Sequence[Column],
     strict: Literal[True] = True,
 ) -> dict[str, ColumnType]: ...
 
 
 @overload
 def import_record(
+    log: EvalLog,
     record: EvalLog
     | EvalSampleSummary
     | EvalSample
     | ChatMessage
     | Event
     | dict[str, JsonValue],
-    columns: list[Column],
+    columns: Sequence[Column],
     strict: Literal[False],
 ) -> tuple[dict[str, ColumnType], list[ColumnError]]: ...
 
 
 def import_record(
+    log: EvalLog,
     record: EvalLog
     | EvalSampleSummary
     | EvalSample
     | ChatMessage
     | Event
     | dict[str, JsonValue],
-    columns: list[Column],
+    columns: Sequence[Column],
     strict: bool = True,
 ) -> dict[str, ColumnType] | tuple[dict[str, ColumnType], list[ColumnError]]:
     # resolve the record BaseModel into a dict (and optionally a summary dict).
@@ -80,7 +83,7 @@ def import_record(
         try:
             result[name] = _resolve_value(value, column.type)
         except ValueError as ex:
-            error = ColumnError(name, path=column.path, message=str(ex))
+            error = ColumnError(name, path=column.path, error=ex, log=log)
             if strict:
                 raise ValueError(str(error))
             else:
@@ -90,10 +93,10 @@ def import_record(
     def field_not_found(
         name: str, path: JSONPath | None, required_type: str | None = None
     ) -> None:
-        message = (
+        ex = ValueError(
             f"field not of type {required_type}" if required_type else "field not found"
         )
-        error = ColumnError(name, path=path, message=f"{message}")
+        error = ColumnError(name, path=path, error=ex, log=log)
         if strict:
             raise ValueError(str(error))
         else:
@@ -157,7 +160,8 @@ def import_record(
             error = ColumnError(
                 column.name,
                 path=str(column.path) if column.path else None,
-                message=str(ex),
+                error=ex,
+                log=log,
             )
             if strict:
                 raise ValueError(str(error))
@@ -190,7 +194,7 @@ def import_record(
         return result, errors
 
 
-def resolve_duplicate_columns(columns: list[Column]) -> list[Column]:
+def resolve_duplicate_columns(columns: Sequence[Column]) -> list[Column]:
     """Remove duplicate columns (with the later columns winning)"""
     seen = set[str]()
     deduped: list[Column] = []

inspect_ai/analysis/beta/_dataframe/samples/columns.py

@@ -7,9 +7,13 @@ from typing_extensions import override
 from inspect_ai.log._log import EvalSample, EvalSampleSummary
 
 from ..columns import Column, ColumnType
-from ..extract import input_as_str, list_as_str, score_values
+from ..extract import list_as_str, score_values
 from ..validate import resolved_schema
-from .extract import sample_messages_as_str, sample_path_requires_full
+from .extract import (
+    sample_input_as_str,
+    sample_messages_as_str,
+    sample_path_requires_full,
+)
 
 
 class SampleColumn(Column):
@@ -54,14 +58,14 @@ class SampleColumn(Column):
 SampleSummary: list[Column] = [
     SampleColumn("id", path="id", required=True, type=str),
     SampleColumn("epoch", path="epoch", required=True),
-    SampleColumn("input", path="input", required=True, value=input_as_str),
+    SampleColumn("input", path=sample_input_as_str, required=True),
     SampleColumn("target", path="target", required=True, value=list_as_str),
     SampleColumn("metadata_*", path="metadata"),
     SampleColumn("score_*", path="scores", value=score_values),
     SampleColumn("model_usage", path="model_usage"),
     SampleColumn("total_time", path="total_time"),
     SampleColumn("working_time", path="total_time"),
-    SampleColumn("error", path="error"),
+    SampleColumn("error", path="error", default=""),
     SampleColumn("limit", path="limit"),
     SampleColumn("retries", path="retries"),
 ]

inspect_ai/analysis/beta/_dataframe/samples/extract.py

@@ -3,45 +3,17 @@ from typing import Callable
 from jsonpath_ng import JSONPath  # type: ignore
 from pydantic import JsonValue
 
-from inspect_ai.analysis.beta._dataframe.extract import auto_id
 from inspect_ai.log._log import EvalSample, EvalSampleSummary
-from inspect_ai.model._chat_message import ChatMessageAssistant, ChatMessageTool
 
+from ..extract import auto_id, messages_as_str
 
-def sample_messages_as_str(sample: EvalSample) -> str:
-    # format each message for the transcript
-    transcript: list[str] = []
-    for msg in sample.messages:
-        role = msg.role
-        content = msg.text.strip() if msg.text else ""
-
-        # assistant messages with tool calls
-        if isinstance(msg, ChatMessageAssistant) and msg.tool_calls is not None:
-            entry = f"{role}:\n{content}\n"
-
-            for tool in msg.tool_calls:
-                func_name = tool.function
-                args = tool.arguments
 
-                if isinstance(args, dict):
-                    args_text = "\n".join(f"{k}: {v}" for k, v in args.items())
-                    entry += f"\nTool Call: {func_name}\nArguments:\n{args_text}"
-                else:
-                    entry += f"\nTool Call: {func_name}\nArguments: {args}"
+def sample_input_as_str(sample: EvalSample) -> str:
+    return messages_as_str(sample.input)
 
-            transcript.append(entry)
 
-        # tool responses with errors
-        elif isinstance(msg, ChatMessageTool) and msg.error is not None:
-            func_name = msg.function or "unknown"
-            entry = f"{role}:\n{content}\n\nError in tool call '{func_name}':\n{msg.error.message}\n"
-            transcript.append(entry)
-
-        # normal messages
-        else:
-            transcript.append(f"{role}:\n{content}\n")
-
-    return "\n".join(transcript)
+def sample_messages_as_str(sample: EvalSample) -> str:
+    return messages_as_str(sample.messages)
 
 
 def sample_path_requires_full(