inspect-ai 0.3.94__py3-none-any.whl → 0.3.95__py3-none-any.whl

inspect_ai/_view/www/src/state/logSlice.ts

@@ -1,4 +1,4 @@
-import { LogState, ScoreFilter, ScoreLabel } from "../app/types";
+import { FilterError, LogState, ScoreLabel } from "../app/types";
 import { EvalSummary, PendingSamples } from "../client/api/types";
 import { kDefaultSort, kLogViewInfoTabId } from "../constants";
 import { createLogger } from "../utils/logger";
@@ -23,7 +23,13 @@ export interface LogSlice {
     setPendingSampleSummaries: (samples: PendingSamples) => void;
 
     // Set filter criteria
-    setFilter: (filter: ScoreFilter) => void;
+    setFilter: (filter: string) => void;
+
+    // Set the filter error
+    setFilterError: (error: FilterError) => void;
+
+    // Clear the filter error
+    clearFilterError: () => void;
 
     // Set epoch filter
     setEpoch: (epoch: string) => void;
@@ -60,7 +66,9 @@ const initialState = {
   loadedLog: undefined,
 
   // Filter state
-  filter: {},
+  filter: "",
+  filterError: undefined,
+
   epoch: "all",
   sort: kDefaultSort,
   score: undefined,
@@ -110,10 +118,19 @@ export const createLogSlice = (
           state.log.pendingSampleSummaries = pendingSampleSummaries;
         }),
 
-      setFilter: (filter: ScoreFilter) =>
+      setFilter: (filter: string) =>
         set((state) => {
           state.log.filter = filter;
         }),
+      setFilterError: (error: FilterError) =>
+        set((state) => {
+          state.log.filterError = error;
+        }),
+      clearFilterError: () => {
+        set((state) => {
+          state.log.filterError = undefined;
+        });
+      },
       setEpoch: (epoch: string) =>
         set((state) => {
           state.log.epoch = epoch;
@@ -132,7 +149,8 @@ export const createLogSlice = (
         }),
       resetFiltering: () =>
         set((state) => {
-          state.log.filter = {};
+          state.log.filter = "";
+          state.log.filterError = undefined;
           state.log.epoch = "all";
           state.log.sort = kDefaultSort;
           state.log.score = undefined;

inspect_ai/_view/www/yarn.lock

@@ -4254,17 +4254,17 @@ react-refresh@^0.17.0:
   resolved "https://registry.yarnpkg.com/react-refresh/-/react-refresh-0.17.0.tgz#b7e579c3657f23d04eccbe4ad2e58a8ed51e7e53"
   integrity sha512-z6F7K9bV85EfseRCp2bzrpyQ0Gkw1uLoCel9XBVWPg/TjRj94SkJzUTGfOa4bs7iJvBWtQG0Wq7wnI0syw3EBQ==
 
-react-router-dom@^7.5.0:
-  version "7.5.1"
-  resolved "https://registry.yarnpkg.com/react-router-dom/-/react-router-dom-7.5.1.tgz#16ffa63006dfdbab53cf243be84c29535e7dc4e0"
-  integrity sha512-5DPSPc7ENrt2tlKPq0FtpG80ZbqA9aIKEyqX6hSNJDlol/tr6iqCK4crqdsusmOSSotq6zDsn0y3urX9TuTNmA==
+react-router-dom@^7.5.3:
+  version "7.5.3"
+  resolved "https://registry.yarnpkg.com/react-router-dom/-/react-router-dom-7.5.3.tgz#496e9f6d90f731703c7772668b41747028e0a2d5"
+  integrity sha512-cK0jSaTyW4jV9SRKAItMIQfWZ/D6WEZafgHuuCb9g+SjhLolY78qc+De4w/Cz9ybjvLzShAmaIMEXt8iF1Cm+A==
   dependencies:
-    react-router "7.5.1"
+    react-router "7.5.3"
 
-react-router@7.5.1:
-  version "7.5.1"
-  resolved "https://registry.yarnpkg.com/react-router/-/react-router-7.5.1.tgz#e0bae54e913d139e727e7a88d165174a2cdbeb27"
-  integrity sha512-/jjU3fcYNd2bwz9Q0xt5TwyiyoO8XjSEFXJY4O/lMAlkGTHWuHRAbR9Etik+lSDqMC7A7mz3UlXzgYT6Vl58sA==
+react-router@7.5.3:
+  version "7.5.3"
+  resolved "https://registry.yarnpkg.com/react-router/-/react-router-7.5.3.tgz#9e5420832af8c3690740c1797d4fa54613fea06d"
+  integrity sha512-3iUDM4/fZCQ89SXlDa+Ph3MevBrozBAI655OAfWQlTm9nBR0IKlrmNwFow5lPHttbwvITZfkeeeZFP6zt3F7pw==
   dependencies:
     cookie "^1.0.1"
     set-cookie-parser "^2.6.0"

inspect_ai/agent/_bridge/patch.py

@@ -3,7 +3,7 @@ import re
 from contextvars import ContextVar
 from functools import wraps
 from time import time
-from typing import Any, AsyncGenerator, Optional, Type, cast
+from typing import Any, AsyncGenerator, Type, cast
 
 from openai._base_client import AsyncAPIClient, _AsyncStreamT
 from openai._models import FinalRequestOptions
@@ -65,7 +65,6 @@ def init_openai_request_patch() -> None:
             *,
             stream: bool = False,
             stream_cls: type[_AsyncStreamT] | None = None,
-            remaining_retries: Optional[int] = None,
         ) -> Any:
             # we have patched the underlying request method so now need to figure out when to
             # patch and when to stand down
@@ -88,7 +87,6 @@ def init_openai_request_patch() -> None:
                 options,
                 stream=stream,
                 stream_cls=stream_cls,
-                remaining_retries=remaining_retries,
             )
 
         setattr(AsyncAPIClient, "request", patched_request)

inspect_ai/analysis/beta/__init__.py

@@ -0,0 +1,57 @@
+from ._dataframe.columns import (
+    Column,
+    ColumnError,
+    ColumnErrors,
+    ColumnType,
+)
+from ._dataframe.evals.columns import (
+    EvalColumn,
+    EvalColumns,
+    EvalConfig,
+    EvalInfo,
+    EvalModel,
+    EvalResults,
+    EvalScores,
+    EvalTask,
+)
+from ._dataframe.evals.table import evals_df
+from ._dataframe.events.columns import EventColumn
+from ._dataframe.events.table import events_df
+from ._dataframe.messages.columns import (
+    MessageColumn,
+    MessageColumns,
+    MessageContent,
+    MessageToolCalls,
+)
+from ._dataframe.messages.table import MessageFilter, messages_df
+from ._dataframe.samples.columns import SampleColumn, SampleMessages, SampleSummary
+from ._dataframe.samples.table import samples_df
+
+__all__ = [
+    "evals_df",
+    "EvalColumn",
+    "EvalColumns",
+    "EvalInfo",
+    "EvalTask",
+    "EvalModel",
+    "EvalColumns",
+    "EvalConfig",
+    "EvalResults",
+    "EvalScores",
+    "samples_df",
+    "SampleColumn",
+    "SampleSummary",
+    "SampleMessages",
+    "messages_df",
+    "MessageColumn",
+    "MessageContent",
+    "MessageToolCalls",
+    "MessageColumns",
+    "MessageFilter",
+    "events_df",
+    "EventColumn",
+    "Column",
+    "ColumnType",
+    "ColumnError",
+    "ColumnErrors",
+]

inspect_ai/analysis/beta/_dataframe/columns.py

@@ -0,0 +1,145 @@
+import abc
+from dataclasses import KW_ONLY, dataclass
+from datetime import date, datetime, time
+from typing import Any, Callable, Mapping, Type, TypeAlias
+
+from jsonpath_ng import JSONPath  # type: ignore
+from jsonpath_ng.ext import parse  # type: ignore
+from pydantic import JsonValue
+
+from .validate import jsonpath_in_schema
+
+ColumnType: TypeAlias = int | float | bool | str | date | time | datetime | None
+"""Valid types for columns.
+
+Values of `list` and `dict` are converted into column values as JSON `str`.
+"""
+
+
+class Column(abc.ABC):
+    """
+    Specification for importing a column into a dataframe.
+
+    Extract columns from an `EvalLog` path either using [JSONPath](https://github.com/h2non/jsonpath-ng) expressions
+    or a function that takes `EvalLog` and returns a value.
+
+    By default, columns are not required, pass `required=True` to make them required. Non-required
+    columns are extracted as `None`, provide a `default` to yield an alternate value.
+
+    The `type` option serves as both a validation check and a directive to attempt to coerce the
+    data into the specified `type`. Coercion from `str` to other types is done after interpreting
+    the string using YAML (e.g. `"true"` -> `True`).
+
+    The `value` function provides an additional hook for transformation of the value read
+    from the log before it is realized as a column (e.g. list to a comma-separated string).
+
+    The `root` option indicates which root eval log context the columns select from.
+    """
+
+    def __init__(
+        self,
+        name: str,
+        *,
+        path: str | JSONPath | None,
+        required: bool = False,
+        default: JsonValue | None = None,
+        type: Type[ColumnType] | None = None,
+        value: Callable[[JsonValue], JsonValue] | None = None,
+    ) -> None:
+        self._name = name
+        self._path: str | JSONPath | None = path
+        self._required = required
+        self._default = default
+        self._type = type
+        self._value = value
+        self._validated: bool | None = None
+
+    @property
+    def name(self) -> str:
+        """Column name."""
+        return self._name
+
+    @property
+    def path(self) -> JSONPath | None:
+        """Path to column in `EvalLog`"""
+        if isinstance(self._path, str):
+            self._path = parse(self._path)
+        return self._path
+
+    @property
+    def required(self) -> bool:
+        """Is the column required? (error is raised if required columns aren't found)."""
+        return self._required
+
+    @property
+    def default(self) -> JsonValue | None:
+        """Default value for column when it is read from the log as `None`."""
+        return self._default
+
+    @property
+    def type(self) -> Type[ColumnType] | None:
+        """Column type (import will attempt to coerce to the specified type)."""
+        return self._type
+
+    def value(self, x: JsonValue) -> JsonValue:
+        """Convert extracted value into a column value (defaults to identity function).
+
+        Params:
+            x: Value to convert.
+
+        Returns:
+            Converted value.
+        """
+        if self._value:
+            return self._value(x)
+        else:
+            return x
+
+    def validate_path(self) -> bool:
+        if self.path is not None:
+            if self._validated is None:
+                schema = self.path_schema()
+                self._validated = (
+                    jsonpath_in_schema(self.path, schema) if schema else True
+                )
+            return self._validated
+        else:
+            return True
+
+    @abc.abstractmethod
+    def path_schema(self) -> Mapping[str, Any] | None: ...
+
+
+@dataclass
+class ColumnError:
+    """Error which occurred parsing a column."""
+
+    column: str
+    """Target column name."""
+
+    _: KW_ONLY
+
+    path: str | None
+    """Path to select column value. """
+
+    message: str
+    """Error message."""
+
+    def __str__(self) -> str:
+        msg = f"Error reading column '{self.column}'"
+        if self.path:
+            msg = f"{msg} from path '{self.path}'"
+        return f"{msg}: {self.message}"
+
+
+class ColumnErrors(dict[str, list[ColumnError]]):
+    """Dictionary of column errors keyed by log file."""
+
+    def __str__(self) -> str:
+        lines: list[str] = [""]
+        for file, errors in self.items():
+            lines.append(file)
+            for error in errors:
+                lines.append(f" - {error}")
+            lines.append("")
+        return "\n".join(lines)

inspect_ai/analysis/beta/_dataframe/evals/columns.py

@@ -0,0 +1,132 @@
+from datetime import datetime
+from typing import Any, Callable, Mapping, Type
+
+from jsonpath_ng import JSONPath  # type: ignore
+from pydantic import JsonValue
+from typing_extensions import override
+
+from inspect_ai.log._log import EvalLog
+
+from ..columns import Column, ColumnType
+from ..extract import list_as_str
+from ..validate import resolved_schema
+from .extract import eval_log_location, eval_log_scores_dict
+
+
+class EvalColumn(Column):
+    """Column which maps to `EvalLog`."""
+
+    def __init__(
+        self,
+        name: str,
+        *,
+        path: str | JSONPath | Callable[[EvalLog], JsonValue],
+        required: bool = False,
+        default: JsonValue | None = None,
+        type: Type[ColumnType] | None = None,
+        value: Callable[[JsonValue], JsonValue] | None = None,
+    ) -> None:
+        super().__init__(
+            name=name,
+            path=path if not callable(path) else None,
+            required=required,
+            default=default,
+            type=type,
+            value=value,
+        )
+        self._extract_eval = path if callable(path) else None
+
+    @override
+    def path_schema(self) -> Mapping[str, Any]:
+        return self.schema
+
+    schema = resolved_schema(EvalLog)
+
+
+EvalId: list[Column] = [
+    EvalColumn("eval_id", path="eval.eval_id", required=True),
+]
+"""Eval id column."""
+
+EvalInfo: list[Column] = [
+    EvalColumn("run_id", path="eval.run_id", required=True),
+    EvalColumn("task_id", path="eval.task_id", required=True),
+    EvalColumn("log", path=eval_log_location),
+    EvalColumn("created", path="eval.created", type=datetime, required=True),
+    EvalColumn("tags", path="eval.tags", default="", value=list_as_str),
+    EvalColumn("git_origin", path="eval.revision.origin"),
+    EvalColumn("git_commit", path="eval.revision.commit"),
+    EvalColumn("packages", path="eval.packages"),
+    EvalColumn("metadata", path="eval.metadata"),
+]
+"""Eval basic information columns."""
+
+EvalTask: list[Column] = [
+    EvalColumn("task_name", path="eval.task", required=True),
+    EvalColumn("task_version", path="eval.task_version", required=True),
+    EvalColumn("task_file", path="eval.task_file"),
+    EvalColumn("task_attribs", path="eval.task_attribs"),
+    EvalColumn("task_arg_*", path="eval.task_args"),
+    EvalColumn("solver", path="eval.solver"),
+    EvalColumn("solver_args", path="eval.solver_args"),
+    EvalColumn("sandbox_type", path="eval.sandbox.type"),
+    EvalColumn("sandbox_config", path="eval.sandbox.config"),
+]
+"""Eval task configuration columns."""
+
+EvalModel: list[Column] = [
+    EvalColumn("model", path="eval.model", required=True),
+    EvalColumn("model_base_url", path="eval.model_base_url"),
+    EvalColumn("model_args", path="eval.model_base_url"),
+    EvalColumn("model_generate_config", path="eval.model_generate_config"),
+    EvalColumn("model_roles", path="eval.model_roles"),
+]
+"""Eval model columns."""
+
+EvalDataset: list[Column] = [
+    EvalColumn("dataset_name", path="eval.dataset.name"),
+    EvalColumn("dataset_location", path="eval.dataset.location"),
+    EvalColumn("dataset_samples", path="eval.dataset.samples"),
+    EvalColumn("dataset_sample_ids", path="eval.dataset.sample_ids"),
+    EvalColumn("dataset_shuffled", path="eval.dataset.shuffled"),
+]
+"""Eval dataset columns."""
+
+EvalConfig: list[Column] = [
+    EvalColumn("epochs", path="eval.config.epochs"),
+    EvalColumn("epochs_reducer", path="eval.config.epochs_reducer"),
+    EvalColumn("approval", path="eval.config.approval"),
+    EvalColumn("message_limit", path="eval.config.message_limit"),
+    EvalColumn("token_limit", path="eval.config.token_limit"),
+    EvalColumn("time_limit", path="eval.config.time_limit"),
+    EvalColumn("working_limit", path="eval.config.working_limit"),
+]
+"""Eval configuration columns."""
+
+EvalResults: list[Column] = [
+    EvalColumn("status", path="status", required=True),
+    EvalColumn("error_message", path="error.message"),
+    EvalColumn("error_traceback", path="error.traceback"),
+    EvalColumn("total_samples", path="results.total_samples"),
+    EvalColumn("completed_samples", path="results.completed_samples"),
+    EvalColumn("score_headline_name", path="results.scores[0].scorer"),
+    EvalColumn("score_headline_metric", path="results.scores[0].metrics.*.name"),
+    EvalColumn("score_headline_value", path="results.scores[0].metrics.*.value"),
+]
+"""Eval results columns."""
+
+EvalScores: list[Column] = [
+    EvalColumn("score_*_*", path=eval_log_scores_dict),
+]
+"""Eval scores (one score/metric per-columns)."""
+
+EvalColumns: list[Column] = (
+    EvalInfo
+    + EvalTask
+    + EvalModel
+    + EvalDataset
+    + EvalConfig
+    + EvalResults
+    + EvalScores
+)
+"""Default columns to import for `evals_df()`."""

inspect_ai/analysis/beta/_dataframe/evals/extract.py

@@ -0,0 +1,23 @@
+from inspect_ai._util.path import native_path
+from inspect_ai.log._log import EvalLog
+
+
+def eval_log_location(log: EvalLog) -> str:
+    return native_path(log.location)
+
+
+def eval_log_scores_dict(
+    log: EvalLog,
+) -> list[dict[str, dict[str, int | float]]] | None:
+    if log.results is not None:
+        metrics = [
+            {
+                score.name: {
+                    metric.name: metric.value for metric in score.metrics.values()
+                }
+            }
+            for score in log.results.scores
+        ]
+        return metrics
+    else:
+        return None

inspect_ai/analysis/beta/_dataframe/evals/table.py

@@ -0,0 +1,140 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Literal, overload
+
+from inspect_ai._display import display
+from inspect_ai._util.path import pretty_path
+from inspect_ai.log._file import (
+    read_eval_log,
+)
+
+from ..columns import Column, ColumnErrors, ColumnType
+from ..record import import_record, resolve_duplicate_columns
+from ..util import (
+    LogPaths,
+    add_unreferenced_columns,
+    records_to_pandas,
+    resolve_columns,
+    resolve_logs,
+    verify_prerequisites,
+)
+from .columns import EvalColumns, EvalId
+
+if TYPE_CHECKING:
+    import pandas as pd
+
+EVAL_ID = "eval_id"
+EVAL_SUFFIX = "_eval"
+
+
+@overload
+def evals_df(
+    logs: LogPaths,
+    columns: list[Column] = EvalColumns,
+    recursive: bool = True,
+    reverse: bool = False,
+    strict: Literal[True] = True,
+) -> "pd.DataFrame": ...
+
+
+@overload
+def evals_df(
+    logs: LogPaths,
+    columns: list[Column] = EvalColumns,
+    recursive: bool = True,
+    reverse: bool = False,
+    strict: Literal[False] = False,
+) -> tuple["pd.DataFrame", ColumnErrors]: ...
+
+
+def evals_df(
+    logs: LogPaths,
+    columns: list[Column] = EvalColumns,
+    recursive: bool = True,
+    reverse: bool = False,
+    strict: bool = True,
+) -> "pd.DataFrame" | tuple["pd.DataFrame", ColumnErrors]:
+    """Read a dataframe containing evals.
+
+    Args:
+       logs: One or more paths to log files or log directories.
+       columns: Specification for what columns to read from log files.
+       recursive: Include recursive contents of directories (defaults to `True`)
+       reverse: Reverse the order of the dataframe (by default, items
+          are ordered from oldest to newest).
+       strict: Raise import errors immediately. Defaults to `True`.
+          If `False` then a tuple of `DataFrame` and errors is returned.
+
+    Returns:
+       For `strict`, a Pandas `DataFrame` with information for the specified logs.
+       For `strict=False`, a tuple of Pandas `DataFrame` and a dictionary of errors
+       encountered (by log file) during import.
+    """
+    verify_prerequisites()
+
+    # resolve logs
+    log_paths = resolve_logs(logs, recursive=recursive, reverse=reverse)
+
+    # resolve duplicate columns
+    columns = resolve_duplicate_columns(columns)
+
+    # accumulate errors for strict=False
+    all_errors = ColumnErrors()
+
+    # ensure eval_id
+    ensure_eval_id(columns)
+
+    # read logs
+    records: list[dict[str, ColumnType]] = []
+    with display().progress(total=len(log_paths)) as p:
+        for log_path in log_paths:
+            log = read_eval_log(log_path, header_only=True)
+            if strict:
+                record = import_record(log, columns, strict=True)
+            else:
+                record, errors = import_record(log, columns, strict=False)
+                all_errors[pretty_path(log_path)] = errors
+            records.append(record)
+
+            p.update()
+
+    # return table (+errors if strict=False)
+    evals_table = records_to_pandas(records)
+    evals_table = reorder_evals_df_columns(evals_table, columns)
+
+    if strict:
+        return evals_table
+    else:
+        return evals_table, all_errors
+
+
+def ensure_eval_id(columns: list[Column]) -> None:
+    if not any([column.name == EVAL_ID for column in columns]):
+        columns.extend(EvalId)
+
+
+def reorder_evals_df_columns(
+    df: "pd.DataFrame", eval_columns: list[Column]
+) -> "pd.DataFrame":
+    actual_columns = list(df.columns)
+    ordered_columns: list[str] = []
+
+    # eval_id first
+    if EVAL_ID in actual_columns:
+        ordered_columns.append(EVAL_ID)
+
+    # eval columns
+    for col in eval_columns:
+        col_pattern = col.name
+        if col_pattern == EVAL_ID:
+            continue  # Already handled
+
+        ordered_columns.extend(
+            resolve_columns(col_pattern, EVAL_SUFFIX, actual_columns, ordered_columns)
+        )
+
+    # add any unreferenced columns
+    ordered_columns = add_unreferenced_columns(actual_columns, ordered_columns)
+
+    # reorder the DataFrame
+    return df[ordered_columns]

inspect_ai/analysis/beta/_dataframe/events/columns.py

@@ -0,0 +1,37 @@
+from typing import Any, Callable, Mapping, Type
+
+from jsonpath_ng import JSONPath  # type: ignore
+from pydantic import JsonValue
+from typing_extensions import override
+
+from inspect_ai.log._transcript import Event
+
+from ..columns import Column, ColumnType
+
+
+class EventColumn(Column):
+    """Column which maps to `Event`."""
+
+    def __init__(
+        self,
+        name: str,
+        *,
+        path: str | JSONPath | Callable[[Event], JsonValue],
+        required: bool = False,
+        default: JsonValue | None = None,
+        type: Type[ColumnType] | None = None,
+        value: Callable[[JsonValue], JsonValue] | None = None,
+    ) -> None:
+        super().__init__(
+            name=name,
+            path=path if not callable(path) else None,
+            required=required,
+            default=default,
+            type=type,
+            value=value,
+        )
+        self._extract_event = path if callable(path) else None
+
+    @override
+    def path_schema(self) -> Mapping[str, Any] | None:
+        return None

inspect_ai/analysis/beta/_dataframe/events/table.py

@@ -0,0 +1,14 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    import pandas as pd
+
+from ..util import LogPaths, verify_prerequisites
+
+
+def events_df(logs: LogPaths, recursive: bool = True) -> "pd.DataFrame":
+    verify_prerequisites()
+
+    raise NotImplementedError("events_df has not been implemented yet.")