inspect-ai 0.3.93__py3-none-any.whl → 0.3.95__py3-none-any.whl

inspect_ai/agent/_as_tool.py

@@ -11,6 +11,7 @@ from inspect_ai.tool._tool_def import ToolDef, validate_tool_parameters
 from inspect_ai.tool._tool_info import ToolInfo, parse_tool_info
 from inspect_ai.tool._tool_params import ToolParam
 from inspect_ai.util._limit import Limit, apply_limits
+from inspect_ai.util._span import span
 
 from ._agent import AGENT_DESCRIPTION, Agent, AgentState
 
@@ -49,13 +50,17 @@ def as_tool(
             "Agent passed to as_tool was not created by an @agent decorated function"
         )
 
+    # get tool_info
+    tool_info = agent_tool_info(agent, description, **agent_kwargs)
+
     async def execute(input: str, *args: Any, **kwargs: Any) -> ToolResult:
         # prepare state
         state = AgentState(messages=[ChatMessageUser(content=input, source="input")])
 
         # run the agent with limits
         with apply_limits(limits):
-            state = await agent(state, *args, **(agent_kwargs | kwargs))
+            async with span(name=tool_info.name, type="agent"):
+                state = await agent(state, *args, **(agent_kwargs | kwargs))
 
         # find assistant message to read content from (prefer output)
         if not state.output.empty:
@@ -67,9 +72,6 @@ def as_tool(
         else:
             return ""
 
-    # get tool_info
-    tool_info = agent_tool_info(agent, description, **agent_kwargs)
-
     # add "input" param
     tool_info.parameters.properties = {
         "input": ToolParam(type="string", description="Input message.")

inspect_ai/agent/_bridge/patch.py

@@ -3,7 +3,7 @@ import re
 from contextvars import ContextVar
 from functools import wraps
 from time import time
-from typing import Any, AsyncGenerator, Optional, Type, cast
+from typing import Any, AsyncGenerator, Type, cast
 
 from openai._base_client import AsyncAPIClient, _AsyncStreamT
 from openai._models import FinalRequestOptions
@@ -65,7 +65,6 @@ def init_openai_request_patch() -> None:
             *,
             stream: bool = False,
             stream_cls: type[_AsyncStreamT] | None = None,
-            remaining_retries: Optional[int] = None,
         ) -> Any:
             # we have patched the underlying request method so now need to figure out when to
             # patch and when to stand down
@@ -88,7 +87,6 @@ def init_openai_request_patch() -> None:
                 options,
                 stream=stream,
                 stream_cls=stream_cls,
-                remaining_retries=remaining_retries,
             )
 
         setattr(AsyncAPIClient, "request", patched_request)

inspect_ai/agent/_handoff.py

@@ -57,7 +57,9 @@ def handoff(
     tool_info = agent_tool_info(agent, description, **agent_kwargs)
 
     # AgentTool calls will be intercepted by execute_tools
-    agent_tool = AgentTool(agent, input_filter, output_filter, limits, **agent_kwargs)
+    agent_tool = AgentTool(
+        agent, tool_info.name, input_filter, output_filter, limits, **agent_kwargs
+    )
     tool_name = tool_name or f"transfer_to_{tool_info.name}"
     set_registry_info(agent_tool, RegistryInfo(type="tool", name=tool_name))
     set_tool_description(
@@ -75,12 +77,14 @@ class AgentTool(Tool):
     def __init__(
         self,
         agent: Agent,
+        name: str,
         input_filter: MessageFilter | None = None,
         output_filter: MessageFilter | None = None,
         limits: list[Limit] = [],
         **kwargs: Any,
     ):
         self.agent = agent
+        self.name = name
         self.input_filter = input_filter
         self.output_filter = output_filter
         self.limits = limits

inspect_ai/agent/_react.py

@@ -195,9 +195,10 @@ def react(
                     answer = submission(messages)
                     if answer is not None:
                         # set the output to the answer for scoring
-                        state.output.completion = (
-                            f"{state.output.completion}\n\n{answer}".strip()
-                        )
+                        if submit.answer_only:
+                            state.output.completion = answer
+                        else:
+                            state.output.completion = f"{state.output.completion}{submit.answer_delimiter}{answer}".strip()
 
                         # exit if we are at max_attempts
                         attempt_count += 1

inspect_ai/agent/_run.py

@@ -1,8 +1,10 @@
 from copy import copy
 from typing import Any
 
+from inspect_ai._util.registry import registry_unqualified_name
 from inspect_ai.model._chat_message import ChatMessage, ChatMessageUser
 from inspect_ai.util._limit import Limit, apply_limits
+from inspect_ai.util._span import span
 
 from ._agent import Agent, AgentState
 
@@ -52,4 +54,7 @@ async def run(
 
     # run the agent with limits
     with apply_limits(limits):
-        return await agent(state, **agent_kwargs)
+        # run the agent
+        agent_name = registry_unqualified_name(agent)
+        async with span(name=agent_name, type="agent"):
+            return await agent(state, **agent_kwargs)

inspect_ai/agent/_types.py

@@ -96,3 +96,12 @@ class AgentSubmit(NamedTuple):
 
     The tool should return the `answer` provided to it for scoring.
     """
+
+    answer_only: bool = False
+    """Set the completion to only the answer provided by the submit tool.
+
+    By default, the answer is appended (with `answer_delimiter`) to whatever
+    other content the model generated along with the call to `submit()`."""
+
+    answer_delimiter: str = "\n\n"
+    """Delimter used when appending submit tool answer to other content the model generated along with the call to `submit()`."""

inspect_ai/analysis/beta/__init__.py

@@ -0,0 +1,57 @@
+from ._dataframe.columns import (
+    Column,
+    ColumnError,
+    ColumnErrors,
+    ColumnType,
+)
+from ._dataframe.evals.columns import (
+    EvalColumn,
+    EvalColumns,
+    EvalConfig,
+    EvalInfo,
+    EvalModel,
+    EvalResults,
+    EvalScores,
+    EvalTask,
+)
+from ._dataframe.evals.table import evals_df
+from ._dataframe.events.columns import EventColumn
+from ._dataframe.events.table import events_df
+from ._dataframe.messages.columns import (
+    MessageColumn,
+    MessageColumns,
+    MessageContent,
+    MessageToolCalls,
+)
+from ._dataframe.messages.table import MessageFilter, messages_df
+from ._dataframe.samples.columns import SampleColumn, SampleMessages, SampleSummary
+from ._dataframe.samples.table import samples_df
+
+__all__ = [
+    "evals_df",
+    "EvalColumn",
+    "EvalColumns",
+    "EvalInfo",
+    "EvalTask",
+    "EvalModel",
+    "EvalColumns",
+    "EvalConfig",
+    "EvalResults",
+    "EvalScores",
+    "samples_df",
+    "SampleColumn",
+    "SampleSummary",
+    "SampleMessages",
+    "messages_df",
+    "MessageColumn",
+    "MessageContent",
+    "MessageToolCalls",
+    "MessageColumns",
+    "MessageFilter",
+    "events_df",
+    "EventColumn",
+    "Column",
+    "ColumnType",
+    "ColumnError",
+    "ColumnErrors",
+]

inspect_ai/analysis/beta/_dataframe/columns.py

@@ -0,0 +1,145 @@
+import abc
+from dataclasses import KW_ONLY, dataclass
+from datetime import date, datetime, time
+from typing import Any, Callable, Mapping, Type, TypeAlias
+
+from jsonpath_ng import JSONPath  # type: ignore
+from jsonpath_ng.ext import parse  # type: ignore
+from pydantic import JsonValue
+
+from .validate import jsonpath_in_schema
+
+ColumnType: TypeAlias = int | float | bool | str | date | time | datetime | None
+"""Valid types for columns.
+
+Values of `list` and `dict` are converted into column values as JSON `str`.
+"""
+
+
+class Column(abc.ABC):
+    """
+    Specification for importing a column into a dataframe.
+
+    Extract columns from an `EvalLog` path either using [JSONPath](https://github.com/h2non/jsonpath-ng) expressions
+    or a function that takes `EvalLog` and returns a value.
+
+    By default, columns are not required, pass `required=True` to make them required. Non-required
+    columns are extracted as `None`, provide a `default` to yield an alternate value.
+
+    The `type` option serves as both a validation check and a directive to attempt to coerce the
+    data into the specified `type`. Coercion from `str` to other types is done after interpreting
+    the string using YAML (e.g. `"true"` -> `True`).
+
+    The `value` function provides an additional hook for transformation of the value read
+    from the log before it is realized as a column (e.g. list to a comma-separated string).
+
+    The `root` option indicates which root eval log context the columns select from.
+    """
+
+    def __init__(
+        self,
+        name: str,
+        *,
+        path: str | JSONPath | None,
+        required: bool = False,
+        default: JsonValue | None = None,
+        type: Type[ColumnType] | None = None,
+        value: Callable[[JsonValue], JsonValue] | None = None,
+    ) -> None:
+        self._name = name
+        self._path: str | JSONPath | None = path
+        self._required = required
+        self._default = default
+        self._type = type
+        self._value = value
+        self._validated: bool | None = None
+
+    @property
+    def name(self) -> str:
+        """Column name."""
+        return self._name
+
+    @property
+    def path(self) -> JSONPath | None:
+        """Path to column in `EvalLog`"""
+        if isinstance(self._path, str):
+            self._path = parse(self._path)
+        return self._path
+
+    @property
+    def required(self) -> bool:
+        """Is the column required? (error is raised if required columns aren't found)."""
+        return self._required
+
+    @property
+    def default(self) -> JsonValue | None:
+        """Default value for column when it is read from the log as `None`."""
+        return self._default
+
+    @property
+    def type(self) -> Type[ColumnType] | None:
+        """Column type (import will attempt to coerce to the specified type)."""
+        return self._type
+
+    def value(self, x: JsonValue) -> JsonValue:
+        """Convert extracted value into a column value (defaults to identity function).
+
+        Params:
+            x: Value to convert.
+
+        Returns:
+            Converted value.
+        """
+        if self._value:
+            return self._value(x)
+        else:
+            return x
+
+    def validate_path(self) -> bool:
+        if self.path is not None:
+            if self._validated is None:
+                schema = self.path_schema()
+                self._validated = (
+                    jsonpath_in_schema(self.path, schema) if schema else True
+                )
+            return self._validated
+        else:
+            return True
+
+    @abc.abstractmethod
+    def path_schema(self) -> Mapping[str, Any] | None: ...
+
+
+@dataclass
+class ColumnError:
+    """Error which occurred parsing a column."""
+
+    column: str
+    """Target column name."""
+
+    _: KW_ONLY
+
+    path: str | None
+    """Path to select column value. """
+
+    message: str
+    """Error message."""
+
+    def __str__(self) -> str:
+        msg = f"Error reading column '{self.column}'"
+        if self.path:
+            msg = f"{msg} from path '{self.path}'"
+        return f"{msg}: {self.message}"
+
+
+class ColumnErrors(dict[str, list[ColumnError]]):
+    """Dictionary of column errors keyed by log file."""
+
+    def __str__(self) -> str:
+        lines: list[str] = [""]
+        for file, errors in self.items():
+            lines.append(file)
+            for error in errors:
+                lines.append(f" - {error}")
+            lines.append("")
+        return "\n".join(lines)

inspect_ai/analysis/beta/_dataframe/evals/columns.py

@@ -0,0 +1,132 @@
+from datetime import datetime
+from typing import Any, Callable, Mapping, Type
+
+from jsonpath_ng import JSONPath  # type: ignore
+from pydantic import JsonValue
+from typing_extensions import override
+
+from inspect_ai.log._log import EvalLog
+
+from ..columns import Column, ColumnType
+from ..extract import list_as_str
+from ..validate import resolved_schema
+from .extract import eval_log_location, eval_log_scores_dict
+
+
+class EvalColumn(Column):
+    """Column which maps to `EvalLog`."""
+
+    def __init__(
+        self,
+        name: str,
+        *,
+        path: str | JSONPath | Callable[[EvalLog], JsonValue],
+        required: bool = False,
+        default: JsonValue | None = None,
+        type: Type[ColumnType] | None = None,
+        value: Callable[[JsonValue], JsonValue] | None = None,
+    ) -> None:
+        super().__init__(
+            name=name,
+            path=path if not callable(path) else None,
+            required=required,
+            default=default,
+            type=type,
+            value=value,
+        )
+        self._extract_eval = path if callable(path) else None
+
+    @override
+    def path_schema(self) -> Mapping[str, Any]:
+        return self.schema
+
+    schema = resolved_schema(EvalLog)
+
+
+EvalId: list[Column] = [
+    EvalColumn("eval_id", path="eval.eval_id", required=True),
+]
+"""Eval id column."""
+
+EvalInfo: list[Column] = [
+    EvalColumn("run_id", path="eval.run_id", required=True),
+    EvalColumn("task_id", path="eval.task_id", required=True),
+    EvalColumn("log", path=eval_log_location),
+    EvalColumn("created", path="eval.created", type=datetime, required=True),
+    EvalColumn("tags", path="eval.tags", default="", value=list_as_str),
+    EvalColumn("git_origin", path="eval.revision.origin"),
+    EvalColumn("git_commit", path="eval.revision.commit"),
+    EvalColumn("packages", path="eval.packages"),
+    EvalColumn("metadata", path="eval.metadata"),
+]
+"""Eval basic information columns."""
+
+EvalTask: list[Column] = [
+    EvalColumn("task_name", path="eval.task", required=True),
+    EvalColumn("task_version", path="eval.task_version", required=True),
+    EvalColumn("task_file", path="eval.task_file"),
+    EvalColumn("task_attribs", path="eval.task_attribs"),
+    EvalColumn("task_arg_*", path="eval.task_args"),
+    EvalColumn("solver", path="eval.solver"),
+    EvalColumn("solver_args", path="eval.solver_args"),
+    EvalColumn("sandbox_type", path="eval.sandbox.type"),
+    EvalColumn("sandbox_config", path="eval.sandbox.config"),
+]
+"""Eval task configuration columns."""
+
+EvalModel: list[Column] = [
+    EvalColumn("model", path="eval.model", required=True),
+    EvalColumn("model_base_url", path="eval.model_base_url"),
+    EvalColumn("model_args", path="eval.model_base_url"),
+    EvalColumn("model_generate_config", path="eval.model_generate_config"),
+    EvalColumn("model_roles", path="eval.model_roles"),
+]
+"""Eval model columns."""
+
+EvalDataset: list[Column] = [
+    EvalColumn("dataset_name", path="eval.dataset.name"),
+    EvalColumn("dataset_location", path="eval.dataset.location"),
+    EvalColumn("dataset_samples", path="eval.dataset.samples"),
+    EvalColumn("dataset_sample_ids", path="eval.dataset.sample_ids"),
+    EvalColumn("dataset_shuffled", path="eval.dataset.shuffled"),
+]
+"""Eval dataset columns."""
+
+EvalConfig: list[Column] = [
+    EvalColumn("epochs", path="eval.config.epochs"),
+    EvalColumn("epochs_reducer", path="eval.config.epochs_reducer"),
+    EvalColumn("approval", path="eval.config.approval"),
+    EvalColumn("message_limit", path="eval.config.message_limit"),
+    EvalColumn("token_limit", path="eval.config.token_limit"),
+    EvalColumn("time_limit", path="eval.config.time_limit"),
+    EvalColumn("working_limit", path="eval.config.working_limit"),
+]
+"""Eval configuration columns."""
+
+EvalResults: list[Column] = [
+    EvalColumn("status", path="status", required=True),
+    EvalColumn("error_message", path="error.message"),
+    EvalColumn("error_traceback", path="error.traceback"),
+    EvalColumn("total_samples", path="results.total_samples"),
+    EvalColumn("completed_samples", path="results.completed_samples"),
+    EvalColumn("score_headline_name", path="results.scores[0].scorer"),
+    EvalColumn("score_headline_metric", path="results.scores[0].metrics.*.name"),
+    EvalColumn("score_headline_value", path="results.scores[0].metrics.*.value"),
+]
+"""Eval results columns."""
+
+EvalScores: list[Column] = [
+    EvalColumn("score_*_*", path=eval_log_scores_dict),
+]
+"""Eval scores (one score/metric per-columns)."""
+
+EvalColumns: list[Column] = (
+    EvalInfo
+    + EvalTask
+    + EvalModel
+    + EvalDataset
+    + EvalConfig
+    + EvalResults
+    + EvalScores
+)
+"""Default columns to import for `evals_df()`."""

inspect_ai/analysis/beta/_dataframe/evals/extract.py

@@ -0,0 +1,23 @@
+from inspect_ai._util.path import native_path
+from inspect_ai.log._log import EvalLog
+
+
+def eval_log_location(log: EvalLog) -> str:
+    return native_path(log.location)
+
+
+def eval_log_scores_dict(
+    log: EvalLog,
+) -> list[dict[str, dict[str, int | float]]] | None:
+    if log.results is not None:
+        metrics = [
+            {
+                score.name: {
+                    metric.name: metric.value for metric in score.metrics.values()
+                }
+            }
+            for score in log.results.scores
+        ]
+        return metrics
+    else:
+        return None

inspect_ai/analysis/beta/_dataframe/evals/table.py

@@ -0,0 +1,140 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Literal, overload
+
+from inspect_ai._display import display
+from inspect_ai._util.path import pretty_path
+from inspect_ai.log._file import (
+    read_eval_log,
+)
+
+from ..columns import Column, ColumnErrors, ColumnType
+from ..record import import_record, resolve_duplicate_columns
+from ..util import (
+    LogPaths,
+    add_unreferenced_columns,
+    records_to_pandas,
+    resolve_columns,
+    resolve_logs,
+    verify_prerequisites,
+)
+from .columns import EvalColumns, EvalId
+
+if TYPE_CHECKING:
+    import pandas as pd
+
+EVAL_ID = "eval_id"
+EVAL_SUFFIX = "_eval"
+
+
+@overload
+def evals_df(
+    logs: LogPaths,
+    columns: list[Column] = EvalColumns,
+    recursive: bool = True,
+    reverse: bool = False,
+    strict: Literal[True] = True,
+) -> "pd.DataFrame": ...
+
+
+@overload
+def evals_df(
+    logs: LogPaths,
+    columns: list[Column] = EvalColumns,
+    recursive: bool = True,
+    reverse: bool = False,
+    strict: Literal[False] = False,
+) -> tuple["pd.DataFrame", ColumnErrors]: ...
+
+
+def evals_df(
+    logs: LogPaths,
+    columns: list[Column] = EvalColumns,
+    recursive: bool = True,
+    reverse: bool = False,
+    strict: bool = True,
+) -> "pd.DataFrame" | tuple["pd.DataFrame", ColumnErrors]:
+    """Read a dataframe containing evals.
+
+    Args:
+       logs: One or more paths to log files or log directories.
+       columns: Specification for what columns to read from log files.
+       recursive: Include recursive contents of directories (defaults to `True`)
+       reverse: Reverse the order of the dataframe (by default, items
+          are ordered from oldest to newest).
+       strict: Raise import errors immediately. Defaults to `True`.
+          If `False` then a tuple of `DataFrame` and errors is returned.
+
+    Returns:
+       For `strict`, a Pandas `DataFrame` with information for the specified logs.
+       For `strict=False`, a tuple of Pandas `DataFrame` and a dictionary of errors
+       encountered (by log file) during import.
+    """
+    verify_prerequisites()
+
+    # resolve logs
+    log_paths = resolve_logs(logs, recursive=recursive, reverse=reverse)
+
+    # resolve duplicate columns
+    columns = resolve_duplicate_columns(columns)
+
+    # accumulate errors for strict=False
+    all_errors = ColumnErrors()
+
+    # ensure eval_id
+    ensure_eval_id(columns)
+
+    # read logs
+    records: list[dict[str, ColumnType]] = []
+    with display().progress(total=len(log_paths)) as p:
+        for log_path in log_paths:
+            log = read_eval_log(log_path, header_only=True)
+            if strict:
+                record = import_record(log, columns, strict=True)
+            else:
+                record, errors = import_record(log, columns, strict=False)
+                all_errors[pretty_path(log_path)] = errors
+            records.append(record)
+
+            p.update()
+
+    # return table (+errors if strict=False)
+    evals_table = records_to_pandas(records)
+    evals_table = reorder_evals_df_columns(evals_table, columns)
+
+    if strict:
+        return evals_table
+    else:
+        return evals_table, all_errors
+
+
+def ensure_eval_id(columns: list[Column]) -> None:
+    if not any([column.name == EVAL_ID for column in columns]):
+        columns.extend(EvalId)
+
+
+def reorder_evals_df_columns(
+    df: "pd.DataFrame", eval_columns: list[Column]
+) -> "pd.DataFrame":
+    actual_columns = list(df.columns)
+    ordered_columns: list[str] = []
+
+    # eval_id first
+    if EVAL_ID in actual_columns:
+        ordered_columns.append(EVAL_ID)
+
+    # eval columns
+    for col in eval_columns:
+        col_pattern = col.name
+        if col_pattern == EVAL_ID:
+            continue  # Already handled
+
+        ordered_columns.extend(
+            resolve_columns(col_pattern, EVAL_SUFFIX, actual_columns, ordered_columns)
+        )
+
+    # add any unreferenced columns
+    ordered_columns = add_unreferenced_columns(actual_columns, ordered_columns)
+
+    # reorder the DataFrame
+    return df[ordered_columns]

inspect_ai/analysis/beta/_dataframe/events/columns.py

@@ -0,0 +1,37 @@
+from typing import Any, Callable, Mapping, Type
+
+from jsonpath_ng import JSONPath  # type: ignore
+from pydantic import JsonValue
+from typing_extensions import override
+
+from inspect_ai.log._transcript import Event
+
+from ..columns import Column, ColumnType
+
+
+class EventColumn(Column):
+    """Column which maps to `Event`."""
+
+    def __init__(
+        self,
+        name: str,
+        *,
+        path: str | JSONPath | Callable[[Event], JsonValue],
+        required: bool = False,
+        default: JsonValue | None = None,
+        type: Type[ColumnType] | None = None,
+        value: Callable[[JsonValue], JsonValue] | None = None,
+    ) -> None:
+        super().__init__(
+            name=name,
+            path=path if not callable(path) else None,
+            required=required,
+            default=default,
+            type=type,
+            value=value,
+        )
+        self._extract_event = path if callable(path) else None
+
+    @override
+    def path_schema(self) -> Mapping[str, Any] | None:
+        return None