inspect-ai 0.3.56__py3-none-any.whl → 0.3.58__py3-none-any.whl

inspect_ai/__init__.py

@@ -7,7 +7,7 @@ from inspect_ai._eval.evalset import eval_set
 from inspect_ai._eval.list import list_tasks
 from inspect_ai._eval.registry import task
 from inspect_ai._eval.score import score, score_async
-from inspect_ai._eval.task import Epochs, Task, TaskInfo, Tasks
+from inspect_ai._eval.task import Epochs, Task, TaskInfo, Tasks, task_with
 from inspect_ai._util.constants import PKG_NAME
 from inspect_ai.solver._human_agent.agent import human_agent
 
@@ -29,4 +29,5 @@ __all__ = [
     "TaskInfo",
     "Tasks",
     "task",
+    "task_with",
 ]

inspect_ai/_cli/common.py

@@ -17,7 +17,7 @@ class CommonOptions(TypedDict):
     log_level: str
     log_level_transcript: str
     log_dir: str
-    display: Literal["full", "rich", "plain", "none"]
+    display: Literal["full", "conversation", "rich", "plain", "none"]
     no_ansi: bool | None
     debug: bool
     debug_port: int
@@ -64,7 +64,9 @@ def common_options(func: Callable[..., Any]) -> Callable[..., click.Context]:
     )
     @click.option(
         "--display",
-        type=click.Choice(["full", "rich", "plain", "none"], case_sensitive=False),
+        type=click.Choice(
+            ["full", "conversation", "rich", "plain", "none"], case_sensitive=False
+        ),
         default=DEFAULT_DISPLAY,
         envvar="INSPECT_DISPLAY",
         help="Set the display type (defaults to 'full')",

inspect_ai/_cli/eval.py

@@ -118,6 +118,7 @@ def eval_options(func: Callable[..., Any]) -> Callable[..., click.Context]:
         "--trace",
         type=bool,
         is_flag=True,
+        hidden=True,
         envvar="INSPECT_EVAL_TRACE",
         help="Trace message interactions with evaluated model to terminal.",
     )
@@ -886,6 +887,7 @@ def parse_comma_separated(value: str | None) -> list[str] | None:
     "--trace",
     type=bool,
     is_flag=True,
+    hidden=True,
     help="Trace message interactions with evaluated model to terminal.",
     envvar="INSPECT_EVAL_TRACE",
 )

inspect_ai/_cli/trace.py

@@ -62,11 +62,21 @@ def list_command(json: bool) -> None:
 
 @trace_command.command("dump")
 @click.argument("trace-file", type=str, required=False)
-def dump_command(trace_file: str | None) -> None:
+@click.option(
+    "--filter",
+    type=str,
+    help="Filter (applied to trace message field).",
+)
+def dump_command(trace_file: str | None, filter: str | None) -> None:
     """Dump a trace file to stdout (as a JSON array of log records)."""
     trace_file_path = _resolve_trace_file_path(trace_file)
 
     traces = read_trace_file(trace_file_path)
+
+    if filter:
+        filter = filter.lower()
+        traces = [trace for trace in traces if filter in trace.message.lower()]
+
     print(
         to_json(traces, indent=2, exclude_none=True, fallback=lambda _: None).decode()
     )
@@ -74,17 +84,26 @@ def dump_command(trace_file: str | None) -> None:
 
 @trace_command.command("anomalies")
 @click.argument("trace-file", type=str, required=False)
+@click.option(
+    "--filter",
+    type=str,
+    help="Filter (applied to trace message field).",
+)
 @click.option(
     "--all",
     is_flag=True,
     default=False,
     help="Show all anomolies including errors and timeouts (by default only still running and cancelled actions are shown).",
 )
-def anomolies_command(trace_file: str | None, all: bool) -> None:
+def anomolies_command(trace_file: str | None, filter: str | None, all: bool) -> None:
     """Look for anomalies in a trace file (never completed or cancelled actions)."""
     trace_file_path = _resolve_trace_file_path(trace_file)
     traces = read_trace_file(trace_file_path)
 
+    if filter:
+        filter = filter.lower()
+        traces = [trace for trace in traces if filter in trace.message.lower()]
+
     # Track started actions
     running_actions: dict[str, ActionTraceRecord] = {}
     canceled_actions: dict[str, ActionTraceRecord] = {}

inspect_ai/_display/core/active.py

@@ -4,7 +4,6 @@ from contextvars import ContextVar
 import rich
 
 from inspect_ai.util._display import display_type
-from inspect_ai.util._trace import trace_enabled
 
 from ..rich.display import RichDisplay
 from ..textual.display import TextualDisplay
@@ -17,7 +16,6 @@ def display() -> Display:
         if (
             display_type() == "full"
             and sys.stdout.isatty()
-            and not trace_enabled()
             and not rich.get_console().is_jupyter
         ):
             _active_display = TextualDisplay()

inspect_ai/_display/core/panel.py

@@ -112,7 +112,7 @@ def tasks_title(completed: int, total: int) -> str:
 def task_title(profile: TaskProfile, show_model: bool) -> str:
     eval_epochs = profile.eval_config.epochs or 1
     epochs = f" x {profile.eval_config.epochs}" if eval_epochs > 1 else ""
-    samples = f"{profile.samples//eval_epochs:,}{epochs} sample{'s' if profile.samples != 1 else ''}"
+    samples = f"{profile.samples // eval_epochs:,}{epochs} sample{'s' if profile.samples != 1 else ''}"
     title = f"{registry_unqualified_name(profile.name)} ({samples})"
     if show_model:
         title = f"{title}: {profile.model}"

inspect_ai/_display/rich/display.py

@@ -15,7 +15,6 @@ from inspect_ai._util.constants import CONSOLE_DISPLAY_WIDTH
 from inspect_ai.log._transcript import InputEvent, transcript
 from inspect_ai.util._display import display_type
 from inspect_ai.util._throttle import throttle
-from inspect_ai.util._trace import trace_enabled
 
 from ..core.config import task_config
 from ..core.display import (
@@ -151,7 +150,8 @@ class RichDisplay(Display):
     @throttle(1)
     def _update_display(self) -> None:
         if (
-            self.tasks is not None
+            display_type() != "conversation"
+            and self.tasks is not None
             and self.tasks
             and self.progress_ui is not None
             and self.live is not None
@@ -170,7 +170,7 @@ class RichTaskScreen(TaskScreen):
     def __init__(self, live: Live) -> None:
         self.theme = rich_theme()
         self.live = live
-        status_text = "Working" if trace_enabled() else "Task running"
+        status_text = "Working" if display_type() == "conversation" else "Task running"
         self.status = self.live.console.status(
             f"[{self.theme.meta} bold]{status_text}...[/{self.theme.meta} bold]",
             spinner="clock",
@@ -189,7 +189,7 @@ class RichTaskScreen(TaskScreen):
     ) -> Iterator[Console]:
         # determine transient based on trace mode
         if transient is None:
-            transient = not trace_enabled()
+            transient = display_type() != "conversation"
 
         # clear live task status and transient status
         self.live.update("", refresh=True)

inspect_ai/_display/textual/app.py

@@ -284,7 +284,10 @@ class TaskScreenApp(App[TR]):
 
     def update_samples(self) -> None:
         samples_view = self.query_one(SamplesView)
-        samples_view.set_samples(active_samples())
+        active_and_started_samples = [
+            sample for sample in active_samples() if sample.started is not None
+        ]
+        samples_view.set_samples(active_and_started_samples)
 
     def update_footer(self) -> None:
         left, right = task_footer()

inspect_ai/_display/textual/widgets/samples.py

@@ -25,6 +25,7 @@ from textual.widgets.option_list import Option, Separator
 from inspect_ai._util.format import format_progress_time
 from inspect_ai._util.registry import registry_unqualified_name
 from inspect_ai.log._samples import ActiveSample
+from inspect_ai.log._transcript import ToolEvent
 
 from .clock import Clock
 from .transcript import TranscriptView
@@ -332,16 +333,29 @@ class SandboxesView(Vertical):
 
 
 class SampleToolbar(Horizontal):
+    STATUS_GROUP = "status_group"
+    TIMEOUT_TOOL_CALL = "timeout_tool_call"
     CANCEL_SCORE_OUTPUT = "cancel_score_output"
     CANCEL_RAISE_ERROR = "cancel_raise_error"
     PENDING_STATUS = "pending_status"
     PENDING_CAPTION = "pending_caption"
 
     DEFAULT_CSS = f"""
+    SampleToolbar {{
+        grid-size: 5 1;
+        grid-columns: auto auto 1fr auto auto;
+    }}
+    SampleToolbar #{STATUS_GROUP} {{
+        min-width: 20;
+    }}
     SampleToolbar Button {{
         margin-bottom: 1;
         margin-right: 2;
-        min-width: 20;
+        min-width: 18;
+    }}
+    SampleToolbar #{TIMEOUT_TOOL_CALL} {{
+        color: $secondary-darken-3;
+        min-width: 16;
     }}
     SampleToolbar #{CANCEL_SCORE_OUTPUT} {{
         color: $primary-darken-3;
@@ -356,9 +370,16 @@ class SampleToolbar(Horizontal):
         self.sample: ActiveSample | None = None
 
     def compose(self) -> ComposeResult:
-        with VerticalGroup(id=self.PENDING_STATUS):
-            yield Static("Executing...", id=self.PENDING_CAPTION)
-            yield HorizontalGroup(EventLoadingIndicator(), Clock())
+        with HorizontalGroup(id=self.STATUS_GROUP):
+            with VerticalGroup(id=self.PENDING_STATUS):
+                yield Static("Executing...", id=self.PENDING_CAPTION)
+                yield HorizontalGroup(EventLoadingIndicator(), Clock())
+        yield Button(
+            Text("Timeout Tool"),
+            id=self.TIMEOUT_TOOL_CALL,
+            tooltip="Cancel the tool call and report a timeout to the model.",
+        )
+        yield Horizontal()
         yield Button(
             Text("Cancel (Score)"),
             id=self.CANCEL_SCORE_OUTPUT,
@@ -372,12 +393,21 @@ class SampleToolbar(Horizontal):
 
     def on_mount(self) -> None:
         self.query_one("#" + self.PENDING_STATUS).visible = False
+        self.query_one("#" + self.TIMEOUT_TOOL_CALL).display = False
         self.query_one("#" + self.CANCEL_SCORE_OUTPUT).display = False
         self.query_one("#" + self.CANCEL_RAISE_ERROR).display = False
 
     def on_button_pressed(self, event: Button.Pressed) -> None:
         if self.sample:
-            if event.button.id == self.CANCEL_SCORE_OUTPUT:
+            if event.button.id == self.TIMEOUT_TOOL_CALL:
+                last_event = (
+                    self.sample.transcript.events[-1]
+                    if self.sample.transcript.events
+                    else None
+                )
+                if isinstance(last_event, ToolEvent):
+                    last_event.cancel()
+            elif event.button.id == self.CANCEL_SCORE_OUTPUT:
                 self.sample.interrupt("score")
             elif event.button.id == self.CANCEL_RAISE_ERROR:
                 self.sample.interrupt("error")
@@ -389,6 +419,7 @@ class SampleToolbar(Horizontal):
         self.sample = sample
 
         pending_status = self.query_one("#" + self.PENDING_STATUS)
+        timeout_tool = self.query_one("#" + self.TIMEOUT_TOOL_CALL)
         clock = self.query_one(Clock)
         cancel_score_output = cast(
             Button, self.query_one("#" + self.CANCEL_SCORE_OUTPUT)
@@ -419,14 +450,19 @@ class SampleToolbar(Horizontal):
                 pending_caption.update(
                     Text.from_markup(f"[italic]{pending_caption_text}[/italic]")
                 )
+
+                timeout_tool.display = isinstance(last_event, ToolEvent)
+
                 clock.start(last_event.timestamp.timestamp())
             else:
                 pending_status.visible = False
+                timeout_tool.display = False
                 clock.stop()
 
         else:
             self.display = False
             pending_status.visible = False
+            timeout_tool.display = False
             clock.stop()
 
 

inspect_ai/_eval/eval.py

@@ -7,11 +7,12 @@ from shortuuid import uuid
 from typing_extensions import Unpack
 
 from inspect_ai._cli.util import parse_cli_args
-from inspect_ai._display.core.active import display
+from inspect_ai._display.core.active import display as task_display
 from inspect_ai._util.config import resolve_args
 from inspect_ai._util.constants import DEFAULT_LOG_FORMAT
 from inspect_ai._util.error import PrerequisiteError
 from inspect_ai._util.file import absolute_file_path
+from inspect_ai._util.logger import warn_once
 from inspect_ai._util.platform import platform_init
 from inspect_ai._util.registry import registry_lookup
 from inspect_ai.approval._apply import init_tool_approval
@@ -34,7 +35,7 @@ from inspect_ai.scorer._reducer import reducer_log_names
 from inspect_ai.solver._chain import chain
 from inspect_ai.solver._solver import Solver, SolverSpec
 from inspect_ai.util import SandboxEnvironmentType
-from inspect_ai.util._trace import init_trace
+from inspect_ai.util._display import DisplayType, display_type, init_display_type
 
 from .context import init_eval_context
 from .loader import ResolvedTask, resolve_tasks
@@ -55,6 +56,7 @@ def eval(
     solver: Solver | list[Solver] | SolverSpec | None = None,
     tags: list[str] | None = None,
     trace: bool | None = None,
+    display: DisplayType | None = None,
     approval: str | list[ApprovalPolicy] | None = None,
     log_level: str | None = None,
     log_level_transcript: str | None = None,
@@ -100,7 +102,8 @@ def eval(
         solver (Solver | list[Solver] | SolverSpec | None): Alternative solver for task(s).
           Optional (uses task solver by default).
         tags (list[str] | None): Tags to associate with this evaluation run.
-        trace: (bool | None): Trace message interactions with evaluated model to terminal.
+        trace (bool | None): Trace message interactions with evaluated model to terminal.
+        display (DisplayType | None): Task display type (defaults to 'full').
         approval: (str | list[ApprovalPolicy] | None): Tool use approval policies.
           Either a path to an approval policy config file or a list of approval policies.
           Defaults to no approval policy.
@@ -150,9 +153,11 @@ def eval(
     platform_init()
 
     # resolve eval trace
-    max_tasks, max_samples = init_eval_trace(trace, max_tasks, max_samples, model)
+    max_tasks, max_samples = init_eval_display(
+        display, trace, max_tasks, max_samples, model
+    )
 
-    return display().run_task_app(
+    return task_display().run_task_app(
         main=eval_async(
             tasks=tasks,
             model=model,
@@ -163,7 +168,6 @@ def eval(
             sandbox_cleanup=sandbox_cleanup,
             solver=solver,
             tags=tags,
-            trace=trace,
             approval=approval,
             log_level=log_level,
             log_level_transcript=log_level_transcript,
@@ -201,7 +205,6 @@ async def eval_async(
     sandbox_cleanup: bool | None = None,
     solver: Solver | list[Solver] | SolverSpec | None = None,
     tags: list[str] | None = None,
-    trace: bool | None = None,
     approval: str | list[ApprovalPolicy] | ApprovalPolicyConfig | None = None,
     log_level: str | None = None,
     log_level_transcript: str | None = None,
@@ -247,7 +250,6 @@ async def eval_async(
         solver (Solver | list[Solver] | SolverSpec | None): Alternative solver for task(s).
           Optional (uses task solver by default).
         tags (list[str] | None): Tags to associate with this evaluation run.
-        trace: (bool | None): Trace message interactions with evaluated model to terminal.
         approval: (str | list[ApprovalPolicy] | None): Tool use approval policies.
           Either a path to an approval policy config file or a list of approval policies.
           Defaults to no approval policy.
@@ -329,8 +331,8 @@ async def eval_async(
             log.warning("No inspect tasks were found at the specified paths.")
             return []
 
-        # apply trace mode constraints
-        if trace:
+        # apply conversation display constraints
+        if display_type() == "conversation":
             # single task at a time
             if max_tasks is not None:
                 max_tasks = 1
@@ -371,7 +373,6 @@ async def eval_async(
             epochs_reducer=reducer_log_names(epochs_reducer)
             if epochs_reducer
             else None,
-            trace=trace,
             approval=config_from_approval_policies(approval) if approval else None,
             fail_on_error=fail_on_error,
             message_limit=message_limit,
@@ -467,6 +468,7 @@ def eval_retry(
     max_sandboxes: int | None = None,
     sandbox_cleanup: bool | None = None,
     trace: bool | None = None,
+    display: DisplayType | None = None,
     fail_on_error: bool | float | None = None,
     debug_errors: bool | None = None,
     log_samples: bool | None = None,
@@ -501,6 +503,7 @@ def eval_retry(
         sandbox_cleanup (bool | None): Cleanup sandbox environments after task completes
            (defaults to True)
         trace (bool | None): Trace message interactions with evaluated model to terminal.
+        display (DisplayType | None): Task display type (defaults to 'full').
         fail_on_error (bool | float | None): `True` to fail on first sample error
            (default); `False` to never fail on sample errors; Value between 0 and 1
            to fail if a proportion of total samples fails. Value greater than 1 to fail
@@ -529,9 +532,9 @@ def eval_retry(
     platform_init()
 
     # resolve eval trace
-    max_tasks, max_samples = init_eval_trace(trace, max_tasks, max_samples)
+    max_tasks, max_samples = init_eval_display(display, trace, max_tasks, max_samples)
 
-    return display().run_task_app(
+    return task_display().run_task_app(
         main=eval_retry_async(
             tasks=tasks,
             log_level=log_level,
@@ -800,9 +803,8 @@ def eval_init(
 
     # resolve tasks (set active model to resolve uses of the
     # 'default' model in tools, solvers, and scorers)
-    from inspect_ai._display.core.active import display
 
-    with display().suspend_task_app():
+    with task_display().suspend_task_app():
         resolved_tasks: list[ResolvedTask] = []
         for m in models:
             init_active_model(m, generate_config)
@@ -816,17 +818,27 @@ def eval_init(
     return models, approval, resolved_tasks
 
 
-def init_eval_trace(
+def init_eval_display(
+    display: DisplayType | None,
     trace: bool | None,
     max_tasks: int | None,
     max_samples: int | None,
     model: Any = None,
 ) -> tuple[int | None, int | None]:
-    # init trace setting
-    init_trace(trace)
-
-    # adapt task/samples as required
+    # propagate any trace value to display_type
     if trace:
+        warn_once(
+            log,
+            "WARNING: The --trace flag is deprecated (use --display=conversation instead)",
+        )
+        display = "conversation"
+
+    # apply default and init
+    display = display or display_type()
+    init_display_type(display)
+
+    # adapt task/samples as required if we are in conversation mode
+    if display_type() == "conversation":
         # single task at a time
         if max_tasks is not None:
             max_tasks = 1

inspect_ai/_eval/evalset.py

@@ -33,7 +33,7 @@ from inspect_ai.model import (
 )
 from inspect_ai.model._generate_config import GenerateConfig
 from inspect_ai.solver._solver import Solver, SolverSpec
-from inspect_ai.util import SandboxEnvironmentType
+from inspect_ai.util import DisplayType, SandboxEnvironmentType
 
 from .eval import eval, eval_init
 from .loader import ResolvedTask, resolve_task_args
@@ -59,6 +59,7 @@ def eval_set(
     solver: Solver | list[Solver] | SolverSpec | None = None,
     tags: list[str] | None = None,
     trace: bool | None = None,
+    display: DisplayType | None = None,
     approval: str | list[ApprovalPolicy] | None = None,
     score: bool = True,
     log_level: str | None = None,
@@ -116,6 +117,7 @@ def eval_set(
            evaluating task(s). ptional (uses task solver by default).
         tags (list[str] | None): Tags to associate with this evaluation run.
         trace: (bool | None): Trace message interactions with evaluated model to terminal.
+        display (DisplayType | None): Task display type (defaults to 'full').
         approval: (str | list[ApprovalPolicy] | None): Tool use approval policies.
           Either a path to an approval policy config file or a list of approval policies.
           Defaults to no approval policy.
@@ -180,6 +182,7 @@ def eval_set(
             solver=solver,
             tags=tags,
             trace=trace,
+            display=display,
             approval=approval,
             log_level=log_level,
             log_level_transcript=log_level_transcript,
@@ -501,9 +504,6 @@ def latest_completed_task_eval_logs(
     # take the most recent completed log for each id
     latest_completed_logs: list[Log] = []
     for id, id_logs in logs_by_id.items():
-        # filter on completed
-        id_logs = [id_log for id_log in id_logs if id_log[1].status != "started"]
-
         # continue if there are no target logs
         if len(id_logs) == 0:
             continue
@@ -517,11 +517,13 @@ def latest_completed_task_eval_logs(
         latest_completed_logs.append(id_logs[0])
 
         # remove the rest if requested
+        # (don't remove 'started' in case its needed for post-mortum debugging)
         if cleanup_older:
             fs = filesystem(id_logs[0][0].name)
             for id_log in id_logs[1:]:
                 try:
-                    fs.rm(id_log[0].name)
+                    if id_log.header.status != "started":
+                        fs.rm(id_log.info.name)
                 except Exception as ex:
                     logger.warning(f"Error attempt to remove '{id_log[0].name}': {ex}")
 

inspect_ai/_eval/run.py

@@ -42,7 +42,7 @@ from .task.log import TaskLogger
 from .task.run import TaskRunOptions, task_run
 from .task.rundir import task_run_dir_switching
 from .task.sandbox import TaskSandboxEnvironment, resolve_sandbox_for_task
-from .task.util import task_run_dir
+from .task.util import slice_dataset, task_run_dir
 
 log = logging.getLogger(__name__)
 
@@ -70,12 +70,23 @@ async def eval_run(
     # get cwd before switching to task dir
     eval_wd = os.getcwd()
 
+    # ensure sample ids
+    for resolved_task in tasks:
+        # add sample ids to dataset if they aren't there (start at 1 not 0)
+        task = resolved_task.task
+        for id, sample in enumerate(task.dataset):
+            if sample.id is None:
+                sample.id = id + 1
+
+        # Ensure sample ids are unique
+        ensure_unique_ids(task.dataset)
+
     # run startup pass for the sandbox environments
     shutdown_sandbox_environments: Callable[[], Awaitable[None]] | None = None
     if has_sandbox:
         cleanup = eval_config.sandbox_cleanup is not False
         shutdown_sandbox_environments = await startup_sandbox_environments(
-            resolve_sandbox_environment(eval_sandbox), tasks, cleanup
+            resolve_sandbox_environment(eval_sandbox), tasks, eval_config, cleanup
         )
 
     # resolve solver and solver spec
@@ -146,14 +157,6 @@ async def eval_run(
                 else:
                     task.fail_on_error = task_eval_config.fail_on_error
 
-                # add sample ids to dataset if they aren't there (start at 1 not 0)
-                for id, sample in enumerate(task.dataset):
-                    if sample.id is None:
-                        sample.id = id + 1
-
-                # Ensure sample ids are unique
-                ensure_unique_ids(task.dataset)
-
                 # create and track the logger
                 logger = TaskLogger(
                     task_name=task.name,
@@ -340,13 +343,15 @@ async def run_multiple(tasks: list[TaskRunOptions], parallel: int) -> list[EvalL
 async def startup_sandbox_environments(
     eval_sandbox: SandboxEnvironmentSpec | None,
     tasks: list[ResolvedTask],
+    config: EvalConfig,
     cleanup: bool,
 ) -> Callable[[], Awaitable[None]]:
     # find unique sandboxenvs
     sandboxenvs: Set[TaskSandboxEnvironment] = set()
     for task in tasks:
         # resolve each sample and add to sandboxenvs
-        for sample in task.task.dataset:
+        dataset = slice_dataset(task.task.dataset, config.limit, config.sample_id)
+        for sample in dataset:
             sandbox = resolve_sandbox_for_task(eval_sandbox, task.task, sample)
             if sandbox is not None and sandbox not in sandboxenvs:
                 sandboxenvs.add(sandbox)

inspect_ai/_eval/task/__init__.py

@@ -1,4 +1,4 @@
-from .task import Task, TaskInfo, PreviousTask, Tasks  # noqa: I001, F401
+from .task import Task, TaskInfo, PreviousTask, Tasks, task_with  # noqa: I001, F401
 from .epochs import Epochs
 
-__all__ = ["Epochs", "Task", "TaskInfo", "PreviousTask", "Tasks"]
+__all__ = ["Epochs", "Task", "TaskInfo", "PreviousTask", "Tasks", "task_with"]