inspect-ai 0.3.99__py3-none-any.whl → 0.3.101__py3-none-any.whl

inspect_ai/_cli/eval.py

@@ -35,7 +35,7 @@ from .util import (
 )
 
 MAX_SAMPLES_HELP = "Maximum number of samples to run in parallel (default is running all samples in parallel)"
-MAX_TASKS_HELP = "Maximum number of tasks to run in parallel (default is 1)"
+MAX_TASKS_HELP = "Maximum number of tasks to run in parallel (default is 1 for eval and 4 for eval-set)"
 MAX_SUBPROCESSES_HELP = (
     "Maximum number of subprocesses to run in parallel (default is os.cpu_count())"
 )
@@ -949,6 +949,7 @@ def eval_exec(
             log_images=log_images,
             log_buffer=log_buffer,
             log_shared=log_shared,
+            log_header_only=True,  # cli invocation doesn't need full log
             score=score,
             score_display=score_display,
         )

inspect_ai/_display/core/config.py

@@ -1,4 +1,8 @@
+from rich.console import RenderableType
+from rich.text import Text
+
 from inspect_ai._util.registry import is_model_dict, is_registry_dict
+from inspect_ai._util.text import truncate_text
 from inspect_ai.log._log import eval_config_defaults
 
 from .display import TaskProfile
@@ -6,7 +10,7 @@ from .display import TaskProfile
 
 def task_config(
     profile: TaskProfile, generate_config: bool = True, style: str = ""
-) -> str:
+) -> RenderableType:
     # merge config
     # wind params back for display
     task_args = dict(profile.task_args)
@@ -39,15 +43,17 @@ def task_config(
         elif name not in ["limit", "model", "response_schema", "log_shared"]:
             if isinstance(value, list):
                 value = ",".join([str(v) for v in value])
+            elif isinstance(value, dict):
+                value = "{...}"
             if isinstance(value, str):
+                value = truncate_text(value, 50)
                 value = value.replace("[", "\\[")
             config_print.append(f"{name}: {value}")
     values = ", ".join(config_print)
     if values:
-        if style:
-            return f"[{style}]{values}[/{style}]"
-        else:
-            return values
+        values_text = Text(values, style=style)
+        values_text.truncate(500, overflow="ellipsis")
+        return values_text
     else:
         return ""
 

inspect_ai/_display/core/panel.py

@@ -9,6 +9,7 @@ from rich.text import Text
 from inspect_ai._util.constants import CONSOLE_DISPLAY_WIDTH
 from inspect_ai._util.path import cwd_relative_path
 from inspect_ai._util.registry import registry_unqualified_name
+from inspect_ai.util._display import display_type
 
 from .display import TaskProfile
 from .rich import is_vscode_notebook, rich_theme
@@ -24,7 +25,13 @@ def task_panel(
     | None,
     footer: RenderableType | tuple[RenderableType, RenderableType] | None,
     log_location: str | None,
-) -> Panel:
+) -> RenderableType:
+    # dispatch to plain handler if we are in plain mode
+    if display_type() == "plain":
+        return task_panel_plain(
+            profile, show_model, body, subtitle, footer, log_location
+        )
+
     # rendering context
     theme = rich_theme()
     console = rich.get_console()
@@ -93,7 +100,7 @@ def task_panel(
     # create panel w/ title
     panel = Panel(
         root,
-        title=f"[bold][{theme.meta}]{task_title(profile, show_model)}[/{theme.meta}][/bold]",
+        title=task_panel_title(profile, show_model),
         title_align="left",
         width=width,
         expand=True,
@@ -101,6 +108,63 @@ def task_panel(
     return panel
 
 
+def task_panel_plain(
+    profile: TaskProfile,
+    show_model: bool,
+    body: RenderableType,
+    subtitle: RenderableType
+    | str
+    | Tuple[RenderableType | str, RenderableType | str]
+    | None,
+    footer: RenderableType | tuple[RenderableType, RenderableType] | None,
+    log_location: str | None,
+) -> RenderableType:
+    # delimiter text
+    delimeter = "---------------------------------------------------------"
+
+    # root table for output
+    table = Table.grid(expand=False)
+    table.add_column()
+    table.add_row(delimeter)
+
+    # title and subtitle
+    table.add_row(task_panel_title(profile, show_model))
+    if isinstance(subtitle, tuple):
+        subtitle = subtitle[0]
+    table.add_row(subtitle)
+
+    # task info
+    if body:
+        table.add_row(body)
+
+    # footer
+    if isinstance(footer, tuple):
+        footer = footer[0]
+    if footer:
+        table.add_row(footer)
+
+    # log location
+    if log_location:
+        # Print a cwd relative path
+        try:
+            log_location_relative = cwd_relative_path(log_location, walk_up=True)
+        except ValueError:
+            log_location_relative = log_location
+        table.add_row(f"Log: {log_location_relative}")
+
+    table.add_row(delimeter)
+    table.add_row("")
+
+    return table
+
+
+def task_panel_title(profile: TaskProfile, show_model: bool) -> str:
+    theme = rich_theme()
+    return (
+        f"[bold][{theme.meta}]{task_title(profile, show_model)}[/{theme.meta}][/bold]"
+    )
+
+
 def to_renderable(item: RenderableType | str, style: str = "") -> RenderableType:
     if isinstance(item, str):
         return Text.from_markup(item, style=style)

inspect_ai/_display/core/textual.py

@@ -8,8 +8,6 @@ logger = getLogger(__name__)
 # force mouse support for textual -- this works around an issue where
 # mouse events are disabled after a reload of the vs code ide, see:
 #   https://github.com/Textualize/textual/issues/5380
-# ansi codes for enabling mouse support are idempotent so it is fine
-# to do this even in cases where mouse support is already enabled.
 # we try/catch since we aren't 100% sure there aren't cases where doing
 # this won't raise and we'd rather not fail hard in in these case
 def textual_enable_mouse_support(driver: Driver) -> None:
@@ -17,5 +15,10 @@ def textual_enable_mouse_support(driver: Driver) -> None:
     if enable_mouse_support:
         try:
             enable_mouse_support()
+            # Re-enable SGR-Pixels format if it was previously enabled.
+            # See #1943.
+            enable_mouse_pixels = getattr(driver, "_enable_mouse_pixels", None)
+            if enable_mouse_pixels and getattr(driver, "_mouse_pixels", False):
+                enable_mouse_pixels()
         except Exception as ex:
             logger.warning(f"Error enabling mouse support: {ex}")

inspect_ai/_display/plain/display.py

@@ -208,3 +208,4 @@ class PlainTaskDisplay(TaskDisplay):
     def complete(self, result: TaskResult) -> None:
         self.task.result = result
         self._print_status()
+        print("")

inspect_ai/_display/rich/display.py

@@ -341,8 +341,6 @@ def tasks_live_status(
 
     # get config
     config = task_config(tasks[0].profile, generate_config=False, style=theme.light)
-    if config:
-        config += "\n"
 
     # build footer table
     footer_table = Table.grid(expand=True)
@@ -356,6 +354,8 @@ def tasks_live_status(
     layout_table = Table.grid(expand=True)
     layout_table.add_column()
     layout_table.add_row(config)
+    if config:
+        layout_table.add_row("")
     layout_table.add_row(progress)
     layout_table.add_row(footer_table)
 

inspect_ai/_display/textual/widgets/transcript.py

@@ -105,9 +105,28 @@ class TranscriptView(ScrollableContainer):
         self, events: Sequence[Event], limit: int = 10
     ) -> list[Widget]:
         widgets: list[Widget] = []
-        widget_count = 0
-        # reverse the events so that the N most recents events are displayed
-        for event in events[::-1]:
+
+        # filter the events to the <limit> most recent
+        filtered_events = events
+        if len(events) > limit:
+            filtered_events = filtered_events[-limit:]
+
+        # find the sample init event
+        sample_init: SampleInitEvent | None = None
+        for event in events:
+            if isinstance(event, SampleInitEvent):
+                sample_init = event
+                break
+
+        # add the sample init event if it isn't already in the event list
+        if sample_init and sample_init not in filtered_events:
+            filtered_events = [sample_init] + list(filtered_events)
+
+        # compute how many events we filtered out
+        filtered_count = len(events) - len(filtered_events)
+        showed_filtered_count = False
+
+        for event in filtered_events:
             display = render_event(event)
             if display:
                 for d in display:
@@ -123,14 +142,23 @@ class TranscriptView(ScrollableContainer):
                             set_transcript_markdown_options(d.content)
                         widgets.append(Static(d.content, markup=False))
                         widgets.append(Static(Text(" ")))
-                        widget_count += 1
 
-            # only render the N most recent events
-            if widget_count >= limit:
-                break
+                        if not showed_filtered_count and filtered_count > 0:
+                            showed_filtered_count = True
+
+                            widgets.append(
+                                Static(
+                                    transcript_separator(
+                                        f"{filtered_count} events..."
+                                        if filtered_count > 1
+                                        else "1 event...",
+                                        self.app.current_theme.primary,
+                                    )
+                                )
+                            )
+                            widgets.append(Static(Text(" ")))
 
-        # reverse the list since we added the events in reverse order
-        return widgets[::-1]
+        return widgets
 
 
 class EventDisplay(NamedTuple):

inspect_ai/_eval/eval.py

@@ -105,6 +105,7 @@ def eval(
     log_images: bool | None = None,
     log_buffer: int | None = None,
     log_shared: bool | int | None = None,
+    log_header_only: bool | None = None,
     score: bool = True,
     score_display: bool | None = None,
     **kwargs: Unpack[GenerateConfigArgs],
@@ -181,6 +182,8 @@ def eval(
         log_shared: Sync sample events to log directory so that users on other systems
             can see log updates in realtime (defaults to no syncing). Specify `True`
             to sync every 10 seconds, otherwise an integer to sync every `n` seconds.
+        log_header_only: If `True`, the function should return only log headers rather
+            than full logs with samples (defaults to `False`).
         score: Score output (defaults to True)
         score_display: Show scoring metrics in realtime (defaults to True)
         **kwargs: Model generation options.
@@ -234,6 +237,7 @@ def eval(
                 log_images=log_images,
                 log_buffer=log_buffer,
                 log_shared=log_shared,
+                log_header_only=log_header_only,
                 score=score,
                 score_display=score_display,
                 **kwargs,
@@ -288,6 +292,7 @@ async def eval_async(
     log_images: bool | None = None,
     log_buffer: int | None = None,
     log_shared: bool | int | None = None,
+    log_header_only: bool | None = None,
     score: bool = True,
     score_display: bool | None = None,
     **kwargs: Unpack[GenerateConfigArgs],
@@ -344,7 +349,9 @@ async def eval_async(
         log_buffer: Number of samples to buffer before writing log file.
            If not specified, an appropriate default for the format and filesystem is
            chosen (10 for most all cases, 100 for JSON logs on remote filesystems).
-        log_shared: Indicate that the log directory is shared, which results in additional syncing of realtime log data for Inspect View.
+        log_shared: Indicate that the log directory is shared, which results in additional
+        syncing of realtime log data for Inspect View.
+        log_header_only: If `True`, the function should return only log headers rather than full logs with samples (defaults to `False`).
         score: Score output (defaults to True)
         score_display: Show scoring metrics in realtime (defaults to True)
         **kwargs: Model generation options.
@@ -432,6 +439,9 @@ async def eval_async(
         # resolve log_shared
         log_shared = DEFAULT_LOG_SHARED if log_shared is True else log_shared
 
+        # resolve header only
+        log_header_only = log_header_only is True
+
         # validate that --log-shared can't use used with 'json' format
         if log_shared and log_format == JSON_LOG_FORMAT:
             raise PrerequisiteError(
@@ -507,6 +517,7 @@ async def eval_async(
                         eval_config=eval_config,
                         eval_sandbox=sandbox,
                         recorder=recorder,
+                        header_only=log_header_only,
                         epochs_reducer=epochs_reducer,
                         solver=solver,
                         tags=tags,
@@ -532,6 +543,7 @@ async def eval_async(
                 eval_config=eval_config,
                 eval_sandbox=sandbox,
                 recorder=recorder,
+                header_only=log_header_only,
                 epochs_reducer=epochs_reducer,
                 solver=solver,
                 tags=tags,

inspect_ai/_eval/evalset.py

@@ -168,7 +168,7 @@ def eval_set(
         max_samples: Maximum number of samples to run in parallel
             (default is max_connections)
         max_tasks: Maximum number of tasks to run in parallel
-            (defaults to number of models being evaluated)
+            (defaults to the greater of 4 and the number of models being evaluated)
         max_subprocesses: Maximum number of subprocesses to
             run in parallel (default is os.cpu_count())
         max_sandboxes: Maximum number of sandboxes (per-provider)
@@ -235,6 +235,7 @@ def eval_set(
             log_images=log_images,
             log_buffer=log_buffer,
             log_shared=log_shared,
+            log_header_only=True,
             score=score,
             **kwargs,
         )
@@ -277,7 +278,7 @@ def eval_set(
     retry_connections = retry_connections or 0.5
     retry_cleanup = retry_cleanup is not False
     max_connections = starting_max_connections(models, GenerateConfig(**kwargs))
-    max_tasks = max_tasks if max_tasks is not None else len(models)
+    max_tasks = max_tasks if max_tasks is not None else max(len(models), 4)
 
     # prepare console/status
     console = rich.get_console()

inspect_ai/_eval/run.py

@@ -63,6 +63,7 @@ async def eval_run(
     eval_config: EvalConfig,
     eval_sandbox: SandboxEnvironmentType | None,
     recorder: Recorder,
+    header_only: bool,
     epochs_reducer: list[ScoreReducer] | None = None,
     solver: Solver | SolverSpec | None = None,
     tags: list[str] | None = None,
@@ -212,6 +213,7 @@ async def eval_run(
                     eval_config=task_eval_config,
                     metadata=((metadata or {}) | (task.metadata or {})) or None,
                     recorder=recorder,
+                    header_only=header_only,
                 )
                 await logger.init()
 

inspect_ai/_eval/score.py

@@ -1,7 +1,7 @@
 import functools
 from copy import deepcopy
 from pathlib import Path
-from typing import Any, Callable, Literal, cast
+from typing import Any, Callable, Literal
 
 import anyio
 
@@ -270,9 +270,7 @@ def metrics_from_log(log: EvalLog) -> list[Metric] | dict[str, list[Metric]] | N
 
 
 def metric_from_log(metric: EvalMetricDefinition) -> Metric:
-    return cast(
-        Metric, registry_create("metric", metric.name, **(metric.options or {}))
-    )
+    return registry_create("metric", metric.name, **(metric.options or {}))
 
 
 def reducers_from_log(log: EvalLog) -> list[ScoreReducer] | None:

inspect_ai/_eval/task/log.py

@@ -75,6 +75,7 @@ class TaskLogger:
         eval_config: EvalConfig,
         metadata: dict[str, Any] | None,
         recorder: Recorder,
+        header_only: bool,
     ) -> None:
         # determine versions
         git = git_context()
@@ -153,6 +154,7 @@ class TaskLogger:
 
         # stack recorder and location
         self.recorder = recorder
+        self.header_only = header_only
 
         # number of samples logged
         self._samples_completed = 0
@@ -238,7 +240,7 @@ class TaskLogger:
     ) -> EvalLog:
         # finish and get log
         log = await self.recorder.log_finish(
-            self.eval, status, stats, results, reductions, error
+            self.eval, status, stats, results, reductions, error, self.header_only
         )
 
         # cleanup the events db

inspect_ai/_eval/task/run.py

@@ -35,11 +35,7 @@ from inspect_ai._util.registry import (
     registry_log_name,
     registry_unqualified_name,
 )
-from inspect_ai._util.working import (
-    end_sample_working_limit,
-    init_sample_working_limit,
-    sample_waiting_time,
-)
+from inspect_ai._util.working import init_sample_working_time, sample_waiting_time
 from inspect_ai._view.notify import view_notify_eval
 from inspect_ai.dataset import Dataset, Sample
 from inspect_ai.log import (
@@ -90,6 +86,8 @@ from inspect_ai.solver._fork import set_task_generate
 from inspect_ai.solver._solver import Solver
 from inspect_ai.solver._task_state import sample_state, set_sample_state, state_jsonable
 from inspect_ai.util._limit import LimitExceededError
+from inspect_ai.util._limit import time_limit as create_time_limit
+from inspect_ai.util._limit import working_limit as create_working_limit
 from inspect_ai.util._sandbox.context import sandbox_connections
 from inspect_ai.util._sandbox.environment import SandboxEnvironmentSpec
 from inspect_ai.util._span import span
@@ -635,10 +633,6 @@ async def task_run_sample(
             )
 
             async with sandboxenv_cm:
-                timeout_cm: (
-                    contextlib._GeneratorContextManager[anyio.CancelScope]
-                    | contextlib.nullcontext[None]
-                ) = contextlib.nullcontext()
                 try:
                     # update active sample wth sandboxes now that we are initialised
                     # (ensure that we still exit init context in presence of sandbox error)
@@ -647,19 +641,17 @@ async def task_run_sample(
                     finally:
                         await init_span.__aexit__(None, None, None)
 
-                    # initialise timeout context manager
-                    timeout_cm = (
-                        anyio.fail_after(time_limit)
-                        if time_limit is not None
-                        else contextlib.nullcontext()
-                    )
-
                     # record start time
                     start_time = time.monotonic()
-                    init_sample_working_limit(start_time, working_limit)
-
-                    # run sample w/ optional timeout
-                    with timeout_cm, state._token_limit, state._message_limit:
+                    init_sample_working_time(start_time)
+
+                    # run sample w/ optional limits
+                    with (
+                        state._token_limit,
+                        state._message_limit,
+                        create_time_limit(time_limit),
+                        create_working_limit(working_limit),
+                    ):
                         # mark started
                         active.started = datetime.now().timestamp()
 
@@ -675,24 +667,15 @@ async def task_run_sample(
                             )
 
                         # set progress for plan then run it
-                        state = await plan(state, generate)
-
-                    # disable sample working limit after execution
-                    end_sample_working_limit()
+                        async with span("solvers"):
+                            state = await plan(state, generate)
 
                 except TimeoutError:
-                    if time_limit is not None:
-                        transcript()._event(
-                            SampleLimitEvent(
-                                type="time",
-                                message=f"Sample completed: exceeded time limit ({time_limit:,} seconds)",
-                                limit=time_limit,
-                            )
-                        )
-                    else:
-                        py_logger.warning(
-                            "Unexpected timeout error reached top of sample stack. Are you handling TimeoutError when applying timeouts?"
-                        )
+                    # Scoped time limits manifest themselves as LimitExceededError, not
+                    # TimeoutError.
+                    py_logger.warning(
+                        "Unexpected timeout error reached top of sample stack. Are you handling TimeoutError when applying timeouts?"
+                    )
 
                     # capture most recent state for scoring
                     state = sample_state() or state
@@ -737,54 +720,59 @@ async def task_run_sample(
                 # the cause of the timeout is a hung container and scoring requires
                 # interacting with the container). as a middle ground we use half
                 # of the original timeout value for scoring.
-                if time_limit is not None:
-                    timeout_cm = anyio.fail_after(time_limit / 2)
+                scoring_time_limit = time_limit / 2 if time_limit else None
 
                 set_sample_state(state)
 
                 # scoring
                 try:
                     # timeout during scoring will result in an ordinary sample error
-                    with timeout_cm:
+                    with create_time_limit(scoring_time_limit):
                         if error is None:
-                            for scorer in scorers or []:
-                                scorer_name = unique_scorer_name(
-                                    scorer, list(results.keys())
-                                )
-                                async with span(name=scorer_name, type="scorer"):
-                                    score_result = (
-                                        await scorer(state, Target(sample.target))
-                                        if scorer
-                                        else None
+                            async with span(name="scorers"):
+                                for scorer in scorers or []:
+                                    scorer_name = unique_scorer_name(
+                                        scorer, list(results.keys())
                                     )
-                                    if score_result is not None:
-                                        sample_score = SampleScore(
-                                            score=score_result,
-                                            sample_id=sample.id,
-                                            sample_metadata=sample.metadata,
-                                            scorer=registry_unqualified_name(scorer),
+                                    async with span(name=scorer_name, type="scorer"):
+                                        score_result = (
+                                            await scorer(state, Target(sample.target))
+                                            if scorer
+                                            else None
+                                        )
+                                        if score_result is not None:
+                                            sample_score = SampleScore(
+                                                score=score_result,
+                                                sample_id=sample.id,
+                                                sample_metadata=sample.metadata,
+                                                scorer=registry_unqualified_name(
+                                                    scorer
+                                                ),
+                                            )
+                                            transcript()._event(
+                                                ScoreEvent(
+                                                    score=score_result,
+                                                    target=sample.target,
+                                                )
+                                            )
+                                            results[scorer_name] = sample_score
+
+                                # add scores returned by solvers
+                                if state.scores is not None:
+                                    for name, score in state.scores.items():
+                                        results[name] = SampleScore(
+                                            score=score,
+                                            sample_id=state.sample_id,
+                                            sample_metadata=state.metadata,
                                         )
                                         transcript()._event(
                                             ScoreEvent(
-                                                score=score_result, target=sample.target
+                                                score=score, target=sample.target
                                             )
                                         )
-                                        results[scorer_name] = sample_score
-
-                            # add scores returned by solvers
-                            if state.scores is not None:
-                                for name, score in state.scores.items():
-                                    results[name] = SampleScore(
-                                        score=score,
-                                        sample_id=state.sample_id,
-                                        sample_metadata=state.metadata,
-                                    )
-                                    transcript()._event(
-                                        ScoreEvent(score=score, target=sample.target)
-                                    )
 
-                            # propagate results into scores
-                            state.scores = {k: v.score for k, v in results.items()}
+                                # propagate results into scores
+                                state.scores = {k: v.score for k, v in results.items()}
 
                 except anyio.get_cancelled_exc_class():
                     if active.interrupt_action:
@@ -798,17 +786,7 @@ async def task_run_sample(
                     raise
 
                 except BaseException as ex:
-                    # note timeout
-                    if isinstance(ex, TimeoutError):
-                        transcript()._event(
-                            SampleLimitEvent(
-                                type="time",
-                                message=f"Unable to score sample due to exceeded time limit ({time_limit:,} seconds)",
-                                limit=time_limit,
-                            )
-                        )
-
-                    # handle error (this will throw if we've exceeded the limit)
+                    # handle error
                     error, raise_error = handle_error(ex)
 
         except Exception as ex: