inspect-ai 0.3.103__py3-none-any.whl → 0.3.105__py3-none-any.whl

inspect_ai/_cli/common.py

@@ -60,7 +60,8 @@ def common_options(func: Callable[..., Any]) -> Callable[..., click.Context]:
     @click.option(
         "--display",
         type=click.Choice(
-            ["full", "conversation", "rich", "plain", "none"], case_sensitive=False
+            ["full", "conversation", "rich", "plain", "log", "none"],
+            case_sensitive=False,
         ),
         default=DEFAULT_DISPLAY,
         envvar="INSPECT_DISPLAY",

inspect_ai/_cli/eval.py

@@ -641,7 +641,7 @@ def eval_command(
 @click.option(
     "--retry-connections",
     type=float,
-    help="Reduce max_connections at this rate with each retry (defaults to 0.5)",
+    help="Reduce max_connections at this rate with each retry (defaults to 1.0, which results in no reduction).",
     envvar="INSPECT_EVAL_RETRY_CONNECTIONS",
 )
 @click.option(
@@ -966,7 +966,7 @@ def eval_exec(
         success, _ = eval_set(**params)
         return success
     else:
-        params["log_header_only"] = (True,)  # cli invocation doesn't need full log
+        params["log_header_only"] = True  # cli invocation doesn't need full log
         eval(**params)
         return True
 

inspect_ai/_display/core/active.py

@@ -5,6 +5,7 @@ import rich
 
 from inspect_ai.util._display import display_type
 
+from ..log.display import LogDisplay
 from ..plain.display import PlainDisplay
 from ..rich.display import RichDisplay
 from ..textual.display import TextualDisplay
@@ -24,6 +25,8 @@ def display() -> Display:
             and not rich.get_console().is_jupyter
         ):
             _active_display = TextualDisplay()
+        elif display_type() == "log":
+            _active_display = LogDisplay()
         else:
             _active_display = RichDisplay()
 

inspect_ai/_display/core/config.py

@@ -30,6 +30,7 @@ def task_config(
         config = dict(profile.generate_config.model_dump(exclude_none=True)) | config
     if profile.tags:
         config["tags"] = ",".join(profile.tags)
+    config["dataset"] = profile.dataset
     config_print: list[str] = []
     for name, value in config.items():
         if name == "approval" and isinstance(value, dict):

inspect_ai/_display/core/panel.py

@@ -1,7 +1,7 @@
 from typing import Tuple
 
 import rich
-from rich.console import RenderableType
+from rich.console import Group, RenderableType
 from rich.panel import Panel
 from rich.table import Table
 from rich.text import Text
@@ -9,7 +9,7 @@ from rich.text import Text
 from inspect_ai._util.constants import CONSOLE_DISPLAY_WIDTH
 from inspect_ai._util.path import cwd_relative_path
 from inspect_ai._util.registry import registry_unqualified_name
-from inspect_ai.util._display import display_type
+from inspect_ai.util._display import display_type_plain
 
 from .display import TaskProfile
 from .rich import is_vscode_notebook, rich_theme
@@ -27,7 +27,7 @@ def task_panel(
     log_location: str | None,
 ) -> RenderableType:
     # dispatch to plain handler if we are in plain mode
-    if display_type() == "plain":
+    if display_type_plain():
         return task_panel_plain(
             profile, show_model, body, subtitle, footer, log_location
         )
@@ -89,23 +89,31 @@ def task_panel(
             log_location_relative = log_location
 
         root = Table.grid(expand=True)
-        root.add_column()
+        root.add_column(overflow="fold")
         root.add_row(table)
         root.add_row()
         root.add_row(
             f"[bold][{theme.light}]Log:[/{theme.light}][/bold] "
             + f"[{theme.link}]{log_location_relative}[/{theme.link}]"
         )
+        root.add_row()
 
-    # create panel w/ title
-    panel = Panel(
-        root,
-        title=task_panel_title(profile, show_model),
-        title_align="left",
-        width=width,
-        expand=True,
-    )
-    return panel
+        panel = Panel(
+            task_panel_title(profile, show_model),
+            padding=(0, 0),
+            width=width,
+            height=3,
+            expand=True,
+        )
+        return Group(panel, root)
+    else:
+        return Panel(
+            root,
+            title=task_panel_title(profile, show_model),
+            title_align="left",
+            width=width,
+            expand=True,
+        )
 
 
 def task_panel_plain(

inspect_ai/_display/core/results.py

@@ -18,7 +18,7 @@ from .display import (
     TaskSuccess,
     TaskWithResult,
 )
-from .panel import task_panel, task_targets
+from .panel import task_panel
 from .rich import rich_theme
 
 
@@ -41,8 +41,6 @@ def task_result_cancelled(
 ) -> RenderableType:
     # The contents of the panel
     config = task_config(profile)
-    targets = task_targets(profile)
-    subtitle = config, targets
     body = task_stats(cancelled.stats)
 
     # The panel
@@ -50,7 +48,7 @@ def task_result_cancelled(
         profile=profile,
         show_model=True,
         body=body,
-        subtitle=subtitle,
+        subtitle=config,
         footer=task_interrupted(profile, cancelled.samples_completed),
         log_location=profile.log_location,
     )
@@ -76,8 +74,6 @@ def task_results(profile: TaskProfile, success: TaskSuccess) -> RenderableType:
 def task_result_summary(profile: TaskProfile, success: TaskSuccess) -> RenderableType:
     # The contents of the panel
     config = task_config(profile)
-    targets = task_targets(profile)
-    subtitle = config, targets
     body = task_stats(success.stats)
 
     # the panel
@@ -85,7 +81,7 @@ def task_result_summary(profile: TaskProfile, success: TaskSuccess) -> Renderabl
         profile=profile,
         show_model=True,
         body=body,
-        subtitle=subtitle,
+        subtitle=config,
         footer=task_results(profile, success),
         log_location=profile.log_location,
     )

inspect_ai/_display/core/rich.py

@@ -11,7 +11,7 @@ from typing_extensions import override
 
 from inspect_ai._util.platform import is_running_in_jupyterlab, is_running_in_vscode
 from inspect_ai._util.transcript import transcript_code_theme
-from inspect_ai.util._display import display_type
+from inspect_ai.util._display import display_type, display_type_plain
 
 
 def is_vscode_notebook(console: Console) -> bool:
@@ -20,15 +20,13 @@ def is_vscode_notebook(console: Console) -> bool:
 
 def rich_no_color() -> bool:
     return (
-        display_type() == "plain"
-        or not is_running_in_vscode()
-        or is_running_in_jupyterlab()
+        display_type_plain() or not is_running_in_vscode() or is_running_in_jupyterlab()
     )
 
 
 def rich_initialise() -> None:
     # reflect ansi prefs
-    if display_type() == "plain":
+    if display_type_plain():
         rich.reconfigure(no_color=True, force_terminal=False, force_interactive=False)
     elif rich_no_color():
         rich.reconfigure(no_color=True)

inspect_ai/_display/log/display.py

@@ -0,0 +1,173 @@
+import contextlib
+import logging
+from typing import AsyncIterator, Callable, Coroutine, Iterator
+
+import anyio
+from rich.console import Console
+
+from inspect_ai._util._async import configured_async_backend, run_coroutine
+from inspect_ai._util.platform import running_in_notebook
+
+from ...util import throttle
+from ...util._concurrency import concurrency_status_display
+from ..core.display import (
+    TR,
+    Display,
+    Progress,
+    TaskDisplay,
+    TaskDisplayMetric,
+    TaskProfile,
+    TaskResult,
+    TaskScreen,
+    TaskSpec,
+    TaskWithResult,
+)
+from ..core.footer import task_http_retries_str
+from ..core.results import task_metric, tasks_results
+
+
+class LogDisplay(Display):
+    def __init__(self) -> None:
+        self.total_tasks: int = 0
+        self.tasks: list[TaskWithResult] = []
+        self.parallel = False
+
+    def print(self, message: str) -> None:
+        logging.info(message, stacklevel=2)
+
+    @contextlib.contextmanager
+    def progress(self, total: int) -> Iterator[Progress]:
+        yield LogProgress(total)
+
+    def run_task_app(self, main: Callable[[], Coroutine[None, None, TR]]) -> TR:
+        if running_in_notebook():
+            return run_coroutine(main())
+        else:
+            return anyio.run(main, backend=configured_async_backend())
+
+    @contextlib.contextmanager
+    def suspend_task_app(self) -> Iterator[None]:
+        yield
+
+    @contextlib.asynccontextmanager
+    async def task_screen(
+        self, tasks: list[TaskSpec], parallel: bool
+    ) -> AsyncIterator[TaskScreen]:
+        self.total_tasks = len(tasks)
+        self.tasks = []
+        self.parallel = parallel
+        try:
+            logging.info(f"Running {self.total_tasks} tasks...", stacklevel=3)
+            yield TaskScreen()
+        finally:
+            # Log final results
+            if self.tasks:
+                self._log_results()
+
+    @contextlib.contextmanager
+    def task(self, profile: TaskProfile) -> Iterator[TaskDisplay]:
+        # Create and yield task display
+        task = TaskWithResult(profile, None)
+        self.tasks.append(task)
+        yield LogTaskDisplay(task)
+        self._log_status()
+
+    def display_counter(self, caption: str, value: str) -> None:
+        logging.info(f"{caption}: {value}", stacklevel=2)
+
+    def _log_status(self) -> None:
+        """Log status updates for all tasks"""
+        completed_tasks = sum(1 for task in self.tasks if task.result is not None)
+        total_tasks = len(self.tasks)
+        logging.info(f"{completed_tasks}/{total_tasks} tasks complete", stacklevel=4)
+
+    def _log_results(self) -> None:
+        """Log final results"""
+        results = tasks_results(self.tasks)
+        console = Console(width=120)
+        console.log(results, _stack_offset=4)
+
+
+class LogProgress(Progress):
+    def __init__(self, total: int):
+        self.total = total
+        self.current = 0
+
+    def update(self, n: int = 1) -> None:
+        self.current += n
+
+    def complete(self) -> None:
+        self.current = self.total
+
+
+class LogTaskDisplay(TaskDisplay):
+    def __init__(self, task: TaskWithResult):
+        self.task = task
+        self.progress_display: LogProgress | None = None
+        self.samples_complete = 0
+        self.samples_total = 0
+        self.current_metrics: list[TaskDisplayMetric] | None = None
+
+    @contextlib.contextmanager
+    def progress(self) -> Iterator[Progress]:
+        self.progress_display = LogProgress(self.task.profile.steps)
+        yield self.progress_display
+
+    @throttle(5)
+    def _log_status_throttled(self, stacklevel: int) -> None:
+        self._log_status(stacklevel=stacklevel + 2)
+
+    def _log_status(self, stacklevel: int) -> None:
+        """Log status updates"""
+        status_parts: list[str] = []
+
+        # Add task name and model
+        status_parts.append(f"Task: {self.task.profile.name}")
+        status_parts.append(f"Model: {self.task.profile.model}")
+
+        # Add step progress
+        if self.progress_display:
+            progress_percent = int(
+                self.progress_display.current / self.progress_display.total * 100
+            )
+            status_parts.append(
+                f"Steps: {self.progress_display.current}/{self.progress_display.total} {progress_percent}%"
+            )
+
+        # Add sample progress
+        status_parts.append(f"Samples: {self.samples_complete}/{self.samples_total}")
+
+        # Add metrics
+        if self.current_metrics:
+            metric_str = task_metric(self.current_metrics)
+            status_parts.append(metric_str)
+
+        # Add resource usage
+        resources_dict: dict[str, str] = {}
+        for model, resource in concurrency_status_display().items():
+            resources_dict[model] = f"{resource[0]}/{resource[1]}"
+        resources = ", ".join(
+            [f"{key}: {value}" for key, value in resources_dict.items()]
+        )
+        status_parts.append(resources)
+
+        # Add rate limits
+        rate_limits = task_http_retries_str()
+        if rate_limits:
+            status_parts.append(rate_limits)
+
+        # Print on new line
+        logging.info(", ".join(status_parts), stacklevel=stacklevel)
+
+    def sample_complete(self, complete: int, total: int) -> None:
+        self.samples_complete = complete
+        self.samples_total = total
+        self._log_status_throttled(stacklevel=3)
+
+    def update_metrics(self, metrics: list[TaskDisplayMetric]) -> None:
+        self.current_metrics = metrics
+        self._log_status_throttled(stacklevel=3)
+
+    def complete(self, result: TaskResult) -> None:
+        self.task.result = result
+        self._log_status(stacklevel=3)

inspect_ai/_display/plain/display.py

@@ -25,7 +25,7 @@ from ..core.display import (
     TaskWithResult,
 )
 from ..core.footer import task_http_retries_str
-from ..core.panel import task_panel, task_targets
+from ..core.panel import task_panel
 from ..core.results import task_metric, tasks_results
 
 
@@ -79,7 +79,7 @@ class PlainDisplay(Display):
             profile=profile,
             show_model=True,
             body="",  # Empty body since we haven't started yet
-            subtitle=(task_config(profile), task_targets(profile)),
+            subtitle=task_config(profile),
             footer=None,
             log_location=None,
         )

inspect_ai/_display/rich/display.py

@@ -32,7 +32,7 @@ from ..core.display import (
     TaskWithResult,
 )
 from ..core.footer import task_footer
-from ..core.panel import task_panel, task_targets, task_title, tasks_title
+from ..core.panel import task_panel, task_title, tasks_title
 from ..core.progress import (
     RichProgress,
     progress_description,
@@ -311,15 +311,13 @@ def task_live_status(
 
     # the panel contents
     config = task_config(tasks[0].profile, style=theme.light)
-    targets = task_targets(tasks[0].profile)
-    subtitle = config, targets
 
     # the panel
     return task_panel(
         profile=tasks[0].profile,
         show_model=len(tasks) == 1,
         body=Group("", progress),
-        subtitle=subtitle,
+        subtitle=config,
         footer=task_footer(counters, theme.light),
         log_location=None,
     )

inspect_ai/_display/textual/app.py

@@ -42,7 +42,7 @@ from ..core.display import (
     TaskWithResult,
 )
 from ..core.footer import task_footer
-from ..core.panel import task_targets, task_title, tasks_title
+from ..core.panel import task_title, tasks_title
 from ..core.rich import record_console_input, rich_initialise, rich_theme
 from .theme import inspect_dark, inspect_light
 from .widgets.console import ConsoleView
@@ -296,13 +296,8 @@ class TaskScreenApp(App[TR]):
             tasks.config = task_config(
                 self._tasks[0].profile, generate_config=not self._parallel
             )
-            if not self._parallel:
-                tasks.targets = task_targets(self._tasks[0].profile)
-            else:
-                tasks.targets = " \n "
         else:
             tasks.config = ""
-            tasks.targets = ""
 
     def update_samples(self) -> None:
         samples_view = self.query_one(SamplesView)

inspect_ai/_display/textual/widgets/task_detail.py

@@ -30,6 +30,8 @@ class TaskDetail(Widget):
         width: 100%;
         height: auto;
         grid-gutter: 1 3;
+        grid-size-columns: 3;
+        grid-columns: 1fr 1fr 1fr;
     }
     """
 
@@ -92,20 +94,6 @@ class TaskDetail(Widget):
         if len(self.by_reducer) == 0:
             return
 
-        # Compute the row and column count
-        row_count = len(self.by_reducer)
-        col_count = len(next(iter(self.by_reducer.values())))
-
-        # If this can fit in a single row, make it fit
-        # otherwise place each reducer on their own row
-        self.grid.styles.grid_columns = "auto"
-        if row_count * col_count < 4:
-            self.grid.styles.grid_size_columns = row_count * col_count
-            self.grid.styles.grid_size_rows = 1
-        else:
-            self.grid.styles.grid_size_columns = col_count
-            self.grid.styles.grid_size_rows = row_count
-
         # In order to reduce flashing the below tracks use of widgets
         # and updates them when possible (removing and adding them as needed)
         # Makes keys for tracking Task Metric widgets
@@ -142,6 +130,7 @@ class TaskMetrics(Widget):
     TaskMetrics {
         width: auto;
         height: auto;
+        border: solid $foreground 20%;
     }
     TaskMetrics Grid {
         width: auto;

inspect_ai/_display/textual/widgets/tasks.py

@@ -174,7 +174,7 @@ class TaskProgressView(Widget):
         color:$text-secondary;
     }
     #task-detail {
-        column-span: 8;
+        column-span: 9;
     }
     .hidden {
         display: none;

inspect_ai/_eval/eval.py

@@ -812,7 +812,7 @@ async def eval_retry_async(
         model_roles = model_roles_config_to_model_roles(eval_log.eval.model_roles)
 
         # collect the rest of the params we need for the eval
-        task_args = eval_log.eval.task_args
+        task_args = eval_log.eval.task_args_passed
         tags = eval_log.eval.tags
         limit = eval_log.eval.config.limit
         sample_id = eval_log.eval.config.sample_id

inspect_ai/_eval/evalset.py

@@ -114,7 +114,7 @@ def eval_set(
             (defaults to 30, resulting in waits of 30, 60, 120, 240, etc.). Wait time
             per-retry will in no case by longer than 1 hour.
         retry_connections: Reduce max_connections at this rate with each retry
-            (defaults to 0.5)
+            (defaults to 1.0, which results in no reduction).
         retry_cleanup: Cleanup failed log files after retries
             (defaults to True)
         model: Model(s) for evaluation. If not specified use the value of the INSPECT_EVAL_MODEL
@@ -275,7 +275,7 @@ def eval_set(
     fs.mkdir(log_dir, exist_ok=True)
 
     # resolve some parameters
-    retry_connections = retry_connections or 0.5
+    retry_connections = retry_connections or 1.0
     retry_cleanup = retry_cleanup is not False
     max_connections = starting_max_connections(models, GenerateConfig(**kwargs))
     max_tasks = max_tasks if max_tasks is not None else max(len(models), 4)
@@ -578,7 +578,7 @@ def task_identifier(task: ResolvedTask | EvalLog) -> str:
     else:
         task_file = task.eval.task_file or ""
         task_name = task.eval.task
-        task_args = task.eval.task_args
+        task_args = task.eval.task_args_passed
         model = str(task.eval.model)
         model_roles = task.eval.model_roles or {}
 

inspect_ai/_eval/registry.py

@@ -8,6 +8,7 @@ from inspect_ai._util.error import PrerequisiteError
 from inspect_ai._util.package import get_installed_package_name
 from inspect_ai._util.registry import (
     RegistryInfo,
+    extract_named_params,
     registry_add,
     registry_create,
     registry_info,
@@ -17,7 +18,7 @@ from inspect_ai._util.registry import (
 )
 
 from .task import Task
-from .task.constants import TASK_FILE_ATTR, TASK_RUN_DIR_ATTR
+from .task.constants import TASK_ALL_PARAMS_ATTR, TASK_FILE_ATTR, TASK_RUN_DIR_ATTR
 
 MODEL_PARAM = "model"
 
@@ -133,6 +134,10 @@ def task(*args: Any, name: str | None = None, **attribs: Any) -> Any:
                 **w_kwargs,
             )
 
+            # extract all task parameters including defaults
+            named_params = extract_named_params(task_type, True, *w_args, **w_kwargs)
+            setattr(task_instance, TASK_ALL_PARAMS_ATTR, named_params)
+
             # if its not from an installed package then it is a "local"
             # module import, so set its task file and run dir
             if get_installed_package_name(task_type) is None:

inspect_ai/_eval/run.py

@@ -3,6 +3,7 @@ import os
 import sys
 from typing import Any, Awaitable, Callable, Set, cast
 
+from inspect_ai._eval.task.constants import TASK_ALL_PARAMS_ATTR
 from inspect_ai._eval.task.task import Task
 from inspect_ai._util.environ import environ_vars
 from inspect_ai._util.trace import trace_action
@@ -208,7 +209,10 @@ async def eval_run(
                     metrics=eval_metrics,
                     sandbox=resolved_task.sandbox,
                     task_attribs=task.attribs,
-                    task_args=resolved_task.task_args,
+                    task_args=getattr(
+                        task, TASK_ALL_PARAMS_ATTR, resolved_task.task_args
+                    ),
+                    task_args_passed=resolved_task.task_args,
                     model_args=resolved_task.model.model_args,
                     eval_config=task_eval_config,
                     metadata=((metadata or {}) | (task.metadata or {})) or None,

inspect_ai/_eval/task/constants.py

@@ -1,2 +1,3 @@
 TASK_FILE_ATTR = "__task_file__"
 TASK_RUN_DIR_ATTR = "__task_run_dir__"
+TASK_ALL_PARAMS_ATTR = "__task_all_params__"

inspect_ai/_eval/task/log.py

@@ -71,6 +71,7 @@ class TaskLogger:
         sandbox: SandboxEnvironmentSpec | None,
         task_attribs: dict[str, Any],
         task_args: dict[str, Any],
+        task_args_passed: dict[str, Any],
         model_args: dict[str, Any],
         eval_config: EvalConfig,
         metadata: dict[str, Any] | None,
@@ -128,6 +129,7 @@ class TaskLogger:
             task_registry_name=task_registry_name,
             task_attribs=task_attribs,
             task_args=task_args,
+            task_args_passed=task_args_passed,
             solver=solver.solver if solver else None,
             tags=tags,
             solver_args=solver.args if solver else None,