inspect-ai 0.3.52__py3-none-any.whl → 0.3.53__py3-none-any.whl

inspect_ai/_cli/eval.py

@@ -30,6 +30,7 @@ MAX_TASKS_HELP = "Maximum number of tasks to run in parallel (default is 1)"
 MAX_SUBPROCESSES_HELP = (
     "Maximum number of subprocesses to run in parallel (default is os.cpu_count())"
 )
+MAX_SANDBOXES_HELP = "Maximum number of sandboxes (per-provider) to run in parallel."
 NO_SANDBOX_CLEANUP_HELP = "Do not cleanup sandbox environments after task completes"
 FAIL_ON_ERROR_HELP = "Threshold of sample errors to tolerage (by default, evals fail when any error occurs). Value between 0 to 1 to set a proportion; value greater than 1 to set a count."
 NO_LOG_SAMPLES_HELP = "Do not include samples in the log file."
@@ -192,6 +193,12 @@ def eval_options(func: Callable[..., Any]) -> Callable[..., click.Context]:
         help=MAX_SUBPROCESSES_HELP,
         envvar="INSPECT_EVAL_MAX_SUBPROCESSES",
     )
+    @click.option(
+        "--max-sandboxes",
+        type=int,
+        help=MAX_SANDBOXES_HELP,
+        envvar="INSPECT_EVAL_MAX_SANDBOXES",
+    )
     @click.option(
         "--message-limit",
         type=int,
@@ -361,6 +368,12 @@ def eval_options(func: Callable[..., Any]) -> Callable[..., click.Context]:
         help='Cache prompt prefix (Anthropic only). Defaults to "auto", which will enable caching for requests with tools.',
         envvar="INSPECT_EVAL_CACHE_PROMPT",
     )
+    @click.option(
+        "--reasoning-effort",
+        type=click.Choice(["low", "medium", "high"]),
+        help="Constrains effort on reasoning for reasoning models. Open AI o1 models only.",
+        envvar="INSPECT_EVAL_REASONING_EFFORT",
+    )
     @click.option(
         "--log-format",
         type=click.Choice(["eval", "json"], case_sensitive=False),
@@ -419,12 +432,14 @@ def eval_command(
     parallel_tool_calls: bool | None,
     max_tool_output: int | None,
     cache_prompt: str | None,
+    reasoning_effort: str | None,
     message_limit: int | None,
     token_limit: int | None,
     time_limit: int | None,
     max_samples: int | None,
     max_tasks: int | None,
     max_subprocesses: int | None,
+    max_sandboxes: int | None,
     fail_on_error: bool | float | None,
     no_fail_on_error: bool | None,
     no_log_samples: bool | None,
@@ -472,6 +487,7 @@ def eval_command(
         max_samples=max_samples,
         max_tasks=max_tasks,
         max_subprocesses=max_subprocesses,
+        max_sandboxes=max_sandboxes,
         fail_on_error=fail_on_error,
         no_fail_on_error=no_fail_on_error,
         debug_errors=common["debug_errors"],
@@ -573,12 +589,14 @@ def eval_set_command(
     parallel_tool_calls: bool | None,
     max_tool_output: int | None,
     cache_prompt: str | None,
+    reasoning_effort: str | None,
     message_limit: int | None,
     token_limit: int | None,
     time_limit: int | None,
     max_samples: int | None,
     max_tasks: int | None,
     max_subprocesses: int | None,
+    max_sandboxes: int | None,
     fail_on_error: bool | float | None,
     no_fail_on_error: bool | None,
     no_log_samples: bool | None,
@@ -628,6 +646,7 @@ def eval_set_command(
         max_samples=max_samples,
         max_tasks=max_tasks,
         max_subprocesses=max_subprocesses,
+        max_sandboxes=max_sandboxes,
         fail_on_error=fail_on_error,
         no_fail_on_error=no_fail_on_error,
         debug_errors=common["debug_errors"],
@@ -679,6 +698,7 @@ def eval_exec(
     max_samples: int | None,
     max_tasks: int | None,
     max_subprocesses: int | None,
+    max_sandboxes: int | None,
     fail_on_error: bool | float | None,
     no_fail_on_error: bool | None,
     debug_errors: bool | None,
@@ -756,6 +776,7 @@ def eval_exec(
             max_samples=max_samples,
             max_tasks=max_tasks,
             max_subprocesses=max_subprocesses,
+            max_sandboxes=max_sandboxes,
             log_samples=log_samples,
             log_images=log_images,
             log_buffer=log_buffer,
@@ -834,6 +855,12 @@ def parse_comma_separated(value: str | None) -> list[str] | None:
     help=MAX_SUBPROCESSES_HELP,
     envvar="INSPECT_EVAL_MAX_SUBPROCESSES",
 )
+@click.option(
+    "--max-sandboxes",
+    type=int,
+    help=MAX_SANDBOXES_HELP,
+    envvar="INSPECT_EVAL_MAX_SANDBOXES",
+)
 @click.option(
     "--no-sandbox-cleanup",
     type=bool,
@@ -904,6 +931,7 @@ def eval_retry_command(
     max_samples: int | None,
     max_tasks: int | None,
     max_subprocesses: int | None,
+    max_sandboxes: int | None,
     no_sandbox_cleanup: bool | None,
     trace: bool | None,
     fail_on_error: bool | float | None,
@@ -947,6 +975,7 @@ def eval_retry_command(
         max_samples=max_samples,
         max_tasks=max_tasks,
         max_subprocesses=max_subprocesses,
+        max_sandboxes=max_sandboxes,
         sandbox_cleanup=sandbox_cleanup,
         trace=trace,
         fail_on_error=fail_on_error,

inspect_ai/_display/core/progress.py

@@ -130,9 +130,15 @@ def progress_time(time: float) -> str:
     return f"{hours:2.0f}:{minutes:02.0f}:{seconds:02.0f}"
 
 
-def progress_count(complete: int, total: int) -> str:
-    # Pad the display to keep it stable
+def progress_count(complete: int, total: int, width: int | None = None) -> str:
+    # Pad the display to keep it stable as the
+    # complete metrics
     total_str = f"{total:,}"
     complete_str = f"{complete:,}"
     padding = max(0, len(total_str) - len(complete_str))
-    return " " * padding + f"[{complete_str}/{total_str}]"
+    padded = " " * padding + f"[{complete_str}/{total_str}]"
+
+    # If a width has ben specified, pad up to this width as well
+    if width is not None:
+        padded = padded.rjust(width)
+    return padded

inspect_ai/_display/core/results.py

@@ -166,7 +166,7 @@ def task_interrupted(profile: TaskProfile, samples_completed: int) -> Renderable
     return message
 
 
-def task_metric(metrics: list[TaskDisplayMetric]) -> str:
+def task_metric(metrics: list[TaskDisplayMetric], width: int | None = None) -> str:
     reducer_names: Set[str] = {
         metric.reducer for metric in metrics if metric.reducer is not None
     }
@@ -180,10 +180,14 @@ def task_metric(metrics: list[TaskDisplayMetric]) -> str:
     else:
         value = f"{metric.value:.2f}"
 
-    if show_reducer:
-        return f"{metric.name}/{metric.reducer}: {value}"
+    if show_reducer and metric.reducer is not None:
+        metric_str = f"{metric.name}/{metric.reducer}: {value}"
     else:
-        return f"{metric.name}: {value}"
+        metric_str = f"{metric.name}: {value}"
+
+    if width is not None:
+        metric_str = metric_str.rjust(width)
+    return metric_str
 
 
 def task_metrics(scores: list[EvalScore]) -> str:

inspect_ai/_display/textual/widgets/task_detail.py

@@ -224,6 +224,9 @@ class TaskMetrics(Widget):
         self.recompute_grid()
 
     def recompute_grid(self) -> None:
+        if not self.is_mounted:
+            return
+
         grid = self.query_one(f"#{self.grid_id()}")
 
         grid.remove_children()

inspect_ai/_display/textual/widgets/tasks.py

@@ -36,6 +36,9 @@ from ...core.progress import (
     progress_model_name,
 )
 
+MAX_METRIC_WIDTH = 25
+MAX_COUNT_WIDTH = 15
+
 
 class TasksView(Container):
     DEFAULT_CSS = """
@@ -68,6 +71,7 @@ class TasksView(Container):
         super().__init__()
         self.description_width = MAX_DESCRIPTION_WIDTH
         self.model_name_width = MAX_MODEL_NAME_WIDTH
+        self.sample_count_width = 0
 
     def init_tasks(self, tasks: list[TaskSpec]) -> None:
         # clear existing tasks
@@ -80,15 +84,41 @@ class TasksView(Container):
         self.model_name_width = min(
             max([len(str(task.model)) for task in tasks]), MAX_MODEL_NAME_WIDTH
         )
+        self.update_progress_widths()
 
     def add_task(self, task: TaskWithResult) -> TaskDisplay:
+        self.update_count_width(task.profile.samples)
         task_display = TaskProgressView(
-            task, self.description_width, self.model_name_width
+            task, self.description_width, self.model_name_width, self.sample_count_width
         )
         self.tasks.mount(task_display)
         self.tasks.scroll_to_widget(task_display)
+        self.update_progress_widths()
+
         return task_display
 
+    def update_count_width(self, samples: int) -> None:
+        sample_count_str = progress_count(samples, samples, self.sample_count_width)
+        self.sample_count_width = min(
+            max(self.sample_count_width, len(sample_count_str)), MAX_COUNT_WIDTH
+        )
+
+    def update_progress_widths(self) -> None:
+        progress_views = self.tasks.query_children(TaskProgressView)
+        metrics_size = 0
+        for progress_view in progress_views:
+            metrics_size = max(
+                metrics_size,
+                progress_view.metrics_width
+                if progress_view.metrics_width is not None
+                else 0,
+            )
+        metrics_size = min(metrics_size, MAX_METRIC_WIDTH)
+
+        for progress_view in progress_views:
+            progress_view.update_metrics_width(metrics_size)
+            progress_view.update_count_width(self.sample_count_width)
+
     def compose(self) -> ComposeResult:
         yield Static(id="tasks-config")
         yield Static(id="tasks-targets")
@@ -139,13 +169,18 @@ class TaskProgressView(Widget):
     """
 
     def __init__(
-        self, task: TaskWithResult, description_width: int, model_name_width: int
+        self,
+        task: TaskWithResult,
+        description_width: int,
+        model_name_width: int,
+        sample_count_width: int,
     ) -> None:
         super().__init__()
         self.t = task
 
         self.description_width = description_width
         self.model_name_width = model_name_width
+
         self.progress_bar = ProgressBar(total=task.profile.steps, show_eta=False)
         self.count_display = Static()
         self.metrics_display = Static(id="task-metrics")
@@ -154,6 +189,14 @@ class TaskProgressView(Widget):
         self.toggle = Toggle()
         self.task_detail = TaskDetail(id="task-detail", classes="hidden")
 
+        self.sample_count_width: int = sample_count_width
+
+    metrics: reactive[list[TaskDisplayMetric] | None] = reactive(None)
+    metrics_width: reactive[int | None] = reactive(None)
+    sample_count_width: reactive[int] = reactive(0)
+    samples_complete: reactive[int] = reactive(0)
+    samples_total: reactive[int] = reactive(0)
+
     def compose(self) -> ComposeResult:
         yield self.toggle
         yield TaskStatusIcon()
@@ -191,13 +234,51 @@ class TaskProgressView(Widget):
         self.task_progress.complete()
 
     def sample_complete(self, complete: int, total: int) -> None:
-        self.count_display.update(progress_count(complete, total))
+        self.samples_complete = complete
+        self.samples_total = total
 
     def update_metrics(self, metrics: list[TaskDisplayMetric]) -> None:
-        if len(metrics) > 0:
-            self.metrics_display.update(task_metric(metrics))
+        self.metrics = metrics
+
+    def update_metrics_width(self, width: int) -> None:
+        self.metrics_width = width
+
+    def update_count_width(self, width: int) -> None:
+        self.sample_count_width = width
+
+    def _watch_sample_count_width(self, width: int) -> None:
+        self.refresh_count()
+
+    def _watch_samples_complete(self, complete: int) -> None:
+        self.refresh_count()
+
+    def _watch_samples_total(self, total: int) -> None:
+        self.refresh_count()
+
+    def _watch_metrics_width(self, width: int) -> None:
+        self.update_metrics_label()
+
+    def _watch_metrics(self, metrics: list[TaskDisplayMetric] | None) -> None:
+        if metrics is not None and len(metrics) > 0:
+            # update label
+            self.update_metrics_label()
+
+            # update details
             self.task_detail.update_metrics(metrics)
 
+    def refresh_count(self) -> None:
+        progress_label = progress_count(
+            self.samples_complete, self.samples_total, self.sample_count_width
+        )
+        self.count_display.update(progress_label)
+
+    def update_metrics_label(self) -> None:
+        # compute the label (with a min size)
+        if self.metrics is not None:
+            metric_label = task_metric(self.metrics, self.metrics_width)
+            self.metrics_width = len(metric_label)
+            self.metrics_display.update(metric_label)
+
 
 class TaskStatusIcon(Static):
     result: reactive[TaskResult | None] = reactive(None)

inspect_ai/_eval/eval.py

@@ -71,6 +71,7 @@ def eval(
     max_samples: int | None = None,
     max_tasks: int | None = None,
     max_subprocesses: int | None = None,
+    max_sandboxes: int | None = None,
     log_samples: bool | None = None,
     log_images: bool | None = None,
     log_buffer: int | None = None,
@@ -129,6 +130,8 @@ def eval(
            (default is 1)
         max_subprocesses (int | None): Maximum number of subprocesses to
            run in parallel (default is os.cpu_count())
+        max_sandboxes (int | None): Maximum number of sandboxes (per-provider)
+           to run in parallel.
         log_samples: (bool | None): Log detailed samples and scores (defaults to True)
         log_images: (bool | None): Log base64 encoded version of images,
            even if specified as a filename or URL (defaults to False)
@@ -175,6 +178,7 @@ def eval(
             max_samples=max_samples,
             max_tasks=max_tasks,
             max_subprocesses=max_subprocesses,
+            max_sandboxes=max_sandboxes,
             log_samples=log_samples,
             log_images=log_images,
             log_buffer=log_buffer,
@@ -211,6 +215,7 @@ async def eval_async(
     max_samples: int | None = None,
     max_tasks: int | None = None,
     max_subprocesses: int | None = None,
+    max_sandboxes: int | None = None,
     log_samples: bool | None = None,
     log_images: bool | None = None,
     log_buffer: int | None = None,
@@ -268,6 +273,8 @@ async def eval_async(
            (default is 1)
         max_subprocesses (int | None): Maximum number of subprocesses to
             run in parallel (default is os.cpu_count())
+        max_sandboxes (int | None): Maximum number of sandboxes (per-provider)
+           to run in parallel.
         log_samples: (bool | None): Log detailed samples and scores (defaults to True)
         log_images: (bool | None): Log base64 encoded version of images,
             even if specified as a filename or URL (defaults to False)
@@ -368,6 +375,7 @@ async def eval_async(
             max_samples=max_samples,
             max_tasks=max_tasks,
             max_subprocesses=max_subprocesses,
+            max_sandboxes=max_sandboxes,
             sandbox_cleanup=sandbox_cleanup,
             log_samples=log_samples,
             log_images=log_images,
@@ -450,6 +458,7 @@ def eval_retry(
     max_samples: int | None = None,
     max_tasks: int | None = None,
     max_subprocesses: int | None = None,
+    max_sandboxes: int | None = None,
     sandbox_cleanup: bool | None = None,
     trace: bool | None = None,
     fail_on_error: bool | float | None = None,
@@ -480,6 +489,8 @@ def eval_retry(
            (default is 1)
         max_subprocesses (int | None): Maximum number of subprocesses to
            run in parallel (default is os.cpu_count())
+        max_sandboxes (int | None): Maximum number of sandboxes (per-provider)
+           to run in parallel.
         sandbox_cleanup (bool | None): Cleanup sandbox environments after task completes
            (defaults to True)
         trace (bool | None): Trace message interactions with evaluated model to terminal.
@@ -522,6 +533,7 @@ def eval_retry(
             max_samples=max_samples,
             max_tasks=max_tasks,
             max_subprocesses=max_subprocesses,
+            max_sandboxes=max_sandboxes,
             sandbox_cleanup=sandbox_cleanup,
             fail_on_error=fail_on_error,
             debug_errors=debug_errors,
@@ -545,6 +557,7 @@ async def eval_retry_async(
     max_samples: int | None = None,
     max_tasks: int | None = None,
     max_subprocesses: int | None = None,
+    max_sandboxes: int | None = None,
     sandbox_cleanup: bool | None = None,
     fail_on_error: bool | float | None = None,
     debug_errors: bool | None = None,
@@ -574,6 +587,7 @@ async def eval_retry_async(
            (default is 1)
         max_subprocesses (int): Maximum number of subprocesses to
            run in parallel (default is os.cpu_count())
+        max_sandboxes (int): Maximum number of sandboxes (per-provider) to run in parallel.
         sandbox_cleanup (bool | None): Cleanup sandbox environments after task completes
            (defaults to True)
         fail_on_error (bool | float | None): `True` to fail on first sample error
@@ -665,6 +679,7 @@ async def eval_retry_async(
         max_samples = max_samples or eval_log.eval.config.max_samples
         max_tasks = max_tasks or eval_log.eval.config.max_tasks
         max_subprocesses = max_subprocesses or eval_log.eval.config.max_subprocesses
+        max_sandboxes = max_sandboxes or eval_log.eval.config.max_sandboxes
         sandbox_cleanup = (
             sandbox_cleanup
             if sandbox_cleanup is not None
@@ -720,6 +735,7 @@ async def eval_retry_async(
                 max_samples=max_samples,
                 max_tasks=max_tasks,
                 max_subprocesses=max_subprocesses,
+                max_sandboxes=max_sandboxes,
                 log_samples=log_samples,
                 log_images=log_images,
                 log_buffer=log_buffer,

inspect_ai/_eval/evalset.py

@@ -75,6 +75,7 @@ def eval_set(
     max_samples: int | None = None,
     max_tasks: int | None = None,
     max_subprocesses: int | None = None,
+    max_sandboxes: int | None = None,
     log_samples: bool | None = None,
     log_images: bool | None = None,
     log_buffer: int | None = None,
@@ -144,6 +145,8 @@ def eval_set(
            (default is 1)
         max_subprocesses (int | None): Maximum number of subprocesses to
            run in parallel (default is os.cpu_count())
+        max_sandboxes (int | None): Maximum number of sandboxes (per-provider)
+           to run in parallel.
         log_samples: (bool | None): Log detailed samples and scores (defaults to True)
         log_images: (bool | None): Log base64 encoded version of images,
             even if specified as a filename or URL (defaults to False)
@@ -193,6 +196,7 @@ def eval_set(
             max_samples=max_samples,
             max_tasks=max_tasks,
             max_subprocesses=max_subprocesses,
+            max_sandboxes=max_sandboxes,
             log_samples=log_samples,
             log_images=log_images,
             log_buffer=log_buffer,

inspect_ai/_eval/registry.py

@@ -146,8 +146,8 @@ def task(*args: Any, name: str | None = None, **attribs: Any) -> Any:
             # module import, so set its task file and run dir
             if get_installed_package_name(task_type) is None:
                 module = inspect.getmodule(task_type)
-                if module and module.__file__:
-                    file = Path(module.__file__)
+                if module and hasattr(module, "__file__"):
+                    file = Path(getattr(module, "__file__"))
                     setattr(task_instance, TASK_FILE_ATTR, file.as_posix())
                     setattr(task_instance, TASK_RUN_DIR_ATTR, file.parent.as_posix())
 

inspect_ai/_eval/task/results.py

@@ -267,10 +267,28 @@ def scorers_from_metric_dict(
                 value = target_metric(metric_scores)
             else:
                 value = float("Nan")
-            result_metrics[metric_name] = EvalMetric(
-                name=metric_name,
-                value=cast(float, value),
-            )
+
+            # convert the value to a float (either by expanding the dict or array)
+            # or by casting to a float
+            if isinstance(value, dict):
+                for key, val in value.items():
+                    name = f"{metric_name}_{key}"
+                    result_metrics[name] = EvalMetric(
+                        name=name,
+                        value=cast(float, val),
+                    )
+            elif isinstance(value, list):
+                for idx, item in enumerate(value):
+                    name = f"{metric_name}_{idx}"
+                    result_metrics[name] = EvalMetric(
+                        name=name,
+                        value=cast(float, item),
+                    )
+            else:
+                result_metrics[metric_name] = EvalMetric(
+                    name=metric_name,
+                    value=cast(float, value),
+                )
 
         # create a scorer result for this metric
         # TODO: What if there is separate simple scorer which has a name collision with

inspect_ai/_eval/task/run.py

@@ -178,6 +178,10 @@ async def task_run(options: TaskRunOptions) -> EvalLog:
         else:
             plan = Plan(unroll(solver), internal=True)
 
+        # add setup solver(s) if specified
+        if task.setup:
+            plan.steps = unroll(task.setup) + plan.steps
+
         # reaolve the scorer
         score = score and task.scorer is not None
         scorers: list[Scorer] | None = task.scorer if (score and task.scorer) else None
@@ -275,6 +279,7 @@ async def task_run(options: TaskRunOptions) -> EvalLog:
                             sample=sample,
                             state=state,
                             sandbox=sandbox,
+                            max_sandboxes=config.max_sandboxes,
                             sandbox_cleanup=sandbox_cleanup,
                             plan=plan,
                             scorers=scorers,
@@ -456,6 +461,7 @@ async def task_run_sample(
     sample: Sample,
     state: TaskState,
     sandbox: SandboxEnvironmentSpec | None,
+    max_sandboxes: int | None,
     sandbox_cleanup: bool,
     plan: Plan,
     scorers: list[Scorer] | None,
@@ -482,8 +488,8 @@ async def task_run_sample(
                 await logger.log_sample(previous_sample, flush=False)
 
             # return score
-            if previous_sample.scores:
-                return {
+            sample_scores = (
+                {
                     key: SampleScore(
                         sample_id=previous_sample.id,
                         value=score.value,
@@ -493,8 +499,11 @@ async def task_run_sample(
                     )
                     for key, score in previous_sample.scores.items()
                 }
-            else:
-                return {}
+                if previous_sample.scores
+                else {}
+            )
+            sample_complete(sample_scores)
+            return sample_scores
 
     # use semaphore if provided
     semaphore_cm: asyncio.Semaphore | contextlib.AbstractAsyncContextManager[None] = (
@@ -510,7 +519,7 @@ async def task_run_sample(
 
     # use sandbox if provided
     sandboxenv_cm = (
-        sandboxenv_context(task_name, sandbox, sandbox_cleanup, sample)
+        sandboxenv_context(task_name, sandbox, max_sandboxes, sandbox_cleanup, sample)
         if sandbox or sample.sandbox is not None
         else contextlib.nullcontext()
     )
@@ -866,10 +875,5 @@ def create_sample_semaphore(
         else DEFAULT_MAX_CONNECTIONS
     )
 
-    # if max_tasks is specified and max_samples is less
-    # than max_tasks then bump it up
-    if config.max_tasks is not None:
-        max_samples = max(max_samples, config.max_tasks)
-
     # return the semaphore
     return asyncio.Semaphore(max_samples)