inspect-ai 0.3.52__py3-none-any.whl → 0.3.54__py3-none-any.whl

inspect_ai/_display/textual/widgets/tasks.py

@@ -36,6 +36,9 @@ from ...core.progress import (
     progress_model_name,
 )
 
+MAX_METRIC_WIDTH = 25
+MAX_COUNT_WIDTH = 15
+
 
 class TasksView(Container):
     DEFAULT_CSS = """
@@ -68,6 +71,8 @@ class TasksView(Container):
         super().__init__()
         self.description_width = MAX_DESCRIPTION_WIDTH
         self.model_name_width = MAX_MODEL_NAME_WIDTH
+        self.sample_count_width = 0
+        self.display_metrics = True
 
     def init_tasks(self, tasks: list[TaskSpec]) -> None:
         # clear existing tasks
@@ -80,15 +85,48 @@ class TasksView(Container):
         self.model_name_width = min(
             max([len(str(task.model)) for task in tasks]), MAX_MODEL_NAME_WIDTH
         )
+        self.update_progress_widths()
 
     def add_task(self, task: TaskWithResult) -> TaskDisplay:
+        self.update_count_width(task.profile.samples)
         task_display = TaskProgressView(
-            task, self.description_width, self.model_name_width
+            task,
+            self.description_width,
+            self.model_name_width,
+            self.sample_count_width,
+            self.display_metrics,
         )
         self.tasks.mount(task_display)
         self.tasks.scroll_to_widget(task_display)
+        self.update_progress_widths()
+
         return task_display
 
+    def set_display_metrics(self, display_metrics: bool) -> None:
+        self.display_metrics = display_metrics
+
+    def update_count_width(self, samples: int) -> None:
+        sample_count_str = progress_count(samples, samples, self.sample_count_width)
+        self.sample_count_width = min(
+            max(self.sample_count_width, len(sample_count_str)), MAX_COUNT_WIDTH
+        )
+
+    def update_progress_widths(self) -> None:
+        progress_views = self.tasks.query_children(TaskProgressView)
+        metrics_size = 0
+        for progress_view in progress_views:
+            metrics_size = max(
+                metrics_size,
+                progress_view.metrics_width
+                if progress_view.metrics_width is not None
+                else 0,
+            )
+        metrics_size = min(metrics_size, MAX_METRIC_WIDTH)
+
+        for progress_view in progress_views:
+            progress_view.update_metrics_width(metrics_size)
+            progress_view.update_count_width(self.sample_count_width)
+
     def compose(self) -> ComposeResult:
         yield Static(id="tasks-config")
         yield Static(id="tasks-targets")
@@ -139,13 +177,19 @@ class TaskProgressView(Widget):
     """
 
     def __init__(
-        self, task: TaskWithResult, description_width: int, model_name_width: int
+        self,
+        task: TaskWithResult,
+        description_width: int,
+        model_name_width: int,
+        sample_count_width: int,
+        display_metrics: bool,
     ) -> None:
         super().__init__()
         self.t = task
 
         self.description_width = description_width
         self.model_name_width = model_name_width
+
         self.progress_bar = ProgressBar(total=task.profile.steps, show_eta=False)
         self.count_display = Static()
         self.metrics_display = Static(id="task-metrics")
@@ -154,8 +198,17 @@ class TaskProgressView(Widget):
         self.toggle = Toggle()
         self.task_detail = TaskDetail(id="task-detail", classes="hidden")
 
+        self.sample_count_width: int = sample_count_width
+        self.display_metrics = display_metrics
+
+    metrics: reactive[list[TaskDisplayMetric] | None] = reactive(None)
+    metrics_width: reactive[int | None] = reactive(None)
+    sample_count_width: reactive[int] = reactive(0)
+    samples_complete: reactive[int] = reactive(0)
+    samples_total: reactive[int] = reactive(0)
+
     def compose(self) -> ComposeResult:
-        yield self.toggle
+        yield (self.toggle if self.display_metrics else Static())
         yield TaskStatusIcon()
         yield Static(
             progress_description(self.t.profile, self.description_width, pad=True)
@@ -191,13 +244,51 @@ class TaskProgressView(Widget):
         self.task_progress.complete()
 
     def sample_complete(self, complete: int, total: int) -> None:
-        self.count_display.update(progress_count(complete, total))
+        self.samples_complete = complete
+        self.samples_total = total
 
     def update_metrics(self, metrics: list[TaskDisplayMetric]) -> None:
-        if len(metrics) > 0:
-            self.metrics_display.update(task_metric(metrics))
+        self.metrics = metrics
+
+    def update_metrics_width(self, width: int) -> None:
+        self.metrics_width = width
+
+    def update_count_width(self, width: int) -> None:
+        self.sample_count_width = width
+
+    def _watch_sample_count_width(self, width: int) -> None:
+        self.refresh_count()
+
+    def _watch_samples_complete(self, complete: int) -> None:
+        self.refresh_count()
+
+    def _watch_samples_total(self, total: int) -> None:
+        self.refresh_count()
+
+    def _watch_metrics_width(self, width: int) -> None:
+        self.update_metrics_label()
+
+    def _watch_metrics(self, metrics: list[TaskDisplayMetric] | None) -> None:
+        if metrics is not None and len(metrics) > 0:
+            # update label
+            self.update_metrics_label()
+
+            # update details
             self.task_detail.update_metrics(metrics)
 
+    def refresh_count(self) -> None:
+        progress_label = progress_count(
+            self.samples_complete, self.samples_total, self.sample_count_width
+        )
+        self.count_display.update(progress_label)
+
+    def update_metrics_label(self) -> None:
+        # compute the label (with a min size)
+        if self.metrics is not None and self.metrics_display is not None:
+            metric_label = task_metric(self.metrics, self.metrics_width)
+            self.metrics_width = len(metric_label)
+            self.metrics_display.update(metric_label)
+
 
 class TaskStatusIcon(Static):
     result: reactive[TaskResult | None] = reactive(None)

inspect_ai/_eval/eval.py

@@ -71,10 +71,12 @@ def eval(
     max_samples: int | None = None,
     max_tasks: int | None = None,
     max_subprocesses: int | None = None,
+    max_sandboxes: int | None = None,
     log_samples: bool | None = None,
     log_images: bool | None = None,
     log_buffer: int | None = None,
     score: bool = True,
+    score_display: bool | None = None,
     **kwargs: Unpack[GenerateConfigArgs],
 ) -> list[EvalLog]:
     r"""Evaluate tasks using a Model.
@@ -129,6 +131,8 @@ def eval(
            (default is 1)
         max_subprocesses (int | None): Maximum number of subprocesses to
            run in parallel (default is os.cpu_count())
+        max_sandboxes (int | None): Maximum number of sandboxes (per-provider)
+           to run in parallel.
         log_samples: (bool | None): Log detailed samples and scores (defaults to True)
         log_images: (bool | None): Log base64 encoded version of images,
            even if specified as a filename or URL (defaults to False)
@@ -136,6 +140,7 @@ def eval(
            If not specified, an appropriate default for the format and filesystem is
            chosen (10 for most all cases, 100 for JSON logs on remote filesystems).
         score (bool): Score output (defaults to True)
+        score_display (bool | None): Show scoring metrics in realtime (defaults to True)
         **kwargs (GenerateConfigArgs): Model generation options.
 
     Returns:
@@ -175,10 +180,12 @@ def eval(
             max_samples=max_samples,
             max_tasks=max_tasks,
             max_subprocesses=max_subprocesses,
+            max_sandboxes=max_sandboxes,
             log_samples=log_samples,
             log_images=log_images,
             log_buffer=log_buffer,
             score=score,
+            score_display=score_display,
             **kwargs,
         )
     )
@@ -211,10 +218,12 @@ async def eval_async(
     max_samples: int | None = None,
     max_tasks: int | None = None,
     max_subprocesses: int | None = None,
+    max_sandboxes: int | None = None,
     log_samples: bool | None = None,
     log_images: bool | None = None,
     log_buffer: int | None = None,
     score: bool = True,
+    score_display: bool | None = None,
     **kwargs: Unpack[GenerateConfigArgs],
 ) -> list[EvalLog]:
     r"""Evaluate tasks using a Model (async).
@@ -268,6 +277,8 @@ async def eval_async(
            (default is 1)
         max_subprocesses (int | None): Maximum number of subprocesses to
             run in parallel (default is os.cpu_count())
+        max_sandboxes (int | None): Maximum number of sandboxes (per-provider)
+           to run in parallel.
         log_samples: (bool | None): Log detailed samples and scores (defaults to True)
         log_images: (bool | None): Log base64 encoded version of images,
             even if specified as a filename or URL (defaults to False)
@@ -275,6 +286,7 @@ async def eval_async(
            If not specified, an appropriate default for the format and filesystem is
            chosen (10 for most all cases, 100 for JSON logs on remote filesystems).
         score (bool): Score output (defaults to True)
+        score_display (bool | None): Show scoring metrics in realtime (defaults to True)
         **kwargs (GenerateConfigArgs): Model generation options.
 
     Returns:
@@ -368,10 +380,12 @@ async def eval_async(
             max_samples=max_samples,
             max_tasks=max_tasks,
             max_subprocesses=max_subprocesses,
+            max_sandboxes=max_sandboxes,
             sandbox_cleanup=sandbox_cleanup,
             log_samples=log_samples,
             log_images=log_images,
             log_buffer=log_buffer,
+            score_display=score_display,
         )
 
         # run tasks - 2 codepaths, one for the traditional task at a time
@@ -450,6 +464,7 @@ def eval_retry(
     max_samples: int | None = None,
     max_tasks: int | None = None,
     max_subprocesses: int | None = None,
+    max_sandboxes: int | None = None,
     sandbox_cleanup: bool | None = None,
     trace: bool | None = None,
     fail_on_error: bool | float | None = None,
@@ -458,6 +473,7 @@ def eval_retry(
     log_images: bool | None = None,
     log_buffer: int | None = None,
     score: bool = True,
+    score_display: bool | None = None,
     max_retries: int | None = None,
     timeout: int | None = None,
     max_connections: int | None = None,
@@ -480,6 +496,8 @@ def eval_retry(
            (default is 1)
         max_subprocesses (int | None): Maximum number of subprocesses to
            run in parallel (default is os.cpu_count())
+        max_sandboxes (int | None): Maximum number of sandboxes (per-provider)
+           to run in parallel.
         sandbox_cleanup (bool | None): Cleanup sandbox environments after task completes
            (defaults to True)
         trace (bool | None): Trace message interactions with evaluated model to terminal.
@@ -496,6 +514,7 @@ def eval_retry(
            If not specified, an appropriate default for the format and filesystem is
            chosen (10 for most all cases, 100 for JSON logs on remote filesystems).
         score (bool): Score output (defaults to True)
+        score_display (bool | None): Show scoring metrics in realtime (defaults to True)
         max_retries (int | None):
            Maximum number of times to retry request.
         timeout: (int | None):
@@ -522,6 +541,7 @@ def eval_retry(
             max_samples=max_samples,
             max_tasks=max_tasks,
             max_subprocesses=max_subprocesses,
+            max_sandboxes=max_sandboxes,
             sandbox_cleanup=sandbox_cleanup,
             fail_on_error=fail_on_error,
             debug_errors=debug_errors,
@@ -529,6 +549,7 @@ def eval_retry(
             log_images=log_images,
             log_buffer=log_buffer,
             score=score,
+            score_display=score_display,
             max_retries=max_retries,
             timeout=timeout,
             max_connections=max_connections,
@@ -545,6 +566,7 @@ async def eval_retry_async(
     max_samples: int | None = None,
     max_tasks: int | None = None,
     max_subprocesses: int | None = None,
+    max_sandboxes: int | None = None,
     sandbox_cleanup: bool | None = None,
     fail_on_error: bool | float | None = None,
     debug_errors: bool | None = None,
@@ -552,6 +574,7 @@ async def eval_retry_async(
     log_images: bool | None = None,
     log_buffer: int | None = None,
     score: bool = True,
+    score_display: bool | None = None,
     max_retries: int | None = None,
     timeout: int | None = None,
     max_connections: int | None = None,
@@ -574,6 +597,7 @@ async def eval_retry_async(
            (default is 1)
         max_subprocesses (int): Maximum number of subprocesses to
            run in parallel (default is os.cpu_count())
+        max_sandboxes (int): Maximum number of sandboxes (per-provider) to run in parallel.
         sandbox_cleanup (bool | None): Cleanup sandbox environments after task completes
            (defaults to True)
         fail_on_error (bool | float | None): `True` to fail on first sample error
@@ -589,6 +613,7 @@ async def eval_retry_async(
            If not specified, an appropriate default for the format and filesystem is
            chosen (10 for most all cases, 100 for JSON logs on remote filesystems).
         score (bool): Score output (defaults to True)
+        score_display (bool | None): Show scoring metrics in realtime (defaults to True)
         max_retries (int | None):
            Maximum number of times to retry request.
         timeout: (int | None):
@@ -665,6 +690,7 @@ async def eval_retry_async(
         max_samples = max_samples or eval_log.eval.config.max_samples
         max_tasks = max_tasks or eval_log.eval.config.max_tasks
         max_subprocesses = max_subprocesses or eval_log.eval.config.max_subprocesses
+        max_sandboxes = max_sandboxes or eval_log.eval.config.max_sandboxes
         sandbox_cleanup = (
             sandbox_cleanup
             if sandbox_cleanup is not None
@@ -684,6 +710,11 @@ async def eval_retry_async(
         log_buffer = (
             log_buffer if log_buffer is not None else eval_log.eval.config.log_buffer
         )
+        score_display = (
+            score_display
+            if score_display is not None
+            else eval_log.eval.config.score_display
+        )
 
         config = eval_log.plan.config
         config.max_retries = max_retries or config.max_retries
@@ -720,10 +751,12 @@ async def eval_retry_async(
                 max_samples=max_samples,
                 max_tasks=max_tasks,
                 max_subprocesses=max_subprocesses,
+                max_sandboxes=max_sandboxes,
                 log_samples=log_samples,
                 log_images=log_images,
                 log_buffer=log_buffer,
                 score=score,
+                score_display=score_display,
                 **dict(config),
             )
         )[0]

inspect_ai/_eval/evalset.py

@@ -75,6 +75,7 @@ def eval_set(
     max_samples: int | None = None,
     max_tasks: int | None = None,
     max_subprocesses: int | None = None,
+    max_sandboxes: int | None = None,
     log_samples: bool | None = None,
     log_images: bool | None = None,
     log_buffer: int | None = None,
@@ -144,6 +145,8 @@ def eval_set(
            (default is 1)
         max_subprocesses (int | None): Maximum number of subprocesses to
            run in parallel (default is os.cpu_count())
+        max_sandboxes (int | None): Maximum number of sandboxes (per-provider)
+           to run in parallel.
         log_samples: (bool | None): Log detailed samples and scores (defaults to True)
         log_images: (bool | None): Log base64 encoded version of images,
             even if specified as a filename or URL (defaults to False)
@@ -193,6 +196,7 @@ def eval_set(
             max_samples=max_samples,
             max_tasks=max_tasks,
             max_subprocesses=max_subprocesses,
+            max_sandboxes=max_sandboxes,
             log_samples=log_samples,
             log_images=log_images,
             log_buffer=log_buffer,

inspect_ai/_eval/registry.py

@@ -146,8 +146,8 @@ def task(*args: Any, name: str | None = None, **attribs: Any) -> Any:
             # module import, so set its task file and run dir
             if get_installed_package_name(task_type) is None:
                 module = inspect.getmodule(task_type)
-                if module and module.__file__:
-                    file = Path(module.__file__)
+                if module and hasattr(module, "__file__"):
+                    file = Path(getattr(module, "__file__"))
                     setattr(task_instance, TASK_FILE_ATTR, file.as_posix())
                     setattr(task_instance, TASK_RUN_DIR_ATTR, file.parent.as_posix())
 

inspect_ai/_eval/task/images.py

@@ -30,13 +30,8 @@ async def samples_with_base64_images(samples: list[Sample]) -> list[Sample]:
 
 async def sample_with_base64_images(sample: Sample) -> Sample:
     if isinstance(sample.input, list):
-        return Sample(
-            input=await messages_with_base64_images(sample.input),
-            target=sample.target,
-            id=sample.id,
-            metadata=sample.metadata,
-            files=sample.files,
-            choices=sample.choices,
+        return sample.model_copy(
+            update={"input": await messages_with_base64_images(sample.input)}
         )
     else:
         return sample
@@ -44,13 +39,8 @@ async def sample_with_base64_images(sample: Sample) -> Sample:
 
 def sample_without_base64_images(sample: Sample) -> Sample:
     if isinstance(sample.input, list):
-        return Sample(
-            input=messages_without_base64_images(sample.input),
-            target=sample.target,
-            id=sample.id,
-            metadata=sample.metadata,
-            files=sample.files,
-            choices=sample.choices,
+        return sample.model_copy(
+            update={"input": messages_without_base64_images(sample.input)}
         )
     else:
         return sample

inspect_ai/_eval/task/results.py

@@ -267,10 +267,28 @@ def scorers_from_metric_dict(
                 value = target_metric(metric_scores)
             else:
                 value = float("Nan")
-            result_metrics[metric_name] = EvalMetric(
-                name=metric_name,
-                value=cast(float, value),
-            )
+
+            # convert the value to a float (either by expanding the dict or array)
+            # or by casting to a float
+            if isinstance(value, dict):
+                for key, val in value.items():
+                    name = f"{metric_name}_{key}"
+                    result_metrics[name] = EvalMetric(
+                        name=name,
+                        value=cast(float, val),
+                    )
+            elif isinstance(value, list):
+                for idx, item in enumerate(value):
+                    name = f"{metric_name}_{idx}"
+                    result_metrics[name] = EvalMetric(
+                        name=name,
+                        value=cast(float, item),
+                    )
+            else:
+                result_metrics[metric_name] = EvalMetric(
+                    name=metric_name,
+                    value=cast(float, value),
+                )
 
         # create a scorer result for this metric
         # TODO: What if there is separate simple scorer which has a name collision with

inspect_ai/_eval/task/run.py

@@ -178,6 +178,10 @@ async def task_run(options: TaskRunOptions) -> EvalLog:
         else:
             plan = Plan(unroll(solver), internal=True)
 
+        # add setup solver(s) if specified
+        if task.setup:
+            plan.steps = unroll(task.setup) + plan.steps
+
         # reaolve the scorer
         score = score and task.scorer is not None
         scorers: list[Scorer] | None = task.scorer if (score and task.scorer) else None
@@ -213,7 +217,9 @@ async def task_run(options: TaskRunOptions) -> EvalLog:
             log_location=log_location,
         )
 
-        with display().task(profile) as td:
+        with display().task(
+            profile,
+        ) as td:
             try:
                 # start the log
                 await log_start(logger, plan, generate_config)
@@ -248,7 +254,10 @@ async def task_run(options: TaskRunOptions) -> EvalLog:
 
                     # track when samples complete and update progress as we go
                     progress_results: list[dict[str, SampleScore]] = []
-                    update_metrics_display = update_metrics_display_fn(td)
+                    update_metrics_display = update_metrics_display_fn(
+                        td,
+                        display_metrics=profile.eval_config.score_display is not False,
+                    )
 
                     def sample_complete(sample_score: dict[str, SampleScore]) -> None:
                         # Capture the result
@@ -275,6 +284,7 @@ async def task_run(options: TaskRunOptions) -> EvalLog:
                             sample=sample,
                             state=state,
                             sandbox=sandbox,
+                            max_sandboxes=config.max_sandboxes,
                             sandbox_cleanup=sandbox_cleanup,
                             plan=plan,
                             scorers=scorers,
@@ -395,7 +405,10 @@ async def task_run(options: TaskRunOptions) -> EvalLog:
 
 
 def update_metrics_display_fn(
-    td: TaskDisplay, initial_interval: float = 0, min_interval: float = 0.9
+    td: TaskDisplay,
+    initial_interval: float = 0,
+    min_interval: float = 0.9,
+    display_metrics: bool = True,
 ) -> Callable[
     [
         int,
@@ -415,6 +428,10 @@ def update_metrics_display_fn(
         reducers: ScoreReducer | list[ScoreReducer] | None,
         metrics: list[Metric] | dict[str, list[Metric]] | None,
     ) -> None:
+        # Don't compute metrics if they are not being displayed
+        if not display_metrics:
+            return None
+
         nonlocal next_compute_time
         time_start = time.perf_counter()
         if time_start >= next_compute_time:
@@ -456,6 +473,7 @@ async def task_run_sample(
     sample: Sample,
     state: TaskState,
     sandbox: SandboxEnvironmentSpec | None,
+    max_sandboxes: int | None,
     sandbox_cleanup: bool,
     plan: Plan,
     scorers: list[Scorer] | None,
@@ -482,8 +500,8 @@ async def task_run_sample(
                 await logger.log_sample(previous_sample, flush=False)
 
             # return score
-            if previous_sample.scores:
-                return {
+            sample_scores = (
+                {
                     key: SampleScore(
                         sample_id=previous_sample.id,
                         value=score.value,
@@ -493,8 +511,11 @@ async def task_run_sample(
                     )
                     for key, score in previous_sample.scores.items()
                 }
-            else:
-                return {}
+                if previous_sample.scores
+                else {}
+            )
+            sample_complete(sample_scores)
+            return sample_scores
 
     # use semaphore if provided
     semaphore_cm: asyncio.Semaphore | contextlib.AbstractAsyncContextManager[None] = (
@@ -510,7 +531,7 @@ async def task_run_sample(
 
     # use sandbox if provided
     sandboxenv_cm = (
-        sandboxenv_context(task_name, sandbox, sandbox_cleanup, sample)
+        sandboxenv_context(task_name, sandbox, max_sandboxes, sandbox_cleanup, sample)
         if sandbox or sample.sandbox is not None
         else contextlib.nullcontext()
     )
@@ -559,14 +580,18 @@ async def task_run_sample(
                 state = await plan(state, generate)
 
         except TimeoutError:
-            # notify the user
-            transcript()._event(
-                SampleLimitEvent(
-                    type="time",
-                    message=f"Sample completed: exceeded time limit ({time_limit:,} seconds)",
-                    limit=time_limit,
+            if time_limit is not None:
+                transcript()._event(
+                    SampleLimitEvent(
+                        type="time",
+                        message=f"Sample completed: exceeded time limit ({time_limit:,} seconds)",
+                        limit=time_limit,
+                    )
+                )
+            else:
+                py_logger.warning(
+                    "Unexpected timeout error reached top of sample stack. Are you handling TimeoutError when applying timeouts?"
                 )
-            )
 
             # capture most recent state for scoring
             state = sample_state() or state
@@ -866,10 +891,5 @@ def create_sample_semaphore(
         else DEFAULT_MAX_CONNECTIONS
     )
 
-    # if max_tasks is specified and max_samples is less
-    # than max_tasks then bump it up
-    if config.max_tasks is not None:
-        max_samples = max(max_samples, config.max_tasks)
-
     # return the semaphore
     return asyncio.Semaphore(max_samples)