inspect-ai 0.3.51__py3-none-any.whl → 0.3.53__py3-none-any.whl

inspect_ai/_eval/eval.py

@@ -61,6 +61,7 @@ def eval(
     log_dir: str | None = None,
     log_format: Literal["eval", "json"] | None = None,
     limit: int | tuple[int, int] | None = None,
+    sample_id: str | int | list[str | int] | None = None,
     epochs: int | Epochs | None = None,
     fail_on_error: bool | float | None = None,
     debug_errors: bool | None = None,
@@ -70,6 +71,7 @@ def eval(
     max_samples: int | None = None,
     max_tasks: int | None = None,
     max_subprocesses: int | None = None,
+    max_sandboxes: int | None = None,
     log_samples: bool | None = None,
     log_images: bool | None = None,
     log_buffer: int | None = None,
@@ -110,6 +112,7 @@ def eval(
            to "eval", the native high-performance format).
         limit (int | tuple[int, int] | None): Limit evaluated samples
            (defaults to all samples).
+        sample_id (str | int | list[str | int] | None): Evaluate specific sample(s) from the dataset.
         epochs (int | Epochs | None): Epochs to repeat samples for and optional score
            reducer function(s) used to combine sample scores (defaults to "mean")
         fail_on_error (bool | float | None): `True` to fail on first sample error
@@ -127,6 +130,8 @@ def eval(
            (default is 1)
         max_subprocesses (int | None): Maximum number of subprocesses to
            run in parallel (default is os.cpu_count())
+        max_sandboxes (int | None): Maximum number of sandboxes (per-provider)
+           to run in parallel.
         log_samples: (bool | None): Log detailed samples and scores (defaults to True)
         log_images: (bool | None): Log base64 encoded version of images,
            even if specified as a filename or URL (defaults to False)
@@ -163,6 +168,7 @@ def eval(
             log_dir=log_dir,
             log_format=log_format,
             limit=limit,
+            sample_id=sample_id,
             epochs=epochs,
             fail_on_error=fail_on_error,
             debug_errors=debug_errors,
@@ -172,6 +178,7 @@ def eval(
             max_samples=max_samples,
             max_tasks=max_tasks,
             max_subprocesses=max_subprocesses,
+            max_sandboxes=max_sandboxes,
             log_samples=log_samples,
             log_images=log_images,
             log_buffer=log_buffer,
@@ -198,6 +205,7 @@ async def eval_async(
     log_dir: str | None = None,
     log_format: Literal["eval", "json"] | None = None,
     limit: int | tuple[int, int] | None = None,
+    sample_id: str | int | list[str | int] | None = None,
     epochs: int | Epochs | None = None,
     fail_on_error: bool | float | None = None,
     debug_errors: bool | None = None,
@@ -207,6 +215,7 @@ async def eval_async(
     max_samples: int | None = None,
     max_tasks: int | None = None,
     max_subprocesses: int | None = None,
+    max_sandboxes: int | None = None,
     log_samples: bool | None = None,
     log_images: bool | None = None,
     log_buffer: int | None = None,
@@ -245,8 +254,9 @@ async def eval_async(
             (defaults to file log in ./logs directory).
         log_format (Literal["eval", "json"] | None): Format for writing log files (defaults
            to "eval", the native high-performance format).
-        limit (int | tuple[int, int] | None): Limit evaluated samples
+        limit (str | int | list[str | int] | None): Limit evaluated samples
             (defaults to all samples).
+        sample_id (str | list[str] | None): Evaluate specific sample(s) from the dataset.
         epochs (int | Epochs | None): Epochs to repeat samples for and optional score
             reducer function(s) used to combine sample scores (defaults to "mean")
         fail_on_error (bool | float | None): `True` to fail on first sample error
@@ -263,6 +273,8 @@ async def eval_async(
            (default is 1)
         max_subprocesses (int | None): Maximum number of subprocesses to
             run in parallel (default is os.cpu_count())
+        max_sandboxes (int | None): Maximum number of sandboxes (per-provider)
+           to run in parallel.
         log_samples: (bool | None): Log detailed samples and scores (defaults to True)
         log_images: (bool | None): Log base64 encoded version of images,
             even if specified as a filename or URL (defaults to False)
@@ -335,6 +347,10 @@ async def eval_async(
         # resolve solver
         solver = chain(solver) if isinstance(solver, list) else solver
 
+        # ensure consistency of limit and sample_id
+        if sample_id is not None and limit is not None:
+            raise ValueError("You cannot specify both sample_id and limit.")
+
         # resolve epochs
         if isinstance(epochs, int):
             epochs = Epochs(epochs)
@@ -345,6 +361,7 @@ async def eval_async(
         epochs_reducer = epochs.reducer if epochs else None
         eval_config = EvalConfig(
             limit=limit,
+            sample_id=sample_id,
             epochs=epochs.epochs if epochs else None,
             epochs_reducer=reducer_log_names(epochs_reducer)
             if epochs_reducer
@@ -358,6 +375,7 @@ async def eval_async(
             max_samples=max_samples,
             max_tasks=max_tasks,
             max_subprocesses=max_subprocesses,
+            max_sandboxes=max_sandboxes,
             sandbox_cleanup=sandbox_cleanup,
             log_samples=log_samples,
             log_images=log_images,
@@ -440,6 +458,7 @@ def eval_retry(
     max_samples: int | None = None,
     max_tasks: int | None = None,
     max_subprocesses: int | None = None,
+    max_sandboxes: int | None = None,
     sandbox_cleanup: bool | None = None,
     trace: bool | None = None,
     fail_on_error: bool | float | None = None,
@@ -470,6 +489,8 @@ def eval_retry(
            (default is 1)
         max_subprocesses (int | None): Maximum number of subprocesses to
            run in parallel (default is os.cpu_count())
+        max_sandboxes (int | None): Maximum number of sandboxes (per-provider)
+           to run in parallel.
         sandbox_cleanup (bool | None): Cleanup sandbox environments after task completes
            (defaults to True)
         trace (bool | None): Trace message interactions with evaluated model to terminal.
@@ -512,6 +533,7 @@ def eval_retry(
             max_samples=max_samples,
             max_tasks=max_tasks,
             max_subprocesses=max_subprocesses,
+            max_sandboxes=max_sandboxes,
             sandbox_cleanup=sandbox_cleanup,
             fail_on_error=fail_on_error,
             debug_errors=debug_errors,
@@ -535,6 +557,7 @@ async def eval_retry_async(
     max_samples: int | None = None,
     max_tasks: int | None = None,
     max_subprocesses: int | None = None,
+    max_sandboxes: int | None = None,
     sandbox_cleanup: bool | None = None,
     fail_on_error: bool | float | None = None,
     debug_errors: bool | None = None,
@@ -564,6 +587,7 @@ async def eval_retry_async(
            (default is 1)
         max_subprocesses (int): Maximum number of subprocesses to
            run in parallel (default is os.cpu_count())
+        max_sandboxes (int): Maximum number of sandboxes (per-provider) to run in parallel.
         sandbox_cleanup (bool | None): Cleanup sandbox environments after task completes
            (defaults to True)
         fail_on_error (bool | float | None): `True` to fail on first sample error
@@ -642,6 +666,7 @@ async def eval_retry_async(
         task_args = eval_log.eval.task_args
         tags = eval_log.eval.tags
         limit = eval_log.eval.config.limit
+        sample_id = eval_log.eval.config.sample_id
         epochs = (
             Epochs(eval_log.eval.config.epochs, eval_log.eval.config.epochs_reducer)
             if eval_log.eval.config.epochs
@@ -654,6 +679,7 @@ async def eval_retry_async(
         max_samples = max_samples or eval_log.eval.config.max_samples
         max_tasks = max_tasks or eval_log.eval.config.max_tasks
         max_subprocesses = max_subprocesses or eval_log.eval.config.max_subprocesses
+        max_sandboxes = max_sandboxes or eval_log.eval.config.max_sandboxes
         sandbox_cleanup = (
             sandbox_cleanup
             if sandbox_cleanup is not None
@@ -699,6 +725,7 @@ async def eval_retry_async(
                 log_dir=log_dir,
                 log_format=log_format,
                 limit=limit,
+                sample_id=sample_id,
                 epochs=epochs,
                 fail_on_error=fail_on_error,
                 debug_errors=debug_errors,
@@ -708,6 +735,7 @@ async def eval_retry_async(
                 max_samples=max_samples,
                 max_tasks=max_tasks,
                 max_subprocesses=max_subprocesses,
+                max_sandboxes=max_sandboxes,
                 log_samples=log_samples,
                 log_images=log_images,
                 log_buffer=log_buffer,

inspect_ai/_eval/evalset.py

@@ -65,6 +65,7 @@ def eval_set(
     log_level_transcript: str | None = None,
     log_format: Literal["eval", "json"] | None = None,
     limit: int | tuple[int, int] | None = None,
+    sample_id: str | int | list[str | int] | None = None,
     epochs: int | Epochs | None = None,
     fail_on_error: bool | float | None = None,
     debug_errors: bool | None = None,
@@ -74,6 +75,7 @@ def eval_set(
     max_samples: int | None = None,
     max_tasks: int | None = None,
     max_subprocesses: int | None = None,
+    max_sandboxes: int | None = None,
     log_samples: bool | None = None,
     log_images: bool | None = None,
     log_buffer: int | None = None,
@@ -125,6 +127,7 @@ def eval_set(
           log files (defaults to "eval", the native high-performance format).
         limit (int | tuple[int, int] | None): Limit evaluated samples
            (defaults to all samples).
+        sample_id (str | int | list[str | int] | None): Evaluate specific sample(s) from the dataset.
         epochs (int | Epochs | None): Epochs to repeat samples for and optional score
            reducer function(s) used to combine sample scores (defaults to "mean")
         fail_on_error (bool | float | None): `True` to fail on first sample error
@@ -142,6 +145,8 @@ def eval_set(
            (default is 1)
         max_subprocesses (int | None): Maximum number of subprocesses to
            run in parallel (default is os.cpu_count())
+        max_sandboxes (int | None): Maximum number of sandboxes (per-provider)
+           to run in parallel.
         log_samples: (bool | None): Log detailed samples and scores (defaults to True)
         log_images: (bool | None): Log base64 encoded version of images,
             even if specified as a filename or URL (defaults to False)
@@ -181,6 +186,7 @@ def eval_set(
             log_dir=log_dir,
             log_format=log_format,
             limit=limit,
+            sample_id=sample_id,
             epochs=epochs,
             fail_on_error=fail_on_error,
             debug_errors=debug_errors,
@@ -190,6 +196,7 @@ def eval_set(
             max_samples=max_samples,
             max_tasks=max_tasks,
             max_subprocesses=max_subprocesses,
+            max_sandboxes=max_sandboxes,
             log_samples=log_samples,
             log_images=log_images,
             log_buffer=log_buffer,

inspect_ai/_eval/registry.py

@@ -146,8 +146,8 @@ def task(*args: Any, name: str | None = None, **attribs: Any) -> Any:
             # module import, so set its task file and run dir
             if get_installed_package_name(task_type) is None:
                 module = inspect.getmodule(task_type)
-                if module and module.__file__:
-                    file = Path(module.__file__)
+                if module and hasattr(module, "__file__"):
+                    file = Path(getattr(module, "__file__"))
                     setattr(task_instance, TASK_FILE_ATTR, file.as_posix())
                     setattr(task_instance, TASK_RUN_DIR_ATTR, file.parent.as_posix())
 

inspect_ai/_eval/task/log.py

@@ -83,7 +83,12 @@ class TaskLogger:
         # ensure that the dataset has sample ids and record them
         sample_ids = cast(
             list[int | str],
-            [sample.id for sample in slice_dataset(dataset, eval_config.limit)],
+            [
+                sample.id
+                for sample in slice_dataset(
+                    dataset, eval_config.limit, eval_config.sample_id
+                )
+            ],
         )
 
         # create eval spec

inspect_ai/_eval/task/results.py

@@ -267,10 +267,28 @@ def scorers_from_metric_dict(
                 value = target_metric(metric_scores)
             else:
                 value = float("Nan")
-            result_metrics[metric_name] = EvalMetric(
-                name=metric_name,
-                value=cast(float, value),
-            )
+
+            # convert the value to a float (either by expanding the dict or array)
+            # or by casting to a float
+            if isinstance(value, dict):
+                for key, val in value.items():
+                    name = f"{metric_name}_{key}"
+                    result_metrics[name] = EvalMetric(
+                        name=name,
+                        value=cast(float, val),
+                    )
+            elif isinstance(value, list):
+                for idx, item in enumerate(value):
+                    name = f"{metric_name}_{idx}"
+                    result_metrics[name] = EvalMetric(
+                        name=name,
+                        value=cast(float, item),
+                    )
+            else:
+                result_metrics[metric_name] = EvalMetric(
+                    name=metric_name,
+                    value=cast(float, value),
+                )
 
         # create a scorer result for this metric
         # TODO: What if there is separate simple scorer which has a name collision with

inspect_ai/_eval/task/run.py

@@ -162,6 +162,7 @@ async def task_run(options: TaskRunOptions) -> EvalLog:
             dataset=task.dataset,
             model_name=model_name,
             limit=config.limit,
+            sample_id=config.sample_id,
             epochs=epochs,
             log_images=log_images,
             message_limit=config.message_limit,
@@ -177,6 +178,10 @@ async def task_run(options: TaskRunOptions) -> EvalLog:
         else:
             plan = Plan(unroll(solver), internal=True)
 
+        # add setup solver(s) if specified
+        if task.setup:
+            plan.steps = unroll(task.setup) + plan.steps
+
         # reaolve the scorer
         score = score and task.scorer is not None
         scorers: list[Scorer] | None = task.scorer if (score and task.scorer) else None
@@ -274,6 +279,7 @@ async def task_run(options: TaskRunOptions) -> EvalLog:
                             sample=sample,
                             state=state,
                             sandbox=sandbox,
+                            max_sandboxes=config.max_sandboxes,
                             sandbox_cleanup=sandbox_cleanup,
                             plan=plan,
                             scorers=scorers,
@@ -455,6 +461,7 @@ async def task_run_sample(
     sample: Sample,
     state: TaskState,
     sandbox: SandboxEnvironmentSpec | None,
+    max_sandboxes: int | None,
     sandbox_cleanup: bool,
     plan: Plan,
     scorers: list[Scorer] | None,
@@ -481,8 +488,8 @@ async def task_run_sample(
                 await logger.log_sample(previous_sample, flush=False)
 
             # return score
-            if previous_sample.scores:
-                return {
+            sample_scores = (
+                {
                     key: SampleScore(
                         sample_id=previous_sample.id,
                         value=score.value,
@@ -492,8 +499,11 @@ async def task_run_sample(
                     )
                     for key, score in previous_sample.scores.items()
                 }
-            else:
-                return {}
+                if previous_sample.scores
+                else {}
+            )
+            sample_complete(sample_scores)
+            return sample_scores
 
     # use semaphore if provided
     semaphore_cm: asyncio.Semaphore | contextlib.AbstractAsyncContextManager[None] = (
@@ -509,7 +519,7 @@ async def task_run_sample(
 
     # use sandbox if provided
     sandboxenv_cm = (
-        sandboxenv_context(task_name, sandbox, sandbox_cleanup, sample)
+        sandboxenv_context(task_name, sandbox, max_sandboxes, sandbox_cleanup, sample)
         if sandbox or sample.sandbox is not None
         else contextlib.nullcontext()
     )
@@ -748,13 +758,14 @@ async def resolve_dataset(
     dataset: Dataset,
     model_name: ModelName,
     limit: int | tuple[int, int] | None,
+    sample_id: str | int | list[str | int] | None,
     epochs: int,
     log_images: bool,
     message_limit: int | None,
     token_limit: int | None,
 ) -> tuple[Dataset, list[Sample], list[TaskState]]:
-    # apply limit to dataset
-    dataset = slice_dataset(dataset, limit)
+    # slice dataset
+    dataset = slice_dataset(dataset, limit, sample_id)
 
     # apply epochs (deepcopy the samples so they remain independent)
     samples: list[Sample] = []
@@ -864,10 +875,5 @@ def create_sample_semaphore(
         else DEFAULT_MAX_CONNECTIONS
     )
 
-    # if max_tasks is specified and max_samples is less
-    # than max_tasks then bump it up
-    if config.max_tasks is not None:
-        max_samples = max(max_samples, config.max_tasks)
-
     # return the semaphore
     return asyncio.Semaphore(max_samples)

inspect_ai/_eval/task/sandbox.py

@@ -1,7 +1,8 @@
 import asyncio
 import base64
 import contextlib
-from typing import AsyncGenerator, NamedTuple
+from random import random
+from typing import AsyncGenerator, Callable, NamedTuple, cast
 
 from inspect_ai._eval.task.task import Task
 from inspect_ai._eval.task.util import task_run_dir
@@ -9,6 +10,7 @@ from inspect_ai._util.file import file, filesystem
 from inspect_ai._util.registry import registry_unqualified_name
 from inspect_ai._util.url import data_uri_to_base64, is_data_uri
 from inspect_ai.dataset import Sample
+from inspect_ai.util._concurrency import concurrency
 from inspect_ai.util._sandbox.context import (
     cleanup_sandbox_environments_sample,
     init_sandbox_environments_sample,
@@ -18,12 +20,14 @@ from inspect_ai.util._sandbox.environment import (
     SandboxEnvironmentConfigType,
     SandboxEnvironmentSpec,
 )
+from inspect_ai.util._sandbox.registry import registry_find_sandboxenv
 
 
 @contextlib.asynccontextmanager
 async def sandboxenv_context(
     task_name: str,
     sandbox: SandboxEnvironmentSpec | None,
+    max_sandboxes: int | None,
     cleanup: bool,
     sample: Sample,
 ) -> AsyncGenerator[None, None]:
@@ -32,52 +36,77 @@ async def sandboxenv_context(
     if not sandbox:
         raise ValueError("sandboxenv_context called with no sandbox specified")
 
-    # read files from sample
-    files: dict[str, bytes] = {}
-    if sample.files:
-        for path, contents in sample.files.items():
-            files[path] = read_sandboxenv_file(contents)
-
-    # read setup script from sample (add bash shebang if necessary)
-    setup: bytes | None = None
-    if sample.setup:
-        setup = read_sandboxenv_file(sample.setup)
-        setup_str = setup.decode(encoding="utf-8")
-        if not setup_str.strip().startswith("#!"):
-            setup_str = f"#!/usr/bin/env bash\n\n{setup_str}"
-            setup = setup_str.encode(encoding="utf-8")
-
-    interrupted = False
-    environments: dict[str, SandboxEnvironment] | None = None
-    try:
-        # initialize sandbox environment,
-        environments = await init_sandbox_environments_sample(
-            type=sandbox.type,
-            task_name=registry_unqualified_name(task_name),
-            config=sandbox.config,
-            files=files,
-            setup=setup,
-            metadata=sample.metadata if sample.metadata else {},
-        )
-
-        # run sample
-        yield
-
-    except asyncio.CancelledError as ex:
-        interrupted = True
-        raise ex
+    # get sandboxenv_type
+    sandboxenv_type = registry_find_sandboxenv(sandbox.type)
 
-    finally:
-        # cleanup sandbox environment
-        if environments and cleanup:
-            await cleanup_sandbox_environments_sample(
-                type=sandbox.type,
-                task_name=task_name,
+    # see if there is a max_sandboxes in play (passed or from type)
+    if max_sandboxes is None:
+        default_concurrency_fn = cast(
+            Callable[[], int | None], getattr(sandboxenv_type, "default_concurrency")
+        )
+        max_sandboxes = default_concurrency_fn()
+
+    # if we are enforcing max_sandboxes, then when samples are scheduled they may
+    # not get interleaved properly across tasks (because the first task will come
+    # in and grab all of the sandboxes). Therefore, in this case we wait a random
+    # delay so that all tasks/samples have an equal shot at getting scheduled.
+    if max_sandboxes is not None:
+        await asyncio.sleep(random())
+
+    # enforce concurrency if required
+    sandboxes_cm = (
+        concurrency(sandbox.type, max_sandboxes, f"sandboxes/{sandbox.type}")
+        if max_sandboxes is not None
+        else contextlib.nullcontext()
+    )
+
+    async with sandboxes_cm:
+        # read files from sample
+        files: dict[str, bytes] = {}
+        if sample.files:
+            for path, contents in sample.files.items():
+                files[path] = read_sandboxenv_file(contents)
+
+        # read setup script from sample (add bash shebang if necessary)
+        setup: bytes | None = None
+        if sample.setup:
+            setup = read_sandboxenv_file(sample.setup)
+            setup_str = setup.decode(encoding="utf-8")
+            if not setup_str.strip().startswith("#!"):
+                setup_str = f"#!/usr/bin/env bash\n\n{setup_str}"
+                setup = setup_str.encode(encoding="utf-8")
+
+        interrupted = False
+        environments: dict[str, SandboxEnvironment] | None = None
+        try:
+            # initialize sandbox environment,
+            environments = await init_sandbox_environments_sample(
+                sandboxenv_type=sandboxenv_type,
+                task_name=registry_unqualified_name(task_name),
                 config=sandbox.config,
-                environments=environments,
-                interrupted=interrupted,
+                files=files,
+                setup=setup,
+                metadata=sample.metadata if sample.metadata else {},
             )
 
+            # run sample
+            yield
+
+        except asyncio.CancelledError as ex:
+            interrupted = True
+            raise ex
+
+        finally:
+            # cleanup sandbox environment
+            if environments and cleanup:
+                await cleanup_sandbox_environments_sample(
+                    type=sandbox.type,
+                    task_name=task_name,
+                    config=sandbox.config,
+                    environments=environments,
+                    interrupted=interrupted,
+                )
+
 
 def read_sandboxenv_file(contents: str) -> bytes:
     if is_data_uri(contents):

inspect_ai/_eval/task/task.py

@@ -39,6 +39,8 @@ class Task:
 
     Args:
         dataset (Dataset | Sequence[Sample]): Dataset to evaluate
+        setup: (Solver | list[Solver] | None): Setup step (always run
+          even when the main `solver` is replaced).
         solver: (Solver | list[Solver]): Solver or list of solvers.
           Defaults to generate(), a normal call to the model.
         scorer: (Scorer | list[Scorer] | None): Scorer used to evaluate model output.
@@ -68,6 +70,7 @@ class Task:
     def __init__(
         self,
         dataset: Dataset | Sequence[Sample] | None = None,
+        setup: Solver | list[Solver] | None = None,
         solver: Solver | list[Solver] = generate(),
         scorer: Scorer | list[Scorer] | None = None,
         metrics: list[Metric] | dict[str, list[Metric]] | None = None,
@@ -119,6 +122,7 @@ class Task:
         self.dataset: Dataset = (
             dataset if isinstance(dataset, Dataset) else MemoryDataset(list(dataset))
         )
+        self.setup = setup
         self.solver = chain(solver) if isinstance(solver, list) else solver
         self.scorer = (
             scorer

inspect_ai/_eval/task/util.py

@@ -39,10 +39,21 @@ def task_file(task: Task, relative: bool = False) -> str | None:
 def slice_dataset(
     dataset: Dataset,
     limit: int | tuple[int, int] | None,
+    sample_id: str | int | list[str | int] | None,
 ) -> Dataset:
-    dataset_limit = (
-        slice(0, len(dataset))
-        if limit is None
-        else (slice(*limit) if isinstance(limit, tuple) else slice(0, limit))
-    )
-    return dataset[dataset_limit]
+    def normalise(id: str | int | None) -> str:
+        if isinstance(id, str) and id.isdigit():
+            id = int(id)
+        return id if isinstance(id, str) else str(id).zfill(20)
+
+    if sample_id is not None:
+        sample_id = sample_id if isinstance(sample_id, list) else [sample_id]
+        sample_id = [normalise(id) for id in sample_id]
+        return dataset.filter(lambda sample: normalise(sample.id) in sample_id)
+    else:
+        dataset_limit = (
+            slice(0, len(dataset))
+            if limit is None
+            else (slice(*limit) if isinstance(limit, tuple) else slice(0, limit))
+        )
+        return dataset[dataset_limit]

inspect_ai/_util/logger.py

@@ -1,5 +1,6 @@
 import os
 from logging import (
+    DEBUG,
     INFO,
     WARNING,
     FileHandler,
@@ -129,7 +130,7 @@ def init_logger(
     # init logging handler on demand
     global _logHandler
     if not _logHandler:
-        _logHandler = LogHandler(min(HTTP, levelno), transcript_levelno)
+        _logHandler = LogHandler(min(DEBUG, levelno), transcript_levelno)
         getLogger().addHandler(_logHandler)
 
     # establish default capture level
@@ -139,6 +140,7 @@ def init_logger(
     getLogger().setLevel(capture_level)
     getLogger(PKG_NAME).setLevel(capture_level)
     getLogger("httpx").setLevel(capture_level)
+    getLogger("botocore").setLevel(DEBUG)
 
     # set the levelno on the global handler
     _logHandler.display_level = levelno
@@ -154,7 +156,13 @@ def notify_logger_record(record: LogRecord, write: bool) -> None:
     if write:
         transcript()._event(LoggerEvent(message=LoggingMessage.from_log_record(record)))
     global _rate_limit_count
-    if record.levelno <= INFO and "429" in record.getMessage():
+    if (record.levelno <= INFO and "429" in record.getMessage()) or (
+        record.levelno == DEBUG
+        # See https://boto3.amazonaws.com/v1/documentation/api/latest/guide/retries.html#validating-retry-attempts
+        # for boto retry logic / log messages (this is tracking standard or adapative retries)
+        and "botocore.retries.standard" in record.name
+        and "Retry needed, retrying request after delay of:" in record.getMessage()
+    ):
         _rate_limit_count = _rate_limit_count + 1
 
 

inspect_ai/_util/samples.py

@@ -7,3 +7,10 @@ def parse_samples_limit(limit: str | None) -> int | tuple[int, int] | None:
             return (limit_split[0] - 1, limit_split[1])
     else:
         return None
+
+
+def parse_sample_id(sample_id: str | None) -> list[str] | None:
+    if sample_id is not None:
+        return [id.strip() for id in sample_id.split(",")]
+    else:
+        return None

inspect_ai/_util/transcript.py

@@ -1,4 +1,5 @@
 import html
+from typing import Any
 
 from rich.align import AlignMethod
 from rich.box import ROUNDED, Box
@@ -8,6 +9,8 @@ from rich.panel import Panel
 from rich.rule import Rule
 from rich.text import Text
 
+from .format import format_function_call
+
 
 def transcript_code_theme() -> str:
     return "github-dark"
@@ -81,6 +84,11 @@ def transcript_separator(title: str, color: str) -> RenderableType:
     return Rule(title=title, style=f"{color} bold", align="center", end="\n\n")
 
 
+def transcript_function(function: str, arguments: dict[str, Any]) -> RenderableType:
+    call = format_function_call(function, arguments)
+    return transcript_markdown("```python\n" + call + "\n```\n")
+
+
 LINE = Box(" ── \n" "    \n" "    \n" "    \n" "    \n" "    \n" "    \n" "    \n")
 
 DOTTED = Box(" ·· \n" "    \n" "    \n" "    \n" "    \n" "    \n" "    \n" "    \n")