inspect-ai 0.3.73__py3-none-any.whl → 0.3.75__py3-none-any.whl

inspect_ai/_eval/evalset.py

@@ -1,7 +1,7 @@
 import hashlib
 import logging
 from copy import deepcopy
-from typing import Any, Callable, Literal, NamedTuple, Set, cast
+from typing import Any, Literal, NamedTuple, Set, cast
 
 import rich
 from pydantic_core import to_json
@@ -17,6 +17,7 @@ from typing_extensions import Unpack
 
 from inspect_ai._util.error import PrerequisiteError
 from inspect_ai._util.file import basename, filesystem
+from inspect_ai._util.notgiven import NOT_GIVEN, NotGiven
 from inspect_ai.approval._policy import ApprovalPolicy
 from inspect_ai.log import EvalLog
 from inspect_ai.log._bundle import bundle_log_dir
@@ -34,11 +35,14 @@ from inspect_ai.model import (
 from inspect_ai.model._generate_config import GenerateConfig
 from inspect_ai.solver._solver import Solver, SolverSpec
 from inspect_ai.util import DisplayType, SandboxEnvironmentType
+from inspect_ai.util._display import init_display_type
 
 from .eval import eval, eval_init
-from .loader import ResolvedTask, resolve_task_args
-from .task import Epochs, Tasks
-from .task.task import PreviousTask, Task
+from .loader import resolve_task_args
+from .task import Epochs
+from .task.resolved import ResolvedTask
+from .task.task import PreviousTask
+from .task.tasks import Tasks
 
 logger = logging.getLogger(__name__)
 
@@ -56,7 +60,7 @@ def eval_set(
     retry_wait: float | None = None,
     retry_connections: float | None = None,
     retry_cleanup: bool | None = None,
-    model: str | Model | list[str] | list[Model] | None = None,
+    model: str | Model | list[str] | list[Model] | None | NotGiven = NOT_GIVEN,
     model_base_url: str | None = None,
     model_args: dict[str, Any] | str = dict(),
     task_args: dict[str, Any] | str = dict(),
@@ -107,9 +111,9 @@ def eval_set(
             (defaults to 0.5)
         retry_cleanup: Cleanup failed log files after retries
             (defaults to True)
-        model: Model(s) for
-            evaluation. If not specified use the value of the INSPECT_EVAL_MODEL
-            environment variable.
+        model: Model(s) for evaluation. If not specified use the value of the INSPECT_EVAL_MODEL
+            environment variable. Specify `None` to define no default model(s), which will
+            leave model usage entirely up to tasks.
         model_base_url: Base URL for communicating
             with the model API.
         model_args: Model creation args
@@ -154,7 +158,7 @@ def eval_set(
         max_samples: Maximum number of samples to run in parallel
             (default is max_connections)
         max_tasks: Maximum number of tasks to run in parallel
-            (default is 1)
+            (defaults to number of models being evaluated)
         max_subprocesses: Maximum number of subprocesses to
             run in parallel (default is os.cpu_count())
         max_sandboxes: Maximum number of sandboxes (per-provider)
@@ -177,13 +181,11 @@ def eval_set(
     """
 
     # helper function to run a set of evals
-    def run_eval(
-        tasks: list[Task] | list[PreviousTask], models: list[Model]
-    ) -> list[EvalLog]:
+    def run_eval(tasks: list[ResolvedTask] | list[PreviousTask]) -> list[EvalLog]:
         # run evals
         results = eval(
             tasks=tasks,
-            model=models,
+            model=None,  # ResolvedTask/PreviousTask already carries its model
             model_base_url=model_base_url,
             model_args=model_args,
             task_args=task_args,
@@ -231,30 +233,10 @@ def eval_set(
         # return results
         return results
 
-    # helper function to run a list of task groups
-    def run_task_groups(
-        task_groups: list[TaskGroup],
-        run_tasks: Callable[[list[ResolvedTask]], list[Task] | list[PreviousTask]],
-    ) -> list[EvalLog]:
-        logs: list[EvalLog] = []
-        for task_group in task_groups:
-            # alias
-            group_models, group_tasks = task_group
-
-            # info log
-            logger.info(
-                f"eval_set (running task group): {','.join([task.task.name for task in group_tasks])}: {group_models}"
-            )
-
-            # run the evals
-            logs.extend(
-                run_eval(
-                    tasks=run_tasks(group_tasks),
-                    models=group_models.models,
-                )
-            )
-
-        return logs
+    # initialise display (otherwise eval_init will set it to full)
+    display = init_display_type(display)
+    if display == "conversation":
+        raise RuntimeError("eval_set cannot be used with conversation display.")
 
     # resolve tasks
     models, _, resolved_tasks = eval_init(
@@ -283,6 +265,7 @@ def eval_set(
     retry_connections = retry_connections or 0.5
     retry_cleanup = retry_cleanup is not False
     max_connections = starting_max_connections(models, GenerateConfig(**kwargs))
+    max_tasks = max_tasks if max_tasks is not None else len(models)
 
     # prepare console/status
     console = rich.get_console()
@@ -331,15 +314,11 @@ def eval_set(
         pending_tasks = [
             task[1] for task in all_tasks if task[0] not in log_task_identifiers
         ]
-        task_groups = schedule_pending_tasks(pending_tasks)
 
         # we have some pending tasks yet to run, run them
-        if len(task_groups) > 0:
+        if len(pending_tasks) > 0:
             # run the tasks
-            run_logs = run_task_groups(
-                task_groups=task_groups,
-                run_tasks=lambda tasks: [task.task for task in tasks],
-            )
+            run_logs = run_eval(pending_tasks)
 
             # if this was the entire list of resolved tasks, return results
             if len(pending_tasks) == len(all_tasks):
@@ -365,42 +344,10 @@ def eval_set(
                     for task in resolved_tasks
                     if task_identifier(task) in failed_task_identifiers
                 ]
-                task_groups = schedule_retry_tasks(failed_tasks)
-
-                # execute task groups (run previous task so we get the samples from the log)
-                def run_previous_tasks(tasks: list[ResolvedTask]) -> list[PreviousTask]:
-                    def task_to_failed_log(task: ResolvedTask) -> Log:
-                        resolved_task_identifier = task_identifier(task)
-                        return next(
-                            log
-                            for log in failed_logs
-                            if log.task_identifier == resolved_task_identifier
-                        )
-
-                    previous_tasks: list[PreviousTask] = []
-                    for task, log in zip(tasks, map(task_to_failed_log, tasks)):
-                        # NOTE: we used to try to recreate registry objects by
-                        # by just passing the task name, but that didn't work
-                        # when evals were run from another directory. we may
-                        # want to bring this back but we'd need to resolve the
-                        # directory issues.
-
-                        # deepcopy so the same instance is not run twice
-                        prev_task = deepcopy(task.task)
-
-                        previous_tasks.append(
-                            PreviousTask(
-                                id=log.header.eval.task_id,
-                                task=prev_task,
-                                task_args=resolve_task_args(task.task),
-                                log=read_eval_log(log.info),
-                            )
-                        )
-
-                    return previous_tasks
-
-                retried_logs = run_task_groups(
-                    task_groups=task_groups, run_tasks=run_previous_tasks
+
+                # run previous tasks (no models passed b/c previous task already carries its model)
+                retried_logs = run_eval(
+                    tasks=as_previous_tasks(failed_tasks, failed_logs)
                 )
 
                 # return success
@@ -443,6 +390,42 @@ def eval_set(
     return success, results
 
 
+# convert resolved tasks to previous tasks
+def as_previous_tasks(
+    tasks: list[ResolvedTask], failed_logs: list[Log]
+) -> list[PreviousTask]:
+    def task_to_failed_log(task: ResolvedTask) -> Log:
+        resolved_task_identifier = task_identifier(task)
+        return next(
+            log
+            for log in failed_logs
+            if log.task_identifier == resolved_task_identifier
+        )
+
+    previous_tasks: list[PreviousTask] = []
+    for task, log in zip(tasks, map(task_to_failed_log, tasks)):
+        # NOTE: we used to try to recreate registry objects by
+        # by just passing the task name, but that didn't work
+        # when evals were run from another directory. we may
+        # want to bring this back but we'd need to resolve the
+        # directory issues.
+
+        # deepcopy so the same instance is not run twice
+        prev_task = deepcopy(task.task)
+
+        previous_tasks.append(
+            PreviousTask(
+                id=log.header.eval.task_id,
+                task=prev_task,
+                task_args=resolve_task_args(task.task),
+                model=task.model,
+                log=read_eval_log(log.info),
+            )
+        )
+
+    return previous_tasks
+
+
 # filters to determine when we are done
 
 
@@ -574,7 +557,7 @@ def task_identifier(task: ResolvedTask | EvalLog) -> str:
         task_file = task.eval.task_file or ""
         task_name = task.eval.task
         task_args = task.eval.task_args
-        model = task.eval.model
+        model = str(task.eval.model)
 
     # hash for task args
     task_args_hash = hashlib.sha256(
@@ -617,80 +600,6 @@ class ModelList:
         return ",".join(model_names)
 
 
-class TaskGroup(NamedTuple):
-    models: ModelList
-    tasks: list[ResolvedTask]
-
-
-# group into models => tasks for maximum parallelism
-def schedule_pending_tasks(pending_tasks: list[ResolvedTask]) -> list[TaskGroup]:
-    # build a map of task identifiers and the models they target
-    task_id_model_targets: dict[str, ModelList] = {}
-    for pending_task in pending_tasks:
-        task_id = task_identifier_without_model(task_identifier(pending_task))
-        if task_id not in task_id_model_targets:
-            task_id_model_targets[task_id] = ModelList([])
-        if pending_task.model not in task_id_model_targets[task_id].models:
-            task_id_model_targets[task_id].models.append(pending_task.model)
-
-    # build a list of unique model targets
-    unique_model_targets: Set[ModelList] = set(task_id_model_targets.values())
-
-    # create schedule
-    schedule: list[TaskGroup] = [
-        TaskGroup(models=model_target, tasks=[])
-        for model_target in unique_model_targets
-    ]
-
-    for models, tasks in schedule:
-        # which task ids have this set of models
-        task_ids: list[str] = []
-        for task_id, task_models in task_id_model_targets.items():
-            if task_models == models:
-                task_ids.append(task_id)
-
-        # find a task for each of these ids
-        for task_id in task_ids:
-            tasks.append(
-                next(
-                    (
-                        task
-                        for task in pending_tasks
-                        if task_id
-                        == task_identifier_without_model(task_identifier(task))
-                    )
-                )
-            )
-
-    # deterministic return order
-    schedule.sort(key=lambda x: str(x[0]))
-
-    return schedule
-
-
-# group into model => tasks (can't do multiple models b/c these are PreviousTask
-# instances (and therefore model/task pair specific -- we don't want to create
-# multiple instances of these tasks)
-def schedule_retry_tasks(retry_tasks: list[ResolvedTask]) -> list[TaskGroup]:
-    # build a list of unique model targets
-    unique_model_targets: Set[ModelList] = set()
-    for retry_task in retry_tasks:
-        unique_model_targets.add(ModelList([retry_task.model]))
-
-    # create a task group for reach model target
-    schedule: list[TaskGroup] = []
-    for model_target in unique_model_targets:
-        group_tasks = [
-            task for task in retry_tasks if ModelList([task.model]) == model_target
-        ]
-        schedule.append(TaskGroup(model_target, group_tasks))
-
-    # deterministic return order
-    schedule.sort(key=lambda x: str(x[0]))
-
-    return schedule
-
-
 def starting_max_connections(models: list[Model], config: GenerateConfig) -> int:
     # if there is an explicit config use that
     if config.max_connections is not None:

inspect_ai/_eval/loader.py

@@ -2,7 +2,6 @@ import ast
 import contextlib
 import inspect
 import os
-from dataclasses import dataclass, field
 from importlib.machinery import SourceFileLoader
 from importlib.util import module_from_spec, spec_from_loader
 from logging import getLogger
@@ -12,6 +11,7 @@ from typing import Any, Callable, Tuple, cast
 
 from typing_extensions import overload
 
+from inspect_ai._eval.task.resolved import ResolvedTask
 from inspect_ai._eval.task.util import task_file, task_run_dir
 from inspect_ai._util._async import configured_async_backend
 from inspect_ai._util.decorator import parse_decorators
@@ -26,44 +26,26 @@ from inspect_ai._util.registry import (
     registry_lookup,
     registry_params,
 )
-from inspect_ai.model import Model, ModelName
+from inspect_ai.model import Model
 from inspect_ai.scorer._scorer import Scorer, ScorerSpec, scorer_create
 from inspect_ai.solver._bridge import bridge
 from inspect_ai.solver._solver import Solver, SolverSpec
 from inspect_ai.util import SandboxEnvironmentSpec, SandboxEnvironmentType
-from inspect_ai.util._sandbox.environment import resolve_sandbox_environment
+from inspect_ai.util._sandbox.environment import (
+    resolve_sandbox_environment,
+)
 from inspect_ai.util._sandbox.registry import registry_find_sandboxenv
 
 from .list import task_files
 from .registry import task_create
-from .task import PreviousTask, Task, TaskInfo, Tasks
+from .task import PreviousTask, Task, TaskInfo
 from .task.constants import TASK_FILE_ATTR, TASK_RUN_DIR_ATTR
-from .task.run import EvalSampleSource, eval_log_sample_source
+from .task.run import eval_log_sample_source
+from .task.tasks import Tasks
 
 logger = getLogger(__name__)
 
 
-@dataclass(frozen=True)
-class ResolvedTask:
-    task: Task
-    task_args: dict[str, Any]
-    task_file: str | None
-    model: Model
-    sandbox: SandboxEnvironmentSpec | None
-    sequence: int
-    id: str | None = field(default=None)
-    sample_source: EvalSampleSource | None = field(default=None)
-
-    @property
-    def has_sandbox(self) -> bool:
-        if self.sandbox:
-            return True
-        else:
-            return any(
-                [True if sample.sandbox else False for sample in self.task.dataset]
-            )
-
-
 def resolve_tasks(
     tasks: Tasks,
     task_args: dict[str, Any],
@@ -76,16 +58,22 @@ def resolve_tasks(
                 task=task,
                 task_args=resolve_task_args(task),
                 task_file=task_file(task, relative=True),
-                model=model,
+                model=task.model or model,
                 sandbox=resolve_task_sandbox(task, sandbox),
                 sequence=sequence,
             )
             for sequence, task in enumerate(tasks)
         ]
 
+    # reflect resolved tasks right back
+    if isinstance(tasks, ResolvedTask):
+        return [tasks]
+    elif isinstance(tasks, list) and isinstance(tasks[0], ResolvedTask):
+        return cast(list[ResolvedTask], tasks)
+
     # take empty lists out of play
     if isinstance(tasks, list) and len(tasks) == 0:
-        return as_resolved_tasks(load_tasks(None, model, task_args))
+        return as_resolved_tasks(load_tasks(None, task_args))
 
     # simple cases of passing us Task objects
     if isinstance(tasks, Task):
@@ -109,9 +97,7 @@ def resolve_tasks(
                 loaded_task = previous_task.task
             else:
                 loaded_task_args = previous_task.task_args
-                loaded_task = load_tasks([previous_task.task], model, loaded_task_args)[
-                    0
-                ]
+                loaded_task = load_tasks([previous_task.task], loaded_task_args)[0]
             loaded_tasks.append(loaded_task)
             loaded_tasks_args.append(loaded_task_args)
 
@@ -120,7 +106,7 @@ def resolve_tasks(
                 task=loaded_task,
                 task_args=loaded_task_args,
                 task_file=previous_task.log.eval.task_file,
-                model=model,
+                model=previous_task.model or loaded_task.model or model,
                 sandbox=previous_task.log.eval.sandbox,
                 sequence=sequence,
                 id=previous_task.id,
@@ -153,19 +139,14 @@ def resolve_tasks(
         tasks = [tasks]
 
     # done! let's load the tasks
-    return as_resolved_tasks(
-        load_tasks(cast(list[str] | None, tasks), model, task_args)
-    )
+    return as_resolved_tasks(load_tasks(cast(list[str] | None, tasks), task_args))
 
 
 def resolve_task_args(task: Task) -> dict[str, Any]:
     # was the task instantiated via the registry or a decorator?
     # if so then we can get the task_args from the registry.
     try:
-        # filter out model as that's dyanmic and automatically passed
         task_args = dict(registry_params(task))
-        if "model" in task_args:
-            del task_args["model"]
         return task_args
 
     # if it wasn't instantiated via the registry or a decorator
@@ -217,34 +198,29 @@ def resolve_task_sandbox(
 
 
 def load_tasks(
-    task_specs: list[str] | None, model: Model, task_args: dict[str, Any] = {}
+    task_specs: list[str] | None, task_args: dict[str, Any] = {}
 ) -> list[Task]:
     """Load one more more tasks (if no tasks are specified, load from the current working directory"""
-    # determine ModelName object for task creation parameterized by model
-    model_name = ModelName(model)
     # load tasks
     return [
         spec
         for task_spec in (task_specs if task_specs else [Path.cwd().as_posix()])
-        for spec in load_task_spec(task_spec, model_name, task_args)
+        for spec in load_task_spec(task_spec, task_args)
     ]
 
 
-def load_task_spec(
-    task_spec: str, model: ModelName, task_args: dict[str, Any] = {}
-) -> list[Task]:
+def load_task_spec(task_spec: str, task_args: dict[str, Any] = {}) -> list[Task]:
     # task in a python package
     if registry_lookup("task", task_spec) is not None:
         # create the task from a python package
-        return [task_create(task_spec, model, **task_args)]
+        return [task_create(task_spec, **task_args)]
     else:
         # load tasks from glob
-        return create_tasks([task_spec], model, task_args)
+        return create_tasks([task_spec], task_args)
 
 
 def create_tasks(
     globs: list[str],
-    model: ModelName,
     task_args: dict[str, Any] = {},
     root_dir: Path | None = None,
 ) -> list[Task]:
@@ -261,9 +237,7 @@ def create_tasks(
         if spec_split[1] is not None:
             task_path = Path(spec_split[0])
             load_file_tasks(task_path.absolute())
-            tasks.extend(
-                create_file_tasks(task_path, model, [spec_split[1]], task_args)
-            )
+            tasks.extend(create_file_tasks(task_path, [spec_split[1]], task_args))
         else:
             # if the glob is the root dir then set it to empty (will result in
             # enumeration of the root dir)
@@ -271,7 +245,7 @@ def create_tasks(
             files = task_files(target, root_dir)
             files = sorted(files, key=lambda f: f.as_posix())
             for file in files:
-                tasks.extend(create_file_tasks(file, model, None, task_args))
+                tasks.extend(create_file_tasks(file, None, task_args))
     return tasks
 
 
@@ -282,7 +256,6 @@ def load_file_tasks(file: Path) -> None:
 
 def create_file_tasks(
     file: Path,
-    model: ModelName,
     task_specs: list[str] | list[RegistryInfo] | None = None,
     task_args: dict[str, Any] = {},
 ) -> list[Task]:
@@ -302,7 +275,7 @@ def create_file_tasks(
             # create the task from the loaded source file and
             # note that it was loaded from this directory
             # (will be used later to ensure it runs in the directory)
-            task = task_create(task_spec, model, **task_args)
+            task = task_create(task_spec, **task_args)
             setattr(task, TASK_FILE_ATTR, file.as_posix())
             setattr(task, TASK_RUN_DIR_ATTR, run_dir)
             tasks.append(task)

inspect_ai/_eval/registry.py

@@ -1,6 +1,5 @@
 import inspect
 import logging
-from copy import deepcopy
 from functools import wraps
 from pathlib import Path
 from typing import Any, Callable, TypeVar, cast, overload
@@ -16,7 +15,6 @@ from inspect_ai._util.registry import (
     registry_name,
     registry_tag,
 )
-from inspect_ai.model import ModelName
 
 from .task import Task
 from .task.constants import TASK_FILE_ATTR, TASK_RUN_DIR_ATTR
@@ -54,7 +52,7 @@ def task_register(
     return task
 
 
-def task_create(name: str, model: ModelName, **kwargs: Any) -> Task:
+def task_create(name: str, **kwargs: Any) -> Task:
     r"""Create a Task based on its registered name.
 
     Tasks can be a function that returns a Task or a
@@ -62,17 +60,11 @@ def task_create(name: str, model: ModelName, **kwargs: Any) -> Task:
 
     Args:
         name (str): Name of task (Optional, defaults to object name)
-        model (ModelName): Model name
         **kwargs (dict): Optional creation arguments for the task
 
     Returns:
         Task with registry info attribute
     """
-    # bring in model arg (first deepcopy as we will mutate it)
-    # add model to task_args
-    kwargs = deepcopy(kwargs)
-    kwargs[MODEL_PARAM] = model
-
     # match kwargs params to signature (warn if param not found)
     # (note that we always pass the 'model' param but tasks aren't
     # required to consume it, so we don't warn for 'model')
@@ -85,7 +77,6 @@ def task_create(name: str, model: ModelName, **kwargs: Any) -> Task:
     for param in kwargs.keys():
         if param in task_params:
             task_args[param] = kwargs[param]
-        elif param != MODEL_PARAM:
             if "kwargs" in task_params:
                 task_args[param] = kwargs[param]
             else:

inspect_ai/_eval/run.py

@@ -2,7 +2,7 @@ import functools
 import logging
 import os
 import sys
-from typing import Any, Awaitable, Callable, Set, cast
+from typing import Awaitable, Callable, Set, cast
 
 from inspect_ai._util.trace import trace_action
 
@@ -44,11 +44,11 @@ from inspect_ai.util._sandbox.environment import (
 from inspect_ai.util._sandbox.registry import registry_find_sandboxenv
 
 from .loader import (
-    ResolvedTask,
     as_solver_spec,
     solver_from_spec,
 )
 from .task.log import TaskLogger
+from .task.resolved import ResolvedTask
 from .task.run import TaskRunOptions, task_run
 from .task.rundir import task_run_dir_switching
 from .task.sandbox import TaskSandboxEnvironment, resolve_sandbox_for_task
@@ -64,7 +64,6 @@ async def eval_run(
     eval_config: EvalConfig,
     eval_sandbox: SandboxEnvironmentType | None,
     recorder: Recorder,
-    model_args: dict[str, Any],
     epochs_reducer: list[ScoreReducer] | None = None,
     solver: Solver | SolverSpec | None = None,
     tags: list[str] | None = None,
@@ -200,7 +199,7 @@ async def eval_run(
                     sandbox=resolved_task.sandbox,
                     task_attribs=task.attribs,
                     task_args=resolved_task.task_args,
-                    model_args=model_args,
+                    model_args=resolved_task.model.model_args,
                     eval_config=task_eval_config,
                     metadata=task.metadata,
                     recorder=recorder,

inspect_ai/_eval/task/__init__.py

@@ -1,4 +1,10 @@
-from .task import Task, TaskInfo, PreviousTask, Tasks, task_with  # noqa: I001, F401
+from .task import Task, TaskInfo, PreviousTask, task_with  # noqa: I001, F401
 from .epochs import Epochs
 
-__all__ = ["Epochs", "Task", "TaskInfo", "PreviousTask", "Tasks", "task_with"]
+__all__ = [
+    "Epochs",
+    "Task",
+    "TaskInfo",
+    "PreviousTask",
+    "task_with",
+]

inspect_ai/_eval/task/log.py

@@ -1,5 +1,6 @@
 from importlib import metadata as importlib_metadata
-from typing import Any, Literal, cast
+from inspect import isgenerator
+from typing import Any, Iterator, Literal, cast
 
 from shortuuid import uuid
 
@@ -83,6 +84,13 @@ class TaskLogger:
             del model_args["api_key"]
         model_args = {k: v for k, v in model_args.items() if not k.startswith("aws_")}
 
+        # don't try to serialise generators
+        model_args = {
+            k: v
+            for k, v in model_args.items()
+            if not isgenerator(v) and not isinstance(v, Iterator)
+        }
+
         # cwd_relative_path for sandbox config
         if sandbox and isinstance(sandbox.config, str):
             sandbox = SandboxEnvironmentSpec(

inspect_ai/_eval/task/resolved.py

@@ -0,0 +1,35 @@
+from dataclasses import dataclass, field
+from typing import Any, Set
+
+from inspect_ai._eval.task import Task
+from inspect_ai._eval.task.run import EvalSampleSource
+from inspect_ai.model import Model
+from inspect_ai.util import SandboxEnvironmentSpec
+
+
+@dataclass(frozen=True)
+class ResolvedTask:
+    task: Task
+    task_args: dict[str, Any]
+    task_file: str | None
+    model: Model
+    sandbox: SandboxEnvironmentSpec | None
+    sequence: int
+    id: str | None = field(default=None)
+    sample_source: EvalSampleSource | None = field(default=None)
+
+    @property
+    def has_sandbox(self) -> bool:
+        if self.sandbox:
+            return True
+        else:
+            return any(
+                [True if sample.sandbox else False for sample in self.task.dataset]
+            )
+
+
+def resolved_model_names(tasks: list[ResolvedTask]) -> list[str]:
+    models: Set[str] = set()
+    for task in tasks:
+        models.add(str(task.model))
+    return list(models)