inspect-ai 0.3.96__py3-none-any.whl → 0.3.98__py3-none-any.whl

inspect_ai/_eval/eval.py

@@ -28,7 +28,7 @@ from inspect_ai._util.error import PrerequisiteError
 from inspect_ai._util.file import absolute_file_path
 from inspect_ai._util.logger import warn_once
 from inspect_ai._util.platform import platform_init
-from inspect_ai._util.registry import registry_lookup
+from inspect_ai._util.registry import registry_lookup, registry_package_name
 from inspect_ai.approval._apply import init_tool_approval
 from inspect_ai.approval._policy import (
     ApprovalPolicy,
@@ -770,7 +770,15 @@ async def eval_retry_async(
             task = f"{task_file}@{task_name}"
         else:
             if registry_lookup("task", task_name) is None:
-                raise FileNotFoundError(f"Task '{task_name}' not found.")
+                # if this object is in a package then let the user know
+                # that they need to register it to work with eval-retry
+                package_name = registry_package_name(task_name)
+                if package_name is not None:
+                    raise FileNotFoundError(
+                        f"Task '{task_name}' is located in package '{package_name}' but has not been registered so cannot be retried. See https://inspect.aisi.org.uk/tasks.html#packaging for additional details on registering tasks in packages."
+                    )
+                else:
+                    raise FileNotFoundError(f"Task '{task_name}' not found.")
             task = task_name
 
         # see if there is solver spec in the eval log

inspect_ai/_eval/run.py

@@ -475,7 +475,12 @@ async def startup_sandbox_environments(
     sandboxenvs: Set[TaskSandboxEnvironment] = set()
     for task in tasks:
         # resolve each sample and add to sandboxenvs
-        dataset = slice_dataset(task.task.dataset, config.limit, config.sample_id)
+        resolved_task_sample_ids = resolve_task_sample_ids(
+            task.task.name, config.sample_id
+        )
+        dataset = slice_dataset(
+            task.task.dataset, config.limit, resolved_task_sample_ids
+        )
         for sample in dataset:
             sandbox = await resolve_sandbox_for_task_and_sample(
                 eval_sandbox, task.task, sample

inspect_ai/_eval/task/util.py

@@ -1,7 +1,11 @@
 import os
+import reprlib
 from copy import deepcopy
+from logging import getLogger
 from typing import cast
 
+from inspect_ai._util.error import PrerequisiteError
+from inspect_ai._util.logger import warn_once
 from inspect_ai._util.path import cwd_relative_path
 from inspect_ai.dataset import Sample
 from inspect_ai.dataset._dataset import Dataset
@@ -10,6 +14,8 @@ from inspect_ai.model import ChatMessage, ChatMessageUser
 from ..task import Task
 from .constants import TASK_FILE_ATTR, TASK_RUN_DIR_ATTR
 
+logger = getLogger(__name__)
+
 
 def sample_messages(sample: Sample) -> list[ChatMessage]:
     if isinstance(sample.input, str):
@@ -47,9 +53,32 @@ def slice_dataset(
         return id if isinstance(id, str) else str(id).zfill(20)
 
     if sample_id is not None:
-        sample_id = sample_id if isinstance(sample_id, list) else [sample_id]
-        sample_id = [normalise(id) for id in sample_id]
-        return dataset.filter(lambda sample: normalise(sample.id) in sample_id)
+        # reduce to list of normalized sample ids
+        sample_ids = sample_id if isinstance(sample_id, list) else [sample_id]
+        sample_id = [normalise(id) for id in sample_ids]
+
+        # validate all the sample ids and warn if they aren't in the dataset
+        all_sample_ids_raw = [sample.id for sample in dataset]
+        all_sample_ids = [normalise(id) for id in all_sample_ids_raw]
+        for id in sample_id:
+            if id not in all_sample_ids:
+                warn_once(
+                    logger, f"sample id '{id}' not found in dataset '{dataset.name}'."
+                )
+
+        # filter the dataset
+        filtered = dataset.filter(lambda sample: normalise(sample.id) in sample_id)
+
+        # raise error if we got no hits
+        if len(filtered) == 0:
+            filter = ",".join([str(id) for id in sample_id])
+            r = reprlib.Repr()
+            r.maxlist = 8
+            raise PrerequisiteError(
+                f"No matches in dataset '{dataset.name}' for sample_id filter '{filter}'\n({dataset.name} ids: {r.repr(all_sample_ids_raw)})"
+            )
+
+        return filtered
     else:
         dataset_limit = (
             slice(0, len(dataset))

inspect_ai/_util/registry.py

@@ -183,6 +183,13 @@ def registry_lookup(type: RegistryType, name: str) -> object | None:
         return o
 
 
+def registry_package_name(name: str) -> str | None:
+    if name.find("/") != -1 and name.find(".") == -1:
+        return name.split("/")[0]
+    else:
+        return None
+
+
 def registry_find(predicate: Callable[[RegistryInfo], bool]) -> list[object]:
     r"""Find objects in the registry that match the passed predicate.
 

inspect_ai/_util/timer.py

@@ -0,0 +1,13 @@
+import time
+from contextlib import contextmanager
+from typing import Iterator
+
+
+@contextmanager
+def execution_timer(name: str | None = None) -> Iterator[None]:
+    start_time = time.perf_counter()
+    yield
+    end_time = time.perf_counter()
+    print(
+        f"{name if name else ''} execution time: {end_time - start_time:.6f} seconds".strip()
+    )