inspect-ai 0.3.62__py3-none-any.whl → 0.3.64__py3-none-any.whl

inspect_ai/approval/_approval.py

@@ -17,6 +17,8 @@ Possible values:
 
 
 class Approval(BaseModel):
+    """Approval details (decision, explanation, etc.)"""
+
     decision: ApprovalDecision
     """Approval decision."""
 

inspect_ai/approval/_approver.py

@@ -20,10 +20,10 @@ class Approver(Protocol):
         Approve or reject a tool call.
 
         Args:
-            message (str): Message genreated by the model along with the tool call.
-            call (ToolCall): The tool call to be approved.
-            view (ToolCallView): Custom rendering of tool context and call.
-            state (state | None): The current task state, if available.
+            message: Message genreated by the model along with the tool call.
+            call: The tool call to be approved.
+            view: Custom rendering of tool context and call.
+            state: The current task state, if available.
 
         Returns:
             Approval: An Approval object containing the decision and explanation.

inspect_ai/approval/_auto.py

@@ -11,7 +11,7 @@ def auto_approver(decision: ApprovalDecision = "approve") -> Approver:
     """Automatically apply a decision to tool calls.
 
     Args:
-       decision (ApprovalDecision): Decision to apply.
+       decision: Decision to apply.
 
     Returns:
        Approver: Auto approver.

inspect_ai/approval/_human/approver.py

@@ -14,6 +14,9 @@ def human_approver(
 ) -> Approver:
     """Interactive human approver.
 
+    Args:
+       choices: Choices to present to human.
+
     Returns:
        Approver: Interactive human approver.
     """

inspect_ai/approval/_policy.py

@@ -20,8 +20,13 @@ from ._call import call_approver, record_approval
 
 @dataclass
 class ApprovalPolicy:
+    """Policy mapping approvers to tools."""
+
     approver: Approver
+    """Approver for policy."""
+
     tools: str | list[str]
+    """Tools to use this approver for (can be full tool names or globs)."""
 
 
 def policy_approver(policies: str | list[ApprovalPolicy]) -> Approver:

inspect_ai/approval/_registry.py

@@ -31,11 +31,11 @@ def approver(*args: Any, name: str | None = None, **attribs: Any) -> Any:
     Args:
       *args: Function returning `Approver` targeted by
         plain approver decorator without attributes (e.g. `@approver`)
-      name (str | None):
+      name:
         Optional name for approver. If the decorator has no name
         argument then the name of the function
         will be used to automatically assign a name.
-      **attribs: (dict[str,Any]): Additional approver attributes.
+      **attribs: Additional approver attributes.
 
     Returns:
         Approver with registry attributes.

inspect_ai/dataset/_dataset.py

@@ -27,6 +27,8 @@ MT = TypeVar("MT", bound=BaseModel)
 
 
 class Sample(BaseModel):
+    r"""Sample for an evaluation task."""
+
     def __init__(
         self,
         input: str | list[ChatMessage],
@@ -38,22 +40,22 @@ class Sample(BaseModel):
         files: dict[str, str] | None = None,
         setup: str | None = None,
     ) -> None:
-        r"""Sample to be used in an evaluation task.
+        r"""Create a Sample.
 
         Args:
-            input (str | list[ChatMessage]): The input to be submitted to the model.
-            choices (list[str] | None): Optional. List of available answer choices
-            (used only for multiple-choice evals).
-            target (str | list[str]): Optional. Ideal target output. May be a literal value
+            input: The input to be submitted to the model.
+            choices: Optional. List of available answer choices
+                (used only for multiple-choice evals).
+            target: Optional. Ideal target output. May be a literal value
                 or narrative text to be used by a model grader.
-            id (int | str | None): Optional. Unique identifier for sample.
-            metadata (dict[str,Any] | None): Optional. Arbitrary metadata associated with the sample.
-            sandbox (SandboxEnvironmentType | None): Sandbox environment type
-            (or optionally a str or tuple with a shorthand spec)
-            files (dict[str, str] | None): Optional. Files that go along with the sample (copied to
-            SandboxEnvironment). Files can be paths, inline text, or inline binary (base64 encoded data URL).
-            setup (str | None): Optional. Setup script to run for sample (run
-            within default SandboxEnvironment).
+            id: Optional. Unique identifier for sample.
+            metadata: Optional. Arbitrary metadata associated with the sample.
+                sandbox (SandboxEnvironmentType | None): Sandbox environment type (or optionally a str or tuple with a shorthand spec)
+            sandbox: Optional. Sandbox specification for this sample.
+            files: Optional. Files that go along with the sample (copied to
+                SandboxEnvironment). Files can be paths, inline text, or inline binary (base64 encoded data URL).
+            setup: Optional. Setup script to run for sample (run
+                within default SandboxEnvironment).
         """
         super().__init__(
             input=input,
@@ -156,14 +158,6 @@ class Dataset(Sequence[Sample], abc.ABC):
     @abc.abstractmethod
     def __len__(self) -> int: ...
 
-    @abc.abstractmethod
-    def shuffle(self, seed: int | None = None) -> None:
-        """Shuffle the order of the dataset (in place).
-
-        Args:
-           seed: (int | None): Random seed for shuffling (optional).
-        """
-
     @abc.abstractmethod
     def sort(
         self,
@@ -177,8 +171,8 @@ class Dataset(Sequence[Sample], abc.ABC):
         The key function defaults to measuring the length of the sample's input field.
 
         Args:
-            reverse (bool): if true, sort in descending order. Defaults to False.
-            key (Callable[[Any], Any]): a callable mapping each item to a numeric value (optional, defaults to sample_input_len).
+            reverse: If `Treu`, sort in descending order. Defaults to False.
+            key: a callable mapping each item to a numeric value (optional, defaults to sample_input_len).
         """
 
     @abc.abstractmethod
@@ -188,28 +182,33 @@ class Dataset(Sequence[Sample], abc.ABC):
         """Filter the dataset using a predicate.
 
         Args:
-          predicate (Callable[[Sample], bool]): Filtering function.
-          name (str | None): Name for filtered dataset (optional).
+          predicate: Filtering function.
+          name: Name for filtered dataset (optional).
 
         Returns:
           Filtered dataset.
         """
 
+    @abc.abstractmethod
+    def shuffle(self, seed: int | None = None) -> None:
+        """Shuffle the order of the dataset (in place).
+
+        Args:
+           seed: Random seed for shuffling (optional).
+        """
+
+    @abc.abstractmethod
+    def shuffle_choices(self, seed: int | None = None) -> None:
+        """Shuffle the order of the choices with each sample.
+
+        Args:
+           seed: Random seed for shuffling (optional).
+        """
+
 
 @dataclass
 class FieldSpec:
-    r"""Specification for mapping data source fields to sample fields.
-
-    Args:
-        input (str): Name of the field containing the sample input.
-        target (str): Name of the field containing the sample target.
-        choices (str): Optional. Name of field containing the list of answer choices.
-        id (str): Optional. Unique identifier for the sample.
-        metadata (list[str] | None): List of additional field names that should be read as metadata.
-        sandbox (str): Optional. Sandbox type along with optional config file
-        files (str): Optional. Files that go along with the sample.
-        setup (str): Optional. Setup script to run for sample .
-    """
+    r"""Specification for mapping data source fields to sample fields."""
 
     input: str = field(default="input")
     """Name of the field containing the sample input."""
@@ -315,6 +314,34 @@ class MemoryDataset(Dataset):
             random.shuffle(self.samples)
         self._shuffled = True
 
+    @override
+    def shuffle_choices(self, seed: int | None = None) -> None:
+        rand = random.Random(seed)
+        for sample in self.samples:
+            if not sample.choices:
+                continue
+            # The original positions
+            positions = list(range(len(sample.choices)))
+
+            # Shuffle the choices
+            rand.shuffle(positions)
+            shuffled_choices = [sample.choices[i] for i in positions]
+
+            # Map of original position / target letter
+            position_map = {i: chr(65 + new_i) for new_i, i in enumerate(positions)}
+
+            # Update to the shuffled choices and target
+            sample.choices = shuffled_choices
+            sample.target = self._remap_target(sample.target, position_map=position_map)
+
+    def _remap_target(
+        self, target: str | list[str], position_map: dict[int, str]
+    ) -> str | list[str]:
+        if isinstance(target, list):
+            return [position_map[ord(t) - 65] for t in target]
+        else:
+            return position_map[ord(target) - 65]
+
     @override
     def sort(
         self,

inspect_ai/dataset/_sources/csv.py

@@ -23,6 +23,7 @@ def csv_dataset(
     auto_id: bool = False,
     shuffle: bool = False,
     seed: int | None = None,
+    shuffle_choices: bool | int | None = None,
     limit: int | None = None,
     dialect: str = "unix",
     encoding: str = "utf-8",
@@ -34,29 +35,30 @@ def csv_dataset(
     r"""Read dataset from CSV file.
 
     Args:
-        csv_file (str): Path to CSV file. Can be a local filesystem path,
+        csv_file: Path to CSV file. Can be a local filesystem path,
             a path to an S3 bucket (e.g. "s3://my-bucket"), or an HTTPS URL.
             Use `fs_options` to pass arguments through to the `S3FileSystem` constructor.
-        sample_fields (FieldSpec | RecordToSample): Method of mapping underlying
+        sample_fields: Method of mapping underlying
             fields in the data source to Sample objects. Pass `None` if the data is already
             stored in `Sample` form (i.e. has "input" and "target" columns.); Pass a
             `FieldSpec` to specify mapping fields by name; Pass a `RecordToSample` to
             handle mapping with a custom function that returns one or more samples.
-        auto_id (bool): Assign an auto-incrementing ID for each sample.
-        shuffle (bool): Randomly shuffle the dataset order.
-        seed: (int | None): Seed used for random shuffle.
-        limit (int | None): Limit the number of records to read.
-        dialect (str): CSV dialect ("unix", "excel" or"excel-tab"). Defaults to "unix". See https://docs.python.org/3/library/csv.html#dialects-and-formatting-parameters for more details
-        encoding (str): Text encoding for file (defaults to "utf-8").
-        name (str): Optional name for dataset (for logging). If not specified,
+        auto_id: Assign an auto-incrementing ID for each sample.
+        shuffle: Randomly shuffle the dataset order.
+        seed: Seed used for random shuffle.
+        shuffle_choices: Whether to shuffle the choices. If an int is passed, this will be used as the seed when shuffling.
+        limit: Limit the number of records to read.
+        dialect: CSV dialect ("unix", "excel" or"excel-tab"). Defaults to "unix". See https://docs.python.org/3/library/csv.html#dialects-and-formatting-parameters for more details
+        encoding: Text encoding for file (defaults to "utf-8").
+        name: Optional name for dataset (for logging). If not specified,
             defaults to the stem of the filename
-        fs_options (dict[str, Any]): Optional. Additional arguments to pass through
+        fs_options: Optional. Additional arguments to pass through
             to the filesystem provider (e.g. `S3FileSystem`). Use `{"anon": True }`
             if you are accessing a public S3 bucket with no credentials.
-        fieldnames (list[str] | None): Optional. A list of fieldnames to use for the CSV.
+        fieldnames: Optional. A list of fieldnames to use for the CSV.
             If None, the values in the first row of the file will be used as the fieldnames.
             Useful for files without a header.
-        delimiter (str): Optional. The delimiter to use when parsing the file. Defaults to ",".
+        delimiter: Optional. The delimiter to use when parsing the file. Defaults to ",".
 
     Returns:
         Dataset read from CSV file.
@@ -86,6 +88,12 @@ def csv_dataset(
         if shuffle:
             dataset.shuffle(seed=seed)
 
+        # shuffle choices, if requested
+        if isinstance(shuffle_choices, int):
+            dataset.shuffle_choices(seed=shuffle_choices)
+        elif shuffle_choices is True:
+            dataset.shuffle_choices()
+
         # limit if requested
         if limit:
             return dataset[0:limit]

inspect_ai/dataset/_sources/file.py

@@ -16,6 +16,7 @@ def file_dataset(
     auto_id: bool = False,
     shuffle: bool = False,
     seed: int | None = None,
+    shuffle_choices: bool | int | None = None,
     limit: int | None = None,
     dialect: str = "unix",
     encoding: str = "utf-8",
@@ -40,6 +41,7 @@ def file_dataset(
         auto_id (bool): Assign an auto-incrementing ID for each sample.
         shuffle (bool): Randomly shuffle the dataset order.
         seed: (int | None): Seed used for random shuffle.
+        shuffle_choices: (bool | int | None): Whether to shuffle the choices. If an int is passed, this will be used as the seed when shuffling.
         limit (int | None): Limit the number of records to read.
         dialect (str): CSV dialect ("unix" or "excel", defaults to "unix"). Only
             applies to reading CSV files.
@@ -66,6 +68,7 @@ def file_dataset(
                 auto_id=auto_id,
                 shuffle=shuffle,
                 seed=seed,
+                shuffle_choices=shuffle_choices,
                 limit=limit,
                 encoding=encoding,
                 name=name,
@@ -78,6 +81,7 @@ def file_dataset(
                 auto_id=auto_id,
                 shuffle=shuffle,
                 seed=seed,
+                shuffle_choices=shuffle_choices,
                 limit=limit,
                 dialect=dialect,
                 encoding=encoding,

inspect_ai/dataset/_sources/hf.py

@@ -29,6 +29,7 @@ def hf_dataset(
     auto_id: bool = False,
     shuffle: bool = False,
     seed: int | None = None,
+    shuffle_choices: bool | int | None = None,
     limit: int | None = None,
     trust: bool = False,
     cached: bool = True,
@@ -40,35 +41,36 @@ def hf_dataset(
     `datasets` package, including remote datasets on Hugging Face Hub.
 
     Args:
-        path (str): Path or name of the dataset. Depending on path, the dataset
-          builder that is used comes from a generic dataset script (JSON, CSV,
-          Parquet, text etc.) or from the dataset script (a python file) inside
-          the dataset directory.
-        split (str): Which split of the data to load.
-        name (str | None): Name of the dataset configuration.
-        data_dir (str | None): data_dir of the dataset configuration
-          to read data from.
-        revision (str | None): Specific revision to load (e.g. "main", a branch
-          name, or a specific commit SHA). When using `revision` the `cached` option
-          is ignored and datasets are revalidated on Hugging Face before loading.
-        sample_fields (FieldSpec | RecordToSample): Method of mapping underlying
-          fields in the data source to Sample objects. Pass `None` if the data is already
-          stored in `Sample` form (i.e. has "input" and "target" columns.); Pass a
-          `FieldSpec` to specify mapping fields by name; Pass a `RecordToSample` to
+      path: Path or name of the dataset. Depending on path, the dataset
+        builder that is used comes from a generic dataset script (JSON, CSV,
+        Parquet, text etc.) or from the dataset script (a python file) inside
+        the dataset directory.
+      split: Which split of the data to load.
+      name: Name of the dataset configuration.
+      data_dir: data_dir of the dataset configuration
+        to read data from.
+      revision: Specific revision to load (e.g. "main", a branch
+        name, or a specific commit SHA). When using `revision` the `cached` option
+        is ignored and datasets are revalidated on Hugging Face before loading.
+      sample_fields: Method of mapping underlying
+        fields in the data source to Sample objects. Pass `None` if the data is already
+        stored in `Sample` form (i.e. has "input" and "target" columns.); Pass a
+        `FieldSpec` to specify mapping fields by name; Pass a `RecordToSample` to
           handle mapping with a custom function that returns one or more samples.
-        auto_id (bool): Assign an auto-incrementing ID for each sample.
-        shuffle (bool): Randomly shuffle the dataset order.
-        seed: (int | None): Seed used for random shuffle.
-        limit (int | None): Limit the number of records to read.
-        trust (bool): Whether or not to allow for datasets defined on the Hub
-          using a dataset script. This option should only be set to True for
-          repositories you trust and in which you have read the code, as it
-          will execute code present on the Hub on your local machine.
-        cached (bool): By default, datasets are read once from HuggingFace
-          Hub and then cached for future reads. Pass `cached=False` to force
-          re-reading the dataset from Hugging Face. Ignored when the `revision`
-          option is specified.
-        **kwargs (dict[str, Any]): Additional arguments to pass through to the
+      auto_id: Assign an auto-incrementing ID for each sample.
+      shuffle: Randomly shuffle the dataset order.
+      seed: Seed used for random shuffle.
+      shuffle_choices: Whether to shuffle the choices. If an int is passed, this will be used as the seed when shuffling.
+      limit: Limit the number of records to read.
+      trust: Whether or not to allow for datasets defined on the Hub
+        using a dataset script. This option should only be set to True for
+        repositories you trust and in which you have read the code, as it
+        will execute code present on the Hub on your local machine.
+      cached: By default, datasets are read once from HuggingFace
+        Hub and then cached for future reads. Pass `cached=False` to force
+        re-reading the dataset from Hugging Face. Ignored when the `revision`
+        option is specified.
+      **kwargs (dict[str, Any]): Additional arguments to pass through to the
           `load_dataset` function of the `datasets` package.
 
     Returns:
@@ -117,8 +119,16 @@ def hf_dataset(
         dataset = dataset.select(range(limit))
 
     # return the dataset
-    return MemoryDataset(
+    memory_dataset = MemoryDataset(
         samples=data_to_samples(dataset.to_list(), data_to_sample, auto_id),
         name=Path(path).stem if Path(path).exists() else path,
         location=path,
     )
+
+    # maybe shuffle the choices
+    if isinstance(shuffle_choices, int):
+        memory_dataset.shuffle_choices(seed=shuffle_choices)
+    elif shuffle_choices is True:
+        memory_dataset.shuffle_choices()
+
+    return memory_dataset

inspect_ai/dataset/_sources/json.py

@@ -25,6 +25,7 @@ def json_dataset(
     auto_id: bool = False,
     shuffle: bool = False,
     seed: int | None = None,
+    shuffle_choices: bool | int | None = None,
     limit: int | None = None,
     encoding: str = "utf-8",
     name: str | None = None,
@@ -38,22 +39,23 @@ def json_dataset(
     the `sample_fields` argument.
 
     Args:
-      json_file (str): Path to JSON file. Can be a local filesystem path or
+      json_file: Path to JSON file. Can be a local filesystem path or
         a path to an S3 bucket (e.g. "s3://my-bucket"). Use `fs_options`
         to pass arguments through to the `S3FileSystem` constructor.
-      sample_fields (FieldSpec | RecordToSample): Method of mapping underlying
+      sample_fields: Method of mapping underlying
         fields in the data source to `Sample` objects. Pass `None` if the data is already
         stored in `Sample` form (i.e. object with "input" and "target" fields); Pass a
         `FieldSpec` to specify mapping fields by name; Pass a `RecordToSample` to
         handle mapping with a custom function that returns one or more samples.
-      auto_id (bool): Assign an auto-incrementing ID for each sample.
-      shuffle (bool): Randomly shuffle the dataset order.
-      seed: (int | None): Seed used for random shuffle.
-      limit (int | None): Limit the number of records to read.
-      encoding (str): Text encoding for file (defaults to "utf-8").
-      name (str): Optional name for dataset (for logging). If not specified,
+      auto_id: Assign an auto-incrementing ID for each sample.
+      shuffle: Randomly shuffle the dataset order.
+      seed: Seed used for random shuffle.
+      shuffle_choices: Whether to shuffle the choices. If an int is passed, this will be used as the seed when shuffling.
+      limit: Limit the number of records to read.
+      encoding: Text encoding for file (defaults to "utf-8").
+      name: Optional name for dataset (for logging). If not specified,
         defaults to the stem of the filename.
-      fs_options (dict[str, Any]): Optional. Additional arguments to pass through
+      fs_options: Optional. Additional arguments to pass through
         to the filesystem provider (e.g. `S3FileSystem`). Use `{"anon": True }`
         if you are accessing a public S3 bucket with no credentials.
 
@@ -86,6 +88,12 @@ def json_dataset(
         if shuffle:
             dataset.shuffle(seed=seed)
 
+        # shuffle choices, if requested
+        if isinstance(shuffle_choices, int):
+            dataset.shuffle_choices(seed=shuffle_choices)
+        elif shuffle_choices is True:
+            dataset.shuffle_choices()
+
         # limit if requested
         if limit:
             return dataset[0:limit]

inspect_ai/log/__init__.py

@@ -22,6 +22,7 @@ from ._log import (
     EvalResults,
     EvalRevision,
     EvalSample,
+    EvalSampleLimit,
     EvalSampleReductions,
     EvalSampleScore,
     EvalScore,
@@ -61,6 +62,7 @@ __all__ = [
     "EvalResults",
     "EvalRevision",
     "EvalSample",
+    "EvalSampleLimit",
     "EvalSampleScore",
     "EvalSampleReductions",
     "EvalScore",

inspect_ai/log/_convert.py

@@ -20,12 +20,12 @@ def convert_eval_logs(
 
     Args:
         path (str): Path to source log file(s). Should be either a single
-          log file or a directory containing log files.
+            log file or a directory containing log files.
         to (Literal["eval", "json"]): Format to convert to. If a file is
-          already in the target format it will just be copied to the output dir.
+            already in the target format it will just be copied to the output dir.
         output_dir (str): Output directory to write converted log file(s) to.
         overwrite (bool): Overwrite existing log files (defaults to `False`,
-          raising an error if the output file path already exists).
+            raising an error if the output file path already exists).
     """
     from inspect_ai._display import display
 

inspect_ai/log/_file.py

@@ -3,6 +3,7 @@ import re
 from logging import getLogger
 from typing import Any, Callable, Generator, Literal, cast
 
+from pydantic import BaseModel
 from pydantic_core import to_json
 
 from inspect_ai._util._async import run_coroutine
@@ -22,7 +23,21 @@ from ._recorders import recorder_type_for_format, recorder_type_for_location
 logger = getLogger(__name__)
 
 
-class EvalLogInfo(FileInfo):
+class EvalLogInfo(BaseModel):
+    """File info and task identifiers for eval log."""
+
+    name: str
+    """Name of file."""
+
+    type: str
+    """Type of file (file or directory)"""
+
+    size: int
+    """File size in bytes."""
+
+    mtime: float | None
+    """File modification time (None if the file is a directory on S3)."""
+
     task: str
     """Task name."""
 
@@ -231,7 +246,7 @@ def write_log_dir_manifest(
 
 
 def read_eval_log(
-    log_file: str | FileInfo,
+    log_file: str | EvalLogInfo,
     header_only: bool = False,
     resolve_attachments: bool = False,
     format: Literal["eval", "json", "auto"] = "auto",
@@ -241,7 +256,7 @@ def read_eval_log(
     Args:
        log_file (str | FileInfo): Log file to read.
        header_only (bool): Read only the header (i.e. exclude
-         the "samples" and "logging" fields). Defaults to False.
+          the "samples" and "logging" fields). Defaults to False.
        resolve_attachments (bool): Resolve attachments (e.g. images)
           to their full content.
        format (Literal["eval", "json", "auto"]): Read from format
@@ -256,7 +271,7 @@ def read_eval_log(
 
 
 async def read_eval_log_async(
-    log_file: str | FileInfo,
+    log_file: str | EvalLogInfo,
     header_only: bool = False,
     resolve_attachments: bool = False,
     format: Literal["eval", "json", "auto"] = "auto",
@@ -304,13 +319,13 @@ async def read_eval_log_async(
 
 
 def read_eval_log_headers(
-    log_files: list[str] | list[FileInfo] | list[EvalLogInfo],
+    log_files: list[str] | list[EvalLogInfo],
 ) -> list[EvalLog]:
     return run_coroutine(read_eval_log_headers_async(log_files))
 
 
 async def read_eval_log_headers_async(
-    log_files: list[str] | list[FileInfo] | list[EvalLogInfo],
+    log_files: list[str] | list[EvalLogInfo],
 ) -> list[EvalLog]:
     return [
         await read_eval_log_async(log_file, header_only=True) for log_file in log_files
@@ -318,7 +333,7 @@ async def read_eval_log_headers_async(
 
 
 def read_eval_log_sample(
-    log_file: str | FileInfo,
+    log_file: str | EvalLogInfo,
     id: int | str,
     epoch: int = 1,
     resolve_attachments: bool = False,
@@ -347,7 +362,7 @@ def read_eval_log_sample(
 
 
 async def read_eval_log_sample_async(
-    log_file: str | FileInfo,
+    log_file: str | EvalLogInfo,
     id: int | str,
     epoch: int = 1,
     resolve_attachments: bool = False,
@@ -386,7 +401,7 @@ async def read_eval_log_sample_async(
 
 
 def read_eval_log_samples(
-    log_file: str | FileInfo,
+    log_file: str | EvalLogInfo,
     all_samples_required: bool = True,
     resolve_attachments: bool = False,
     format: Literal["eval", "json", "auto"] = "auto",