inspect-ai 0.3.62__py3-none-any.whl → 0.3.64__py3-none-any.whl

inspect_ai/scorer/_metric.py

@@ -1,3 +1,4 @@
+from dataclasses import dataclass, field
 from logging import getLogger
 from typing import (
     Any,
@@ -12,11 +13,15 @@ from typing import (
 
 from pydantic import BaseModel, Field
 
+from inspect_ai._util.error import PrerequisiteError
 from inspect_ai._util.registry import (
     RegistryInfo,
+    is_registry_object,
     registry_add,
     registry_create,
+    registry_info,
     registry_name,
+    registry_params,
     registry_tag,
 )
 
@@ -43,19 +48,12 @@ Value = Union[
 """Value provided by a score.
 
 Use the methods of `Score` to easily treat
-the Value as a simple scalar of various types.
+the `Value` as a simple scalar of various types.
 """
 
 
 class Score(BaseModel):
-    """Score generated by a scorer.
-
-    Args:
-       value (Value): Score value.
-       answer (str | None): Answer extracted from model output (optional).
-       explanation (str | None): Explanation of score (optional).
-       metadata (dict[str,Any]): Additional metadata related to the score.
-    """
+    """Score generated by a scorer."""
 
     value: Value
     """Score value."""
@@ -112,12 +110,7 @@ class Score(BaseModel):
 
 
 class SampleScore(BaseModel):
-    """Score for a Sample
-
-    Args:
-       score: Score
-       sample_id: (str | int | None) Unique id of a sample
-    """
+    """Score for a Sample."""
 
     score: Score
     """A score"""
@@ -125,6 +118,9 @@ class SampleScore(BaseModel):
     sample_id: str | int | None = Field(default=None)
     """A sample id"""
 
+    sample_metadata: dict[str, Any] | None = Field(default=None)
+    """Metadata from the sample"""
+
     scorer: str | None = Field(default=None)
     """Registry name of scorer that created this score."""
 
@@ -188,22 +184,57 @@ def value_to_float(
 
 
 @runtime_checkable
-class Metric(Protocol):
-    r"""Evaluate scores using a metric.
-
-    Args:
-        scores (list[Score]): List of scores.
+class MetricDeprecated(Protocol):
+    def __call__(self, scores: list[Score]) -> Value: ...
 
-    Returns:
-        Metric value
-    """
 
-    def __call__(self, scores: list[Score]) -> Value: ...
+@runtime_checkable
+class MetricProtocol(Protocol):
+    def __call__(self, scores: list[SampleScore]) -> Value:
+        r"""Compute a metric on a list of scores.
+
+        Args:
+          scores: List of scores.
+
+        Returns:
+          Metric value
+
+        Examples:
+          ```python
+          @metric
+          def mean() -> Metric:
+              def metric(scores: list[SampleScore]) -> Value:
+                  return np.mean([score.score.as_float() for score in scores]).item()
+              return metric
+          ```
+        """
+        ...
+
+
+Metric = MetricProtocol | MetricDeprecated
+"""Metric protocol.
+
+The Metric signature changed in release v0.3.64. Both
+the previous and new signatures are supported -- you
+should use `MetricProtocol` for new code as the
+depreacated signature will eventually be removed.
+"""
 
 
 P = ParamSpec("P")
 
 
+@dataclass(frozen=True)
+class MetricSpec:
+    """Scorer specification used to (re-)create scorers."""
+
+    metric: str
+    """Metric name"""
+
+    args: dict[str, Any] = field(default_factory=dict)
+    """Metric arguments."""
+
+
 def metric_register(metric: Callable[P, Metric], name: str = "") -> Callable[P, Metric]:
     r"""Register a function or class as a metric.
 
@@ -237,6 +268,26 @@ def metric_create(name: str, **kwargs: Any) -> Metric:
     return cast(Metric, registry_create("metric", name, **kwargs))
 
 
+def to_metric_specs(
+    metrics: list[Metric] | dict[str, list[Metric]],
+) -> list[MetricSpec] | dict[str, list[MetricSpec]]:
+    if isinstance(metrics, list):
+        return [as_metric_spec(m) for m in metrics]
+    else:
+        return {
+            k: [as_metric_spec(v) for v in metric_list]
+            for k, metric_list in metrics.items()
+        }
+
+
+def as_metric_spec(metric: Metric) -> MetricSpec:
+    if not is_registry_object(metric):
+        raise PrerequisiteError(
+            f"The metric {getattr(metric, '__name__', '<unknown>')} was not created by a function decorated with @metric so cannot be recorded."
+        )
+    return MetricSpec(metric=registry_info(metric).name, args=registry_params(metric))
+
+
 @overload
 def metric(name: str) -> Callable[[Callable[P, Metric]], Callable[P, Metric]]: ...
 
@@ -252,10 +303,18 @@ def metric(
     r"""Decorator for registering metrics.
 
     Args:
-        name: (str | MetricType):
-            Optional name for metric. If the decorator has no name
-            argument then the name of the underlying MetricType
-            will be used to automatically assign a name.
+      name: Optional name for metric. If the decorator has no name
+        argument then the name of the underlying MetricType
+        will be used to automatically assign a name.
+
+    Examples:
+      ```python
+      @metric
+      def mean() -> Metric:
+          def metric(scores: list[SampleScore]) -> Value:
+              return np.mean([score.score.as_float() for score in scores]).item()
+          return metric
+    ```
     """
 
     # create_metric_wrapper:

inspect_ai/scorer/_metrics/__init__.py

@@ -1,12 +1,12 @@
 from .accuracy import accuracy
-from .mean import mean, var
-from .std import bootstrap_stderr, std, stderr
+from .mean import mean
+from .std import bootstrap_stderr, std, stderr, var
 
 __all__ = [
     "accuracy",
     "mean",
-    "var",
     "bootstrap_stderr",
     "std",
     "stderr",
+    "var",
 ]

inspect_ai/scorer/_metrics/accuracy.py

@@ -2,7 +2,7 @@ from logging import getLogger
 
 from .._metric import (
     Metric,
-    Score,
+    SampleScore,
     ValueToFloat,
     metric,
     value_to_float,
@@ -16,22 +16,20 @@ def accuracy(to_float: ValueToFloat = value_to_float()) -> Metric:
     r"""Compute proportion of total answers which are correct.
 
     Args:
-      to_float (ValueToFloat): Function for mapping
-        Value to float for computing metrics. The default
-        `value_to_float()` maps CORRECT ("C") to 1.0,
-        INCORRECT ("I") to 0, PARTIAL ("P") to 0.5, and
-        NOANSWER ("N") to 0, casts numeric values to
-        float directly, and prints a warning and returns
-        0 if the Value is a complex object (list or dict).
+       to_float: Function for mapping `Value` to float for computing
+          metrics. The default `value_to_float()` maps CORRECT ("C") to 1.0,
+          INCORRECT ("I") to 0, PARTIAL ("P") to 0.5, and NOANSWER ("N") to 0,
+          casts numeric values to float directly, and prints a warning and returns
+          0 if the Value is a complex object (list or dict).
 
     Returns:
        Accuracy metric
     """
 
-    def metric(scores: list[Score]) -> float:
+    def metric(scores: list[SampleScore]) -> float:
         total = 0.0
         for item in scores:
-            total += to_float(item.value)
+            total += to_float(item.score.value)
         return total / float(len(scores))
 
     return metric

inspect_ai/scorer/_metrics/mean.py

@@ -1,6 +1,6 @@
 import numpy as np
 
-from .._metric import Metric, Score, metric
+from .._metric import Metric, SampleScore, metric
 
 
 @metric
@@ -11,21 +11,7 @@ def mean() -> Metric:
        mean metric
     """
 
-    def metric(scores: list[Score]) -> float:
-        return np.mean([score.as_float() for score in scores]).item()
-
-    return metric
-
-
-@metric
-def var() -> Metric:
-    """Compute variance over all scores.
-
-    Returns:
-       var metric
-    """
-
-    def metric(scores: list[Score]) -> float:
-        return np.var([score.as_float() for score in scores]).item()
+    def metric(scores: list[SampleScore]) -> float:
+        return np.mean([score.score.as_float() for score in scores]).item()
 
     return metric

inspect_ai/scorer/_metrics/std.py

@@ -5,7 +5,7 @@ import numpy as np
 
 from .._metric import (
     Metric,
-    Score,
+    SampleScore,
     ValueToFloat,
     metric,
     value_to_float,
@@ -21,21 +21,21 @@ def bootstrap_stderr(
     """Standard error of the mean using bootstrap.
 
     Args:
-       num_samples (int): Number of bootstrap samples to take.
-       to_float (ValueToFloat): Function for mapping
-         Value to float for computing metrics. The default
-         `value_to_float()` maps CORRECT ("C") to 1.0,
-         INCORRECT ("I") to 0, PARTIAL ("P") to 0.5, and
-         NOANSWER ("N") to 0, casts numeric values to
-         float directly, and prints a warning and returns
-         0 if the Value is a complex object (list or dict).
+       num_samples: Number of bootstrap samples to take.
+       to_float: Function for mapping
+          Value to float for computing metrics. The default
+          `value_to_float()` maps CORRECT ("C") to 1.0,
+          INCORRECT ("I") to 0, PARTIAL ("P") to 0.5, and
+          NOANSWER ("N") to 0, casts numeric values to
+          float directly, and prints a warning and returns
+          0 if the Value is a complex object (list or dict).
 
     Returns:
        bootstrap_stderr metric
     """
 
-    def metric(scores: list[Score]) -> float:
-        values = [to_float(score.value) for score in scores]
+    def metric(scores: list[SampleScore]) -> float:
+        values = [to_float(score.score.value) for score in scores]
         std = np.std(
             [
                 np.mean(np.random.choice(values, len(values), replace=True))
@@ -48,24 +48,71 @@ def bootstrap_stderr(
 
 
 @metric
-def stderr(to_float: ValueToFloat = value_to_float()) -> Metric:
+def stderr(
+    to_float: ValueToFloat = value_to_float(), cluster: str | None = None
+) -> Metric:
     """Standard error of the mean using Central Limit Theorem.
 
     Args:
-        to_float (ValueToFloat): Function for mapping
-            Value to float for computing metrics. The default
-            `value_to_float()` maps CORRECT ("C") to 1.0,
-            INCORRECT ("I") to 0, PARTIAL ("P") to 0.5, and
-            NOANSWER ("N") to 0, casts numeric values to
-            float directly, and prints a warning and returns
-            0 if the Value is a complex object (list or dict).
+       to_float: Function for mapping `Value` to float for computing
+          metrics. The default `value_to_float()` maps CORRECT ("C") to 1.0,
+          INCORRECT ("I") to 0, PARTIAL ("P") to 0.5, and NOANSWER ("N") to 0,
+          casts numeric values to float directly, and prints a warning and returns
+          0 if the Value is a complex object (list or dict).
+       cluster (str | None): The key from the Sample metadata
+          corresponding to a cluster identifier for computing
+          [clustered standard errors](https://en.wikipedia.org/wiki/Clustered_standard_errors).
 
     Returns:
-        stderr metric
+       stderr metric
     """
 
-    def metric(scores: list[Score]) -> float:
-        values = [to_float(score.value) for score in scores]
+    def clustered_metric(scores: list[SampleScore]) -> float:
+        """Computes a clustered standard error.
+
+        For details, see Appendix A of https://arxiv.org/pdf/2411.00640.
+        The version here uses a finite cluster correction (unlike the paper)
+        """
+        assert cluster is not None
+        cluster_list = []
+        value_list = []
+        for sample_score in scores:
+            if (
+                sample_score.sample_metadata is None
+                or cluster not in sample_score.sample_metadata
+            ):
+                raise ValueError(
+                    f"Sample {sample_score.sample_id} has no cluster metadata. To compute `stderr` with clustering, each sample metadata must have a value for '{cluster}'"
+                )
+            cluster_list.append(sample_score.sample_metadata[cluster])
+            value_list.append(to_float(sample_score.score.value))
+        clusters = np.array(cluster_list)
+        values = np.array(value_list)
+        mean = float(np.mean(values))
+
+        # Convert to numpy arrays and get unique clusters
+        unique_clusters = np.unique(clusters)
+        cluster_count = len(unique_clusters)
+
+        # Compute clustered variance using NumPy operations
+        clustered_variance = 0.0
+        for cluster_id in unique_clusters:
+            # get a data vector for this cluster
+            cluster_data = values[clusters == cluster_id]
+            # this computes X' \Omega X = \sum_i \sum_j (s_{i,c} - mean) * (s_{j,c} - mean)
+            clustered_variance += np.outer(
+                cluster_data - mean, cluster_data - mean
+            ).sum()
+
+        # Multiply by C / (C - 1) to unbias the variance estimate
+        standard_error = np.sqrt(
+            clustered_variance * cluster_count / (cluster_count - 1)
+        ) / len(scores)
+
+        return cast(float, standard_error)
+
+    def metric(scores: list[SampleScore]) -> float:
+        values = [to_float(score.score.value) for score in scores]
         n = len(values)
 
         # standard deviation is calculated by dividing by n-ddof so ensure
@@ -81,6 +128,9 @@ def stderr(to_float: ValueToFloat = value_to_float()) -> Metric:
 
         return cast(float, standard_error)
 
+    if cluster is not None:
+        return clustered_metric
+
     return metric
 
 
@@ -88,6 +138,39 @@ def stderr(to_float: ValueToFloat = value_to_float()) -> Metric:
 def std(to_float: ValueToFloat = value_to_float()) -> Metric:
     """Calculates the sample standard deviation of a list of scores.
 
+    Args:
+       to_float: Function for mapping `Value` to float for computing
+          metrics. The default `value_to_float()` maps CORRECT ("C") to 1.0,
+          INCORRECT ("I") to 0, PARTIAL ("P") to 0.5, and NOANSWER ("N") to 0,
+          casts numeric values to float directly, and prints a warning and returns
+          0 if the Value is a complex object (list or dict).
+
+
+    Returns:
+        std metric
+    """
+
+    def metric(scores: list[SampleScore]) -> float:
+        values = [to_float(score.score.value) for score in scores]
+        n = len(values)
+
+        # standard deviation is calculated by dividing by n-ddof so ensure
+        # that we won't divide by zero
+        if (n - 1) < 1:
+            return 0
+
+        # Calculate the sample standard deviation
+        sample_std = np.std(values, ddof=1)
+
+        return cast(float, sample_std)
+
+    return metric
+
+
+@metric
+def var(to_float: ValueToFloat = value_to_float()) -> Metric:
+    """Compute the sample variance of a list of scores.
+
     Args:
         to_float (ValueToFloat): Function for mapping
             Value to float for computing metrics. The default
@@ -98,21 +181,19 @@ def std(to_float: ValueToFloat = value_to_float()) -> Metric:
             0 if the Value is a complex object (list or dict).
 
     Returns:
-        std metric
+       var metric
     """
 
-    def metric(scores: list[Score]) -> float:
-        values = [to_float(score.value) for score in scores]
+    def metric(scores: list[SampleScore]) -> float:
+        values = [to_float(score.score.value) for score in scores]
         n = len(values)
-
-        # standard deviation is calculated by dividing by n-ddof so ensure
+        # variance is calculated by dividing by n-ddof so ensure
         # that we won't divide by zero
         if (n - 1) < 1:
             return 0
 
-        # Calculate the sample standard deviation
-        sample_std = np.std(values, ddof=1)
+        variance = np.var(values, ddof=1)
 
-        return cast(float, sample_std)
+        return cast(float, variance)
 
     return metric

inspect_ai/scorer/_model.py

@@ -35,31 +35,31 @@ def model_graded_fact(
     """Score a question/answer task with a fact response using a model.
 
     Args:
-      template (str): Template for grading prompt. This template uses
+      template: Template for grading prompt. This template uses
         four variables: `question`, `criterion`, `answer`, and
         `instructions` (which is fed from the `instructions` parameter).
         Variables from sample `metadata` are also available in the template.
-      instructions (str): Grading instructions. This should
+      instructions: Grading instructions. This should
         include a prompt for the model to answer (e.g. with
         with chain of thought reasoning) in a way that matches
         the specified `grade_pattern`, for example, the default
         `grade_pattern` looks for one of GRADE: C, GRADE: P, or
         GRADE: I).
-      grade_pattern (str): Regex to extract the grade from the
+      grade_pattern: Regex to extract the grade from the
         model response. Defaults to looking for e.g. GRADE: C
         The regex should have a single capture group that
         extracts exactly the letter C, P, or I.
-      include_history (bool | Callable[[TaskState], str]):
+      include_history:
         Whether to include the full chat history in the presented
         question. Defaults to `False`, which presents only the
         original sample input. Optionally provide a function to
         customise how the chat history is presented.
-      partial_credit (bool): Whether to allow for "partial" credit for
+      partial_credit: Whether to allow for "partial" credit for
          answers (by default assigned a score of 0.5). Defaults
          to `False`. Note that this parameter is only used
          with the default `instructions` (as custom instructions
          provide their own prompts for grades).
-      model (list[str | Model] | str | Model | None): Model or Models to use for grading. If multiple models are passed, a majority vote of their grade will be returned. By default the model being evaluated is used.
+      model: Model or Models to use for grading. If multiple models are passed, a majority vote of their grade will be returned. By default the model being evaluated is used.
     """
     return model_graded_qa(
         template=template if template else DEFAULT_MODEL_GRADED_FACT_TEMPLATE,
@@ -83,32 +83,32 @@ def model_graded_qa(
     """Score a question/answer task using a model.
 
     Args:
-      template (str): Template for grading prompt. This template has
+      template: Template for grading prompt. This template has
         four variables:
            - `question`, `criterion`, `answer`, and
         `instructions` (which is fed from the `instructions` parameter).
         Variables from sample `metadata` are also available in the template.
-      instructions (str): Grading instructions. This should
+      instructions: Grading instructions. This should
         include a prompt for the model to answer (e.g. with
         with chain of thought reasoning) in a way that matches
         the specified `grade_pattern`, for example, the default
         `grade_pattern` looks for one of GRADE: C, GRADE: P, or
         GRADE: I.
-      grade_pattern (str): Regex to extract the grade from the
+      grade_pattern: Regex to extract the grade from the
         model response. Defaults to looking for e.g. GRADE: C
         The regex should have a single capture group that
         extracts exactly the letter C, P, I.
-      include_history (bool | Callable[[TaskState], str]):
+      include_history:
         Whether to include the full chat history in the presented
         question. Defaults to `False`, which presents only the
         original sample input. Optionally provide a function to
         customise how the chat history is presented.
-      partial_credit (bool): Whether to allow for "partial" credit for
+      partial_credit: Whether to allow for "partial" credit for
         answers (by default assigned a score of 0.5). Defaults
         to `False`. Note that this parameter is only used
         with the default `instructions` (as custom instructions
         provide their own prompts for grades).
-      model (list[str | Model] | str | Model | None): Model or Models to use for grading. If     multiple models are passed, a majority vote of their grade will be returned. By default the model being evaluated is used.
+      model: Model or Models to use for grading. If     multiple models are passed, a majority vote of their grade will be returned. By default the model being evaluated is used.
     """
     # bind variables
     get_scorer = partial(

inspect_ai/scorer/_pattern.py

@@ -55,11 +55,11 @@ def pattern(pattern: str, ignore_case: bool = True, match_all: bool = False) ->
     to match either one or all of the extracted groups
 
     Args:
-       pattern (str): Regular expression for extracting the
+       pattern: Regular expression for extracting the
           answer from model output.
-       ignore_case (bool): Ignore case when comparing
+       ignore_case: Ignore case when comparing
           the extract answer to the targets. (Default: True)
-       match_all (bool): With multiple captures, do all captured
+       match_all: With multiple captures, do all captured
           values need to match the target? (Default: False)
     """
 

inspect_ai/scorer/_reducer/reducer.py

@@ -12,6 +12,8 @@ from .types import ScoreReducer
 
 @score_reducer(name="mode")
 def mode_score() -> ScoreReducer:
+    r"""Take the mode from a list of scores."""
+
     def reduce(scores: list[Score]) -> Score:
         r"""A utility function for the most common score in a list of scores.
 
@@ -36,12 +38,13 @@ def mode_score() -> ScoreReducer:
 
 @score_reducer(name="mean")
 def mean_score(value_to_float: ValueToFloat = value_to_float()) -> ScoreReducer:
-    def reduce(scores: list[Score]) -> Score:
-        r"""A utility function for taking a mean value over a list of scores.
+    r"""Take the mean of a list of scores.
 
-        Args:
-            scores: a list of Scores.
-        """
+    Args:
+       value_to_float: Function to convert the value to a float
+    """
+
+    def reduce(scores: list[Score]) -> Score:
         if isinstance(scores[0].value, dict):
             return _compute_dict_stat(scores, value_to_float, statistics.mean)
         elif isinstance(scores[0].value, list):
@@ -54,12 +57,13 @@ def mean_score(value_to_float: ValueToFloat = value_to_float()) -> ScoreReducer:
 
 @score_reducer(name="median")
 def median_score(value_to_float: ValueToFloat = value_to_float()) -> ScoreReducer:
-    def reduce(scores: list[Score]) -> Score:
-        r"""A utility function for taking a median value over a list of scores.
+    r"""Take the median value from a list of scores.
 
-        Args:
-            scores: a list of Scores.
-        """
+    Args:
+       value_to_float: Function to convert the value to a float
+    """
+
+    def reduce(scores: list[Score]) -> Score:
         if isinstance(scores[0].value, dict):
             return _compute_dict_stat(scores, value_to_float, statistics.median)
         elif isinstance(scores[0].value, list):
@@ -74,13 +78,15 @@ def median_score(value_to_float: ValueToFloat = value_to_float()) -> ScoreReduce
 def at_least(
     k: int, value: float = 1.0, value_to_float: ValueToFloat = value_to_float()
 ) -> ScoreReducer:
-    def reduce(scores: list[Score]) -> Score:
-        r"""A utility function for scoring a value as correct if there are at least n score values greater than or equal to the value
+    r"""Score correct if there are at least k score values greater than or equal to the value.
 
-        Args:
-            scores: a list of Scores.
-        """
+    Args:
+       k: Number of score values that must exceed `value`.
+       value: Score value threshold.
+       value_to_float: Function to convert score values to float.
+    """
 
+    def reduce(scores: list[Score]) -> Score:
         def gte_n(
             counter: Counter[str | int | float | bool],
         ) -> str | int | float | bool:
@@ -104,6 +110,14 @@ def at_least(
 def pass_at(
     k: int, value: float = 1.0, value_to_float: ValueToFloat = value_to_float()
 ) -> ScoreReducer:
+    r"""Probability of at least 1 correct sample given `k` epochs (<https://arxiv.org/pdf/2107.03374>).
+
+    Args:
+       k: Epochs to compute probability for.
+       value: Score value threshold.
+       value_to_float: Function to convert score values to float.
+    """
+
     def reduce(scores: list[Score]) -> Score:
         def pass_at_k(values: list[float]) -> float:
             total = len(scores)
@@ -129,12 +143,13 @@ def pass_at(
 
 @score_reducer(name="max")
 def max_score(value_to_float: ValueToFloat = value_to_float()) -> ScoreReducer:
-    def reduce(scores: list[Score]) -> Score:
-        r"""A utility function for taking the maximum value from a list of scores
+    r"""Take the maximum value from a list of scores.
 
-        Args:
-            scores: a list of Scores.
-        """
+    Args:
+       value_to_float: Function to convert the value to a float
+    """
+
+    def reduce(scores: list[Score]) -> Score:
         if isinstance(scores[0].value, dict):
             dict_result: dict[str, str | int | float | bool | None] = {}
             keys = scores[0].value.keys()  # type: ignore
@@ -238,7 +253,7 @@ def _compute_dict_stat(
 
     Args:
         scores: a list of Scores.
-        value_to_float: function to convert the value to a float
+        value_to_float: Function to convert the value to a float
         statistic: the statistic to apply
     """
     # Make sure these are all dictionaries be we proceed