inspect-ai 0.3.62__py3-none-any.whl → 0.3.64__py3-none-any.whl

inspect_ai/_eval/score.py

@@ -1,15 +1,16 @@
 import asyncio
 from copy import deepcopy
-from typing import Callable, cast
+from pathlib import Path
+from typing import Any, Callable, Literal, cast
 
 from inspect_ai._display import display
-from inspect_ai._util.path import chdir_python
+from inspect_ai._eval.loader import scorer_from_spec
 from inspect_ai._util.platform import platform_init
 from inspect_ai._util.registry import registry_create, registry_unqualified_name
 from inspect_ai.log import (
     EvalLog,
-    EvalMetric,
 )
+from inspect_ai.log._log import EvalMetricDefinition
 from inspect_ai.model import ModelName
 from inspect_ai.scorer import Metric, Scorer, Target
 from inspect_ai.scorer._metric import SampleScore
@@ -19,18 +20,19 @@ from inspect_ai.scorer._reducer import (
     create_reducers,
     reducer_log_names,
 )
-from inspect_ai.scorer._scorer import unique_scorer_name
+from inspect_ai.scorer._scorer import ScorerSpec, unique_scorer_name
 from inspect_ai.solver import TaskState
 
-from .task import Task
 from .task.results import eval_results
-from .task.util import task_run_dir
+
+ScoreAction = Literal["append", "overwrite"]
 
 
 def score(
     log: EvalLog,
     scorers: Scorer | list[Scorer],
     epochs_reducer: ScoreReducers | None = None,
+    action: ScoreAction | None = None,
 ) -> EvalLog:
     """Score an evaluation log.
 
@@ -40,6 +42,7 @@ def score(
        epochs_reducer (ScoreReducers | None):
            Reducer function(s) for aggregating scores in each sample.
            Defaults to previously used reducer(s).
+       action: Whether to append or overwrite this score
 
     Returns:
        Log with scores yielded by scorer.
@@ -50,13 +53,14 @@ def score(
     # resolve scorers into a list
     scorers = [scorers] if isinstance(scorers, Scorer) else scorers
 
-    return asyncio.run(score_async(log, scorers, epochs_reducer))
+    return asyncio.run(score_async(log, scorers, epochs_reducer, action))
 
 
 async def score_async(
     log: EvalLog,
     scorers: list[Scorer],
     epochs_reducer: ScoreReducers | None = None,
+    action: ScoreAction | None = None,
 ) -> EvalLog:
     """Score an evaluation log.
 
@@ -66,6 +70,8 @@ async def score_async(
        epochs_reducer (ScoreReducers  | None):
          Reducer function(s) for aggregating scores in each sample.
          Defaults to previously used reducer(s).
+       action: Whether to append or overwrite this score
+
 
 
     Returns:
@@ -109,7 +115,22 @@ async def score_async(
 
         # write them back (gather ensures that they come back in the same order)
         for index, score in enumerate(scores):
-            log.samples[index].scores = {k: v.score for k, v in score.items()}
+            if action == "overwrite":
+                log.samples[index].scores = {k: v.score for k, v in score.items()}
+            else:
+                existing_scores = log.samples[index].scores or {}
+                new_scores = {k: v.score for k, v in score.items()}
+
+                for key, value in new_scores.items():
+                    if key not in existing_scores:
+                        existing_scores[key] = value
+                    else:
+                        # This key already exists, dedupe its name
+                        count = 1
+                        while f"{key}-{count}" in existing_scores.keys():
+                            count = count + 1
+                        existing_scores[f"{key}-{count}"] = value
+                log.samples[index].scores = existing_scores
 
         # collect metrics from EvalLog (they may overlap w/ the scorer metrics,
         # that will be taken care of in eval_results)
@@ -130,30 +151,37 @@ async def score_async(
     return log
 
 
-async def task_score(task: Task, log: EvalLog) -> EvalLog:
-    with chdir_python(task_run_dir(task)):
-        # confirm we have a scorer
-        if task.scorer is None:
-            raise ValueError("You must specify a scorer for evals to be scored.")
+async def task_score(
+    log: EvalLog,
+    scorer: str | None = None,
+    scorer_args: dict[str, Any] | None = None,
+    action: ScoreAction | None = None,
+) -> EvalLog:
+    # confirm we have a scorer
+    scorers = resolve_scorers(log, scorer, scorer_args)
+    if len(scorers) == 0:
+        raise ValueError(
+            "Unable to resolve any scorers for this log. Please specify a scorer using the '--scorer' param."
+        )
 
-        # confirm we have samples
-        if log.samples is None or len(log.samples) == 0:
-            raise ValueError("There are no samples to score in the log.")
+    # confirm we have samples
+    if log.samples is None or len(log.samples) == 0:
+        raise ValueError("There are no samples to score in the log.")
 
-        task_name = task.name
-        display().print(f"Scoring {len(log.samples)} samples for task: {task_name}")
+    task_name = log.eval.task
+    display().print(f"\nScoring {task_name} ({len(log.samples)} samples)")
 
-        # perform scoring
-        log = await score_async(log, task.scorer)
+    # perform scoring
+    log = await score_async(log=log, scorers=scorers, action=action)
 
     # compute and log metrics
-    display().print(f"Aggregating scores for task: {task_name}")
-    if task.scorer and log.samples:
+    if log.samples:
         sample_scores = [
             {
                 score_key: SampleScore(
                     score=score,
                     sample_id=sample.id,
+                    sample_metadata=sample.metadata,
                 )
                 for score_key, score in sample.scores.items()
             }
@@ -161,12 +189,15 @@ async def task_score(task: Task, log: EvalLog) -> EvalLog:
             if sample.scores is not None
         ]
 
+        epochs_reducer = reducers_from_log(log)
+        metrics = metrics_from_log(log)
+
         log.results, log.reductions = eval_results(
             log.results.total_samples if log.results else 0,
             sample_scores,
-            task.epochs_reducer,
-            task.scorer,
-            task.metrics,
+            epochs_reducer,
+            scorers,
+            metrics,
         )
     return log
 
@@ -185,6 +216,7 @@ async def run_score_task(
         results[scorer_name] = SampleScore(
             score=result,
             sample_id=state.sample_id,
+            sample_metadata=state.metadata,
             scorer=registry_unqualified_name(scorer),
         )
 
@@ -192,21 +224,78 @@ async def run_score_task(
     return results
 
 
-def metrics_from_log(log: EvalLog) -> list[Metric]:
+def metrics_from_log(log: EvalLog) -> list[Metric] | dict[str, list[Metric]] | None:
+    # See if we have metrics in the eval itself
+    if log.eval.metrics:
+        if isinstance(log.eval.metrics, list):
+            return [metric_from_log(metric) for metric in log.eval.metrics]
+        else:
+            return {
+                key: [metric_from_log(metric) for metric in metrics]
+                for key, metrics in log.eval.metrics.items()
+            }
+    return None
+
+
+def metric_from_log(metric: EvalMetricDefinition) -> Metric:
+    return cast(
+        Metric, registry_create("metric", metric.name, **(metric.options or {}))
+    )
+
+
+def reducers_from_log(log: EvalLog) -> list[ScoreReducer] | None:
+    return create_reducers(log.eval.config.epochs_reducer)
+
+
+def resolve_scorers(
+    log: EvalLog, scorer: str | None, scorer_args: dict[str, Any] | None
+) -> list[Scorer]:
+    """
+    Create a list of Scorer objects from an evaluation log.
+
+    Args:
+        log: EvalLog object containing evaluation configuration and results
+        scorer:: Scorer name (simple name or file.py@name).
+        scorer_args: Dictionary of scorer arguments
+
+    Returns:
+        list[Scorer]: List of initialized scorers
+    """
+    # resolve the scorer path
+    task_path = Path(log.eval.task_file) if log.eval.task_file else None
+
+    # If there is an explicit scorer
+    if scorer:
+        return [
+            scorer_from_spec(
+                spec=ScorerSpec(scorer=scorer),
+                task_path=task_path,
+                **(scorer_args or {}),
+            )
+        ]
+    # See if we can create scorers from the eval itself
+    elif log.eval.scorers is not None:
+        return (
+            [
+                scorer_from_spec(
+                    spec=ScorerSpec(scorer=score.name),
+                    task_path=task_path,
+                    **(score.options or {}),
+                )
+                for score in log.eval.scorers
+            ]
+            if log.results
+            else []
+        )
+
+    # Otherwise, perhaps we can re-create them from the results
     return (
         [
-            metric_from_log(metric)
+            scorer_from_spec(
+                spec=ScorerSpec(scorer=score.name), task_path=task_path, **score.params
+            )
             for score in log.results.scores
-            for metric in score.metrics.values()
         ]
         if log.results
         else []
     )
-
-
-def metric_from_log(metric: EvalMetric) -> Metric:
-    return cast(Metric, registry_create("metric", metric.name, **metric.options))
-
-
-def reducers_from_log(log: EvalLog) -> list[ScoreReducer] | None:
-    return create_reducers(log.eval.config.epochs_reducer)

inspect_ai/_eval/task/log.py

@@ -4,9 +4,7 @@ from typing import Any, Literal, cast
 from shortuuid import uuid
 
 from inspect_ai._eval.task.util import slice_dataset
-from inspect_ai._util.constants import (
-    PKG_NAME,
-)
+from inspect_ai._util.constants import PKG_NAME
 from inspect_ai._util.datetime import iso_now
 from inspect_ai._util.git import git_context
 from inspect_ai._util.path import cwd_relative_path
@@ -27,7 +25,13 @@ from inspect_ai.log import (
     EvalSpec,
     EvalStats,
 )
-from inspect_ai.log._log import EvalLog, EvalSampleReductions
+from inspect_ai.log._log import (
+    EvalLog,
+    EvalMetricDefinition,
+    EvalSampleReductions,
+    EvalScorer,
+    eval_config_defaults,
+)
 from inspect_ai.log._recorders import Recorder
 from inspect_ai.model import (
     GenerateConfig,
@@ -35,6 +39,8 @@ from inspect_ai.model import (
     ModelName,
 )
 from inspect_ai.model._model import model_usage
+from inspect_ai.scorer._metric import MetricSpec
+from inspect_ai.scorer._scorer import ScorerSpec
 from inspect_ai.solver._plan import Plan
 from inspect_ai.solver._solver import Solver, SolverSpec
 from inspect_ai.util._sandbox.environment import SandboxEnvironmentSpec
@@ -52,6 +58,8 @@ class TaskLogger:
         tags: list[str] | None,
         model: Model,
         dataset: Dataset,
+        scorer: list[ScorerSpec] | None,
+        metrics: list[MetricSpec] | dict[str, list[MetricSpec]] | None,
         sandbox: SandboxEnvironmentSpec | None,
         task_attribs: dict[str, Any],
         task_args: dict[str, Any],
@@ -92,6 +100,17 @@ class TaskLogger:
             ],
         )
 
+        # write defaults for unspecified config
+        for name, value in eval_config_defaults().items():
+            if getattr(eval_config, name, None) is None:
+                setattr(eval_config, name, value)
+
+        # resolve scorers
+        eval_scorers = resolve_eval_scorers(scorer)
+
+        # resolve metrics
+        eval_metrics = resolve_eval_metrics(metrics)
+
         # create eval spec
         self.eval = EvalSpec(
             run_id=run_id,
@@ -114,6 +133,8 @@ class TaskLogger:
                 sample_ids=sample_ids,
                 shuffled=dataset.shuffled,
             ),
+            scorers=eval_scorers,
+            metrics=eval_metrics,
             sandbox=sandbox,
             model_args=model_args,
             config=eval_config,
@@ -200,3 +221,83 @@ def collect_eval_data(stats: EvalStats) -> None:
     # collect stats
     stats.completed_at = iso_now()
     stats.model_usage = model_usage()
+
+
+def resolve_eval_metrics(
+    metrics: list[MetricSpec] | dict[str, list[MetricSpec]] | None,
+) -> list[EvalMetricDefinition] | dict[str, list[EvalMetricDefinition]] | None:
+    if metrics is None:
+        return None
+    elif isinstance(metrics, list):
+        return [EvalMetricDefinition(name=m.metric, options=m.args) for m in metrics]
+    else:
+        return {
+            k: [
+                EvalMetricDefinition(name=v.metric, options=v.args) for v in metric_list
+            ]
+            for k, metric_list in metrics.items()
+        }
+
+
+def resolve_eval_scorers(scorers: list[ScorerSpec] | None) -> list[EvalScorer] | None:
+    if scorers is None:
+        return None
+    else:
+        results = []
+        for scorer in scorers:
+            results.append(
+                EvalScorer(
+                    name=scorer.scorer,
+                    metrics=resolve_scorer_metrics(scorer.metrics),
+                    options=scorer.args,
+                    metadata=scorer.metadata,
+                )
+            )
+        return results
+
+
+def resolve_scorer_metrics(
+    metrics: list[MetricSpec | dict[str, list[MetricSpec]]]
+    | dict[str, list[MetricSpec]]
+    | None,
+) -> (
+    list[EvalMetricDefinition | dict[str, list[EvalMetricDefinition]]]
+    | dict[str, list[EvalMetricDefinition]]
+    | None
+):
+    if metrics is None:
+        return None
+    elif isinstance(metrics, list):
+        resolved_metrics: list[
+            EvalMetricDefinition | dict[str, list[EvalMetricDefinition]]
+        ] = []
+        for metric_item in metrics:
+            if isinstance(metric_item, MetricSpec):
+                resolved_metrics.append(
+                    EvalMetricDefinition(
+                        name=metric_item.metric, options=metric_item.args
+                    )
+                )
+            elif isinstance(metric_item, dict):
+                resolved_metrics.append(
+                    {
+                        metric_group: [
+                            EvalMetricDefinition(
+                                name=metric_spec.metric, options=metric_spec.args
+                            )
+                            for metric_spec in metric_specs
+                        ]
+                        for metric_group, metric_specs in metric_item.items()
+                    }
+                )
+            else:
+                raise TypeError(f"Unexpected item in list: {metric_item}")
+        return resolved_metrics
+    else:
+        return {
+            metric_group: [
+                EvalMetricDefinition(name=metric_spec.metric, options=metric_spec.args)
+                for metric_spec in metric_specs
+            ]
+            for metric_group, metric_specs in metrics.items()
+        }

inspect_ai/_eval/task/results.py

@@ -1,10 +1,13 @@
 import fnmatch
+import inspect
+import logging
 import re
 from collections import defaultdict
 from copy import deepcopy
 from dataclasses import dataclass, field
-from typing import Any, Tuple, cast
+from typing import Any, Tuple, TypeGuard, cast, get_args, get_origin, get_type_hints
 
+from inspect_ai._util.logger import warn_once
 from inspect_ai._util.registry import (
     registry_info,
     registry_log_name,
@@ -19,7 +22,12 @@ from inspect_ai.log import (
 )
 from inspect_ai.log._log import EvalSampleReductions
 from inspect_ai.scorer import Metric, Score, Scorer
-from inspect_ai.scorer._metric import SampleScore
+from inspect_ai.scorer._metric import (
+    MetricDeprecated,
+    MetricProtocol,
+    SampleScore,
+    Value,
+)
 from inspect_ai.scorer._metrics.accuracy import accuracy
 from inspect_ai.scorer._metrics.std import stderr
 from inspect_ai.scorer._reducer import ScoreReducer, mean_score, reducer_log_name
@@ -29,6 +37,8 @@ from inspect_ai.scorer._scorer import (
     unique_scorer_name,
 )
 
+logger = logging.getLogger(__name__)
+
 
 @dataclass
 class ScorerInfo:
@@ -99,12 +109,14 @@ def eval_results(
                 reduced_samples = EvalSampleReductions(
                     scorer=scorer_name,
                     reducer=reducer_display_nm,
-                    samples=reduced_scores,
+                    samples=[
+                        EvalSampleScore(**ss.score.__dict__, sample_id=ss.sample_id)
+                        for ss in reduced_scores
+                    ],
                 )
                 sample_reductions.append(reduced_samples)
 
                 # Compute metrics for this scorer
-                simple_scores = cast(list[Score], reduced_scores)
                 targets = metrics if metrics is not None else scorer_info.metrics
                 if isinstance(targets, list):
                     ## split the metrics into the simple metrics and any dictionary
@@ -119,7 +131,7 @@ def eval_results(
                         scorer_for_metrics(
                             scorer_name=scorer_name,
                             scorer_info=scorer_info,
-                            scores=simple_scores,
+                            sample_scores=reduced_scores,
                             metrics=simple_metrics,
                             reducer_name=reducer_display_nm,
                         )
@@ -129,7 +141,7 @@ def eval_results(
                             scorers_from_metric_dict(
                                 scorer_name=scorer_name,
                                 scorer_info=scorer_info,
-                                scores=simple_scores,
+                                sample_scores=reduced_scores,
                                 metrics=dict_metric,
                                 reducer_name=reducer_display_nm,
                             )
@@ -145,7 +157,7 @@ def eval_results(
                         scorers_from_metric_dict(
                             scorer_name=scorer_name,
                             scorer_info=scorer_info,
-                            scores=simple_scores,
+                            sample_scores=reduced_scores,
                             metrics=targets,
                             reducer_name=reducer_display_nm,
                         )
@@ -184,7 +196,7 @@ def split_metrics(
 def scorer_for_metrics(
     scorer_name: str,
     scorer_info: ScorerInfo,
-    scores: list[Score],
+    sample_scores: list[SampleScore],
     metrics: list[Metric],
     reducer_name: str | None = None,
 ) -> list[EvalScore]:
@@ -200,10 +212,10 @@ def scorer_for_metrics(
         key = metrics_unique_key(
             registry_unqualified_name(metric), list(list_metrics.keys())
         )
-
+        params = registry_params(metric)
         # process metric values
-        if len(scores) > 0:
-            metric_value = metric(scores)
+        if len(sample_scores) > 0:
+            metric_value = call_metric(metric, sample_scores)
         else:
             metric_value = float("Nan")
         base_metric_name = registry_log_name(metric)
@@ -215,8 +227,7 @@ def scorer_for_metrics(
                 if value is not None:
                     name = metrics_unique_key(metric_key, list(list_metrics.keys()))
                     list_metrics[name] = EvalMetric(
-                        name=name,
-                        value=float(value),
+                        name=name, value=float(value), params=params
                     )
 
         # If the metric value is a list, turn each element in the list
@@ -229,13 +240,14 @@ def scorer_for_metrics(
                         with_suffix(key, count), list(list_metrics.keys())
                     )
 
-                    list_metrics[name] = EvalMetric(name=name, value=float(value))
+                    list_metrics[name] = EvalMetric(
+                        name=name, value=float(value), params=params
+                    )
 
         # the metric is a float, str, or int
         else:
             list_metrics[key] = EvalMetric(
-                name=base_metric_name,
-                value=float(metric_value),
+                name=base_metric_name, value=float(metric_value), params=params
             )
 
     # build results
@@ -257,7 +269,7 @@ def scorer_for_metrics(
 def scorers_from_metric_dict(
     scorer_name: str,
     scorer_info: ScorerInfo,
-    scores: list[Score],
+    sample_scores: list[SampleScore],
     metrics: dict[str, list[Metric]],
     reducer_name: str | None = None,
 ) -> list[EvalScore]:
@@ -265,18 +277,22 @@ def scorers_from_metric_dict(
 
     # Expand any metric keys
     resolved_metrics = (
-        resolve_glob_metric_keys(metrics, scores[0]) if len(scores) > 0 else metrics
+        resolve_glob_metric_keys(metrics, sample_scores[0].score)
+        if len(sample_scores) > 0
+        else metrics
     )
 
     for metric_key, metric_list in resolved_metrics.items():
         # filter scores to a list of scalars with the value of the metric name
-        metric_scores: list[Score] = []
-        for score in scores:
-            if isinstance(score.value, dict):
-                if metric_key in score.value:
+        metric_scores: list[SampleScore] = []
+        for sample_score in sample_scores:
+            if isinstance(sample_score.score.value, dict):
+                if metric_key in sample_score.score.value:
                     # Convert the score into a simple scalar value to apply metrics
-                    metric_score = deepcopy(score)
-                    metric_score.value = cast(float, score.value[metric_key])
+                    metric_score = deepcopy(sample_score)
+                    metric_score.score.value = cast(
+                        float, sample_score.score.value[metric_key]
+                    )
                     metric_scores.append(metric_score)
                 else:
                     raise TypeError(
@@ -291,8 +307,9 @@ def scorers_from_metric_dict(
         for target_metric in metric_list:
             # compute the metric value
             metric_name = registry_log_name(target_metric)
+            metric_params = registry_params(target_metric)
             if len(metric_scores) > 0:
-                value = target_metric(metric_scores)
+                value = call_metric(target_metric, metric_scores)
             else:
                 value = float("Nan")
 
@@ -302,20 +319,17 @@ def scorers_from_metric_dict(
                 for key, val in value.items():
                     name = f"{metric_name}_{key}"
                     result_metrics[name] = EvalMetric(
-                        name=name,
-                        value=cast(float, val),
+                        name=name, value=cast(float, val), params=metric_params
                     )
             elif isinstance(value, list):
                 for idx, item in enumerate(value):
                     name = f"{metric_name}_{idx}"
                     result_metrics[name] = EvalMetric(
-                        name=name,
-                        value=cast(float, item),
+                        name=name, value=cast(float, item), params=metric_params
                     )
             else:
                 result_metrics[metric_name] = EvalMetric(
-                    name=metric_name,
-                    value=cast(float, value),
+                    name=metric_name, value=cast(float, value), params=metric_params
                 )
 
         # create a scorer result for this metric
@@ -336,6 +350,48 @@ def scorers_from_metric_dict(
     return results
 
 
+def call_metric(metric: Metric, sample_scores: list[SampleScore]) -> Value:
+    if is_metric_deprecated(metric):
+        warn_once(
+            logger,
+            f"Metric {registry_log_name(metric)} should be updated to take list[SampleScore]. "
+            f"Metrics with list[Score] are deprecated.",
+        )
+        scores = [sample_score.score for sample_score in sample_scores]
+        return metric(scores)
+    else:
+        metric = cast(MetricProtocol, metric)
+        return metric(sample_scores)
+
+
+def is_metric_deprecated(metric: Metric) -> TypeGuard[MetricDeprecated]:
+    """Type guard to check if a metric follows the deprecated signature."""
+    try:
+        # signature and params
+        sig = inspect.signature(metric)
+        param_types = get_type_hints(metric)
+
+        # there should be only one param, check it
+        first_param = next(iter(sig.parameters.values()), None)
+        if first_param is None:
+            # No parameters, who knows what this is, treat it as deprecated
+            return True
+
+        expected_type: Any = param_types.get(first_param.name, None)
+
+        if expected_type is None or expected_type is Any:
+            # no helpful type info, treat it as deprecated
+            return True
+
+        # Extract generic base type and arguments to check if it matches list[Score]
+        origin = get_origin(expected_type)
+        args = get_args(expected_type)
+
+        return origin is list and args == (Score,)
+    except (AttributeError, ValueError, TypeError):
+        return False
+
+
 def resolve_glob_metric_keys(
     metrics: dict[str, list[Metric]], base_score: Score
 ) -> dict[str, list[Metric]]:
@@ -375,7 +431,7 @@ def resolve_glob_metric_keys(
 
 def reduce_scores(
     scores: list[SampleScore], reducer: ScoreReducer
-) -> list[EvalSampleScore]:
+) -> list[SampleScore]:
     # Group the scores by sample_id
     grouped_scores: dict[str, list[SampleScore]] = defaultdict(list)
     for sample_score in scores:
@@ -383,16 +439,14 @@ def reduce_scores(
             grouped_scores[str(sample_score.sample_id)].append(sample_score)
 
     # reduce the scores
-    reduced_scores: list[EvalSampleScore] = []
+    reduced_scores: list[SampleScore] = []
     for scores in grouped_scores.values():
         reduced = reducer([score.score for score in scores])
         reduced_scores.append(
-            EvalSampleScore(
+            SampleScore(
                 sample_id=scores[0].sample_id,
-                value=reduced.value,
-                answer=reduced.answer,
-                explanation=reduced.explanation,
-                metadata=reduced.metadata,
+                sample_metadata=scores[0].sample_metadata,
+                score=reduced,
             )
         )