inspect-ai 0.3.88__py3-none-any.whl → 0.3.90__py3-none-any.whl

inspect_ai/_eval/score.py

@@ -6,6 +6,7 @@ from typing import Any, Callable, Literal, cast
 import anyio
 
 from inspect_ai._display import display
+from inspect_ai._eval.context import init_task_context
 from inspect_ai._eval.loader import scorer_from_spec
 from inspect_ai._util._async import configured_async_backend, run_coroutine, tg_collect
 from inspect_ai._util.platform import platform_init, running_in_notebook
@@ -14,7 +15,9 @@ from inspect_ai.log import (
     EvalLog,
 )
 from inspect_ai.log._log import EvalMetricDefinition
+from inspect_ai.log._model import model_roles_config_to_model_roles
 from inspect_ai.model import ModelName
+from inspect_ai.model._model import get_model
 from inspect_ai.scorer import Metric, Scorer, Target
 from inspect_ai.scorer._metric import SampleScore
 from inspect_ai.scorer._reducer import (
@@ -122,7 +125,7 @@ async def score_async(
         scores: list[dict[str, SampleScore]] = await tg_collect(
             [
                 functools.partial(
-                    run_score_task, state, Target(sample.target), scorers, progress
+                    run_score_task, log, state, Target(sample.target), scorers, progress
                 )
                 for (sample, state) in zip(log.samples, states)
             ]
@@ -218,11 +221,25 @@ async def task_score(
 
 
 async def run_score_task(
+    log: EvalLog,
     state: TaskState,
     target: Target,
     scorers: list[Scorer],
     progress: Callable[..., None],
 ) -> dict[str, SampleScore]:
+    # get the model then initialize the async context
+    model = get_model(
+        model=log.eval.model,
+        config=log.plan.config.merge(log.eval.model_generate_config),
+        **log.eval.model_args,
+    )
+
+    # get the model roles
+    model_roles = model_roles_config_to_model_roles(log.eval.model_roles)
+
+    # initialize active model
+    init_task_context(model, model_roles)
+
     results: dict[str, SampleScore] = {}
     for scorer in scorers:
         result = await scorer(state, target)

inspect_ai/_eval/task/log.py

@@ -1,6 +1,5 @@
 from importlib import metadata as importlib_metadata
-from inspect import isgenerator
-from typing import Any, Iterator, Literal, cast
+from typing import Any, Literal, cast
 
 from shortuuid import uuid
 
@@ -34,6 +33,7 @@ from inspect_ai.log._log import (
     EvalScorer,
     eval_config_defaults,
 )
+from inspect_ai.log._model import model_args_for_log, model_roles_to_model_roles_config
 from inspect_ai.log._recorders import Recorder
 from inspect_ai.log._recorders.buffer import SampleBufferDatabase
 from inspect_ai.log._recorders.types import SampleEvent, SampleSummary
@@ -63,6 +63,7 @@ class TaskLogger:
         solver: SolverSpec | None,
         tags: list[str] | None,
         model: Model,
+        model_roles: dict[str, Model] | None,
         dataset: Dataset,
         scorer: list[ScorerSpec] | None,
         metrics: list[MetricSpec] | dict[str, list[MetricSpec]] | None,
@@ -84,17 +85,7 @@ class TaskLogger:
         packages = {PKG_NAME: importlib_metadata.version(PKG_NAME)}
 
         # redact authentication oriented model_args
-        model_args = model_args.copy()
-        if "api_key" in model_args:
-            del model_args["api_key"]
-        model_args = {k: v for k, v in model_args.items() if not k.startswith("aws_")}
-
-        # don't try to serialise generators
-        model_args = {
-            k: v
-            for k, v in model_args.items()
-            if not isgenerator(v) and not isinstance(v, Iterator)
-        }
+        model_args = model_args_for_log(model_args)
 
         # cwd_relative_path for sandbox config
         if sandbox and isinstance(sandbox.config, str):
@@ -141,6 +132,7 @@ class TaskLogger:
             model=str(ModelName(model)),
             model_generate_config=model.config,
             model_base_url=model.api.base_url,
+            model_roles=model_roles_to_model_roles_config(model_roles),
             dataset=EvalDataset(
                 name=dataset.name,
                 location=cwd_relative_path(dataset.location),

inspect_ai/_eval/task/resolved.py

@@ -13,6 +13,7 @@ class ResolvedTask:
     task_args: dict[str, Any]
     task_file: str | None
     model: Model
+    model_roles: dict[str, Model] | None
     sandbox: SandboxEnvironmentSpec | None
     sequence: int
     id: str | None = field(default=None)