inspect-ai 0.3.75__py3-none-any.whl → 0.3.77__py3-none-any.whl

inspect_ai/_cli/eval.py

@@ -115,6 +115,13 @@ def eval_options(func: Callable[..., Any]) -> Callable[..., click.Context]:
         help="Tags to associate with this evaluation run.",
         envvar="INSPECT_EVAL_TAGS",
     )
+    @click.option(
+        "--metadata",
+        multiple=True,
+        type=str,
+        help="Metadata to associate with this evaluation run (more than one --metadata argument can be specified).",
+        envvar="INSPECT_EVAL_METADATA",
+    )
     @click.option(
         "--trace",
         type=bool,
@@ -449,6 +456,7 @@ def eval_command(
     s: tuple[str] | None,
     solver_config: str | None,
     tags: str | None,
+    metadata: tuple[str] | None,
     trace: bool | None,
     approval: str | None,
     sandbox: str | None,
@@ -525,6 +533,7 @@ def eval_command(
         s=s,
         solver_config=solver_config,
         tags=tags,
+        metadata=metadata,
         trace=trace,
         approval=approval,
         sandbox=sandbox,
@@ -616,6 +625,7 @@ def eval_set_command(
     s: tuple[str] | None,
     solver_config: str | None,
     tags: str | None,
+    metadata: tuple[str] | None,
     sandbox: str | None,
     no_sandbox_cleanup: bool | None,
     epochs: int | None,
@@ -695,6 +705,7 @@ def eval_set_command(
         s=s,
         solver_config=solver_config,
         tags=tags,
+        metadata=metadata,
         trace=trace,
         approval=approval,
         sandbox=sandbox,
@@ -749,6 +760,7 @@ def eval_exec(
     s: tuple[str] | None,
     solver_config: str | None,
     tags: str | None,
+    metadata: tuple[str] | None,
     trace: bool | None,
     approval: str | None,
     sandbox: str | None,
@@ -790,6 +802,9 @@ def eval_exec(
     # parse tags
     eval_tags = parse_comma_separated(tags)
 
+    # parse metadata
+    eval_metadata = parse_cli_args(metadata)
+
     # resolve epochs
     eval_epochs = (
         Epochs(epochs, create_reducers(parse_comma_separated(epochs_reducer)))
@@ -825,6 +840,7 @@ def eval_exec(
             task_args=task_args,
             solver=SolverSpec(solver, solver_args) if solver else None,
             tags=eval_tags,
+            metadata=eval_metadata,
             trace=trace,
             approval=approval,
             sandbox=parse_sandbox(sandbox),

inspect_ai/_display/core/results.py

@@ -131,9 +131,14 @@ def task_stats(stats: EvalStats) -> RenderableType:
         else:
             input_tokens = f"[bold]I: [/bold]{usage.input_tokens:,}"
 
+        if usage.reasoning_tokens is not None:
+            reasoning_tokens = f", [bold]R: [/bold]{usage.reasoning_tokens:,}"
+        else:
+            reasoning_tokens = ""
+
         table.add_row(
             Text(model, style="bold"),
-            f"  {usage.total_tokens:,} tokens [{input_tokens}, [bold]O: [/bold]{usage.output_tokens:,}]",
+            f"  {usage.total_tokens:,} tokens [{input_tokens}, [bold]O: [/bold]{usage.output_tokens:,}{reasoning_tokens}]",
             style=theme.light,
         )
 

inspect_ai/_eval/eval.py

@@ -68,6 +68,7 @@ def eval(
     sandbox_cleanup: bool | None = None,
     solver: Solver | list[Solver] | SolverSpec | None = None,
     tags: list[str] | None = None,
+    metadata: dict[str, Any] | None = None,
     trace: bool | None = None,
     display: DisplayType | None = None,
     approval: str | list[ApprovalPolicy] | None = None,
@@ -116,6 +117,7 @@ def eval(
         solver: Alternative solver for task(s).
             Optional (uses task solver by default).
         tags: Tags to associate with this evaluation run.
+        metadata: Metadata to associate with this evaluation run.
         trace: Trace message interactions with evaluated model to terminal.
         display: Task display type (defaults to 'full').
         approval: Tool use approval policies.
@@ -186,6 +188,7 @@ def eval(
                 sandbox_cleanup=sandbox_cleanup,
                 solver=solver,
                 tags=tags,
+                metadata=metadata,
                 approval=approval,
                 log_level=log_level,
                 log_level_transcript=log_level_transcript,
@@ -235,6 +238,7 @@ async def eval_async(
     sandbox_cleanup: bool | None = None,
     solver: Solver | list[Solver] | SolverSpec | None = None,
     tags: list[str] | None = None,
+    metadata: dict[str, Any] | None = None,
     approval: str | list[ApprovalPolicy] | ApprovalPolicyConfig | None = None,
     log_level: str | None = None,
     log_level_transcript: str | None = None,
@@ -274,7 +278,8 @@ async def eval_async(
         sandbox: Sandbox environment type (or optionally a str or tuple with a shorthand spec)
         sandbox_cleanup: Cleanup sandbox environments after task completes (defaults to True)
         solver: Alternative solver for task(s).  Optional (uses task solver by default).
-        tags (list[str] | None): Tags to associate with this evaluation run.
+        tags: Tags to associate with this evaluation run.
+        metadata: Metadata to associate with this evaluation run.
         approval: Tool use approval policies.
           Either a path to an approval policy config file or a list of approval policies.
           Defaults to no approval policy.
@@ -449,6 +454,7 @@ async def eval_async(
                         epochs_reducer=epochs_reducer,
                         solver=solver,
                         tags=tags,
+                        metadata=metadata,
                         score=score,
                         debug_errors=debug_errors is True,
                         **kwargs,
@@ -473,6 +479,7 @@ async def eval_async(
                 epochs_reducer=epochs_reducer,
                 solver=solver,
                 tags=tags,
+                metadata=metadata,
                 score=score,
                 **kwargs,
             )

inspect_ai/_eval/evalset.py

@@ -35,7 +35,7 @@ from inspect_ai.model import (
 from inspect_ai.model._generate_config import GenerateConfig
 from inspect_ai.solver._solver import Solver, SolverSpec
 from inspect_ai.util import DisplayType, SandboxEnvironmentType
-from inspect_ai.util._display import init_display_type
+from inspect_ai.util._display import display_type_initialized, init_display_type
 
 from .eval import eval, eval_init
 from .loader import resolve_task_args
@@ -68,6 +68,7 @@ def eval_set(
     sandbox_cleanup: bool | None = None,
     solver: Solver | list[Solver] | SolverSpec | None = None,
     tags: list[str] | None = None,
+    metadata: dict[str, Any] | None = None,
     trace: bool | None = None,
     display: DisplayType | None = None,
     approval: str | list[ApprovalPolicy] | None = None,
@@ -127,6 +128,7 @@ def eval_set(
         solver: Alternative solver(s) for
             evaluating task(s). ptional (uses task solver by default).
         tags: Tags to associate with this evaluation run.
+        metadata: Metadata to associate with this evaluation run.
         trace: Trace message interactions with evaluated model to terminal.
         display: Task display type (defaults to 'full').
         approval: Tool use approval policies.
@@ -193,6 +195,7 @@ def eval_set(
             sandbox_cleanup=sandbox_cleanup,
             solver=solver,
             tags=tags,
+            metadata=metadata,
             trace=trace,
             display=display,
             approval=approval,
@@ -234,7 +237,8 @@ def eval_set(
         return results
 
     # initialise display (otherwise eval_init will set it to full)
-    display = init_display_type(display)
+    if not display_type_initialized():
+        display = init_display_type(display)
     if display == "conversation":
         raise RuntimeError("eval_set cannot be used with conversation display.")
 

inspect_ai/_eval/registry.py

@@ -75,12 +75,10 @@ def task_create(name: str, **kwargs: Any) -> Task:
     task_params: list[str] = task_info.metadata["params"]
     task_args: dict[str, Any] = {}
     for param in kwargs.keys():
-        if param in task_params:
+        if param in task_params or "kwargs" in task_params:
             task_args[param] = kwargs[param]
-            if "kwargs" in task_params:
-                task_args[param] = kwargs[param]
-            else:
-                logger.warning(f"param '{param}' not used by task '{name}'")
+        else:
+            logger.warning(f"param '{param}' not used by task '{name}'")
 
     return cast(Task, registry_create("task", name, **task_args))
 

inspect_ai/_eval/run.py

@@ -2,8 +2,9 @@ import functools
 import logging
 import os
 import sys
-from typing import Awaitable, Callable, Set, cast
+from typing import Any, Awaitable, Callable, Set, cast
 
+from inspect_ai._eval.task.task import Task
 from inspect_ai._util.trace import trace_action
 
 if sys.version_info < (3, 11):
@@ -67,6 +68,7 @@ async def eval_run(
     epochs_reducer: list[ScoreReducer] | None = None,
     solver: Solver | SolverSpec | None = None,
     tags: list[str] | None = None,
+    metadata: dict[str, Any] | None = None,
     debug_errors: bool = False,
     score: bool = True,
     **kwargs: Unpack[GenerateConfigArgs],
@@ -81,6 +83,7 @@ async def eval_run(
     eval_wd = os.getcwd()
 
     # ensure sample ids
+    task: Task | None = None
     for resolved_task in tasks:
         # add sample ids to dataset if they aren't there (start at 1 not 0)
         task = resolved_task.task
@@ -91,6 +94,8 @@ async def eval_run(
         # Ensure sample ids are unique
         ensure_unique_ids(task.dataset)
 
+    assert task, "Must encounter a task"
+
     # run startup pass for the sandbox environments
     shutdown_sandbox_environments: Callable[[], Awaitable[None]] | None = None
     if has_sandbox:
@@ -201,7 +206,7 @@ async def eval_run(
                     task_args=resolved_task.task_args,
                     model_args=resolved_task.model.model_args,
                     eval_config=task_eval_config,
-                    metadata=task.metadata,
+                    metadata=((metadata or {}) | (task.metadata or {})) or None,
                     recorder=recorder,
                 )
                 await logger.init()

inspect_ai/_eval/task/run.py

@@ -599,6 +599,10 @@ async def task_run_sample(
             )
 
             async with sandboxenv_cm:
+                timeout_cm: (
+                    contextlib._GeneratorContextManager[anyio.CancelScope, None, None]
+                    | contextlib.nullcontext[None]
+                ) = contextlib.nullcontext()
                 try:
                     # update active sample wth sandboxes now that we are initialised
                     active.sandboxes = await sandbox_connections()

inspect_ai/_util/content.py

@@ -12,6 +12,9 @@ class ContentText(BaseModel):
     text: str
     """Text content."""
 
+    refusal: bool | None = Field(default=None)
+    """Was this a refusal message?"""
+
 
 class ContentReasoning(BaseModel):
     """Reasoning content.

inspect_ai/_util/logger.py

@@ -150,6 +150,9 @@ def init_logger(log_level: str | None, log_level_transcript: str | None = None)
             transcript_levelno=transcript_levelno,
         )
 
+        # set the global log level
+        getLogger().setLevel(log_level)
+
         # set the log level for our package
         getLogger(PKG_NAME).setLevel(capture_level)
         getLogger(PKG_NAME).addHandler(_logHandler)

inspect_ai/_view/www/dist/assets/index.css

@@ -16461,44 +16461,44 @@ ul.jsondiffpatch-textdiff {
   font-weight: 600;
   padding-bottom: 0.3em;
 }
-._output_3axgd_1 {
+._output_15urk_1 {
   padding-top: 1em;
 }
 
-._container_3axgd_5 {
+._container_15urk_5 {
   margin: 0.5em 0;
   width: 100%;
 }
 
-._all_3axgd_10 {
+._all_15urk_10 {
   display: grid;
   grid-template-columns: 1fr 1fr 1fr;
   column-gap: 1em;
 }
 
-._tableSelection_3axgd_16 {
+._tableSelection_15urk_16 {
   width: fit-content;
   align-self: start;
   justify-self: start;
 }
 
-._tools_3axgd_22 {
+._tools_15urk_22 {
   grid-column: -1/1;
 }
 
-._codePre_3axgd_26 {
+._codePre_15urk_26 {
   background: var(--bs-light);
   width: 100%;
   padding: 0.5em;
   border-radius: var(--bs-border-radius);
 }
 
-._code_3axgd_26 {
-  white-space: pre-wrap;
-  word-wrap: anywhere;
+._code_15urk_26 {
+  white-space: pre-wrap !important;
+  word-wrap: anywhere !important;
 }
 
-._toolConfig_3axgd_38 {
+._toolConfig_15urk_38 {
   display: grid;
   grid-template-columns: max-content auto;
   column-gap: 1em;
@@ -17032,12 +17032,14 @@ div.ap-player div.ap-control-bar * {
 div.ap-control-bar svg.ap-icon path {
   fill: var(--term-color-foreground);
 }
-div.ap-control-bar span.ap-playback-button {
+div.ap-control-bar span.ap-button {
   display: flex;
   flex: 0 0 auto;
   cursor: pointer;
-  height: 12px;
+}
+div.ap-control-bar span.ap-playback-button {
   width: 12px;
+  height: 12px;
   padding: 10px;
 }
 div.ap-control-bar span.ap-playback-button svg {
@@ -17104,13 +17106,9 @@ div.ap-control-bar.ap-seekable .ap-progressbar .ap-bar {
   cursor: pointer;
 }
 div.ap-control-bar .ap-fullscreen-button {
-  display: block;
-  flex: 0 0 auto;
   width: 14px;
   height: 14px;
   padding: 9px;
-  cursor: pointer;
-  position: relative;
 }
 div.ap-control-bar .ap-fullscreen-button svg {
   width: 14px;
@@ -17127,6 +17125,20 @@ div.ap-control-bar .ap-fullscreen-button .ap-tooltip {
   left: initial;
   transform: none;
 }
+div.ap-control-bar .ap-kbd-button {
+  height: 14px;
+  padding: 9px;
+  margin: 0 4px;
+}
+div.ap-control-bar .ap-kbd-button svg {
+  width: 26px;
+  height: 14px;
+}
+div.ap-control-bar .ap-kbd-button .ap-tooltip {
+  right: 5px;
+  left: initial;
+  transform: none;
+}
 div.ap-wrapper.ap-hud .ap-control-bar {
   opacity: 1;
 }