inspect-ai 0.3.81__py3-none-any.whl → 0.3.83__py3-none-any.whl

inspect_ai/__init__.py

@@ -10,7 +10,8 @@ from inspect_ai._eval.score import score, score_async
 from inspect_ai._eval.task import Epochs, Task, TaskInfo, task_with
 from inspect_ai._eval.task.tasks import Tasks
 from inspect_ai._util.constants import PKG_NAME
-from inspect_ai.solver._human_agent.agent import human_agent
+from inspect_ai.agent._human.agent import human_cli
+from inspect_ai.solver._human_agent import human_agent
 
 __version__ = importlib_version(PKG_NAME)
 

inspect_ai/_cli/eval.py

@@ -10,6 +10,7 @@ from inspect_ai._util.constants import (
     ALL_LOG_LEVELS,
     DEFAULT_EPOCHS,
     DEFAULT_LOG_LEVEL_TRANSCRIPT,
+    DEFAULT_LOG_SHARED,
     DEFAULT_MAX_CONNECTIONS,
 )
 from inspect_ai._util.file import filesystem
@@ -25,7 +26,12 @@ from .common import (
     common_options,
     process_common_options,
 )
-from .util import parse_cli_args, parse_cli_config, parse_sandbox
+from .util import (
+    int_or_bool_flag_callback,
+    parse_cli_args,
+    parse_cli_config,
+    parse_sandbox,
+)
 
 MAX_SAMPLES_HELP = "Maximum number of samples to run in parallel (default is running all samples in parallel)"
 MAX_TASKS_HELP = "Maximum number of tasks to run in parallel (default is 1)"
@@ -41,6 +47,7 @@ LOG_IMAGES_HELP = (
     "Include base64 encoded versions of filename or URL based images in the log file."
 )
 LOG_BUFFER_HELP = "Number of samples to buffer before writing log file. If not specified, an appropriate default for the format and filesystem is chosen (10 for most all cases, 100 for JSON logs on remote filesystems)."
+LOG_SHARED_HELP = "Sync sample events to log directory so that users on other systems can see log updates in realtime (defaults to no syncing). If enabled will sync every 10 seconds (or pass a value to sync every `n` seconds)."
 NO_SCORE_HELP = (
     "Do not score model output (use the inspect score command to score output later)"
 )
@@ -266,6 +273,15 @@ def eval_options(func: Callable[..., Any]) -> Callable[..., click.Context]:
     @click.option(
         "--log-buffer", type=int, help=LOG_BUFFER_HELP, envvar="INSPECT_EVAL_LOG_BUFFER"
     )
+    @click.option(
+        "--log-shared",
+        is_flag=False,
+        flag_value="true",
+        default=None,
+        callback=int_or_bool_flag_callback(DEFAULT_LOG_SHARED),
+        help=LOG_SHARED_HELP,
+        envvar=["INSPECT_LOG_SHARED", "INSPECT_EVAL_LOG_SHARED"],
+    )
     @click.option(
         "--no-score",
         type=bool,
@@ -396,7 +412,7 @@ def eval_options(func: Callable[..., Any]) -> Callable[..., click.Context]:
     @click.option(
         "--reasoning-effort",
         type=click.Choice(["low", "medium", "high"]),
-        help="Constrains effort on reasoning for reasoning models. Open AI o-series models only.",
+        help="Constrains effort on reasoning for reasoning models (defaults to `medium`). Open AI o-series models only.",
         envvar="INSPECT_EVAL_REASONING_EFFORT",
     )
     @click.option(
@@ -503,6 +519,7 @@ def eval_command(
     no_log_samples: bool | None,
     log_images: bool | None,
     log_buffer: int | None,
+    log_shared: int | None,
     no_score: bool | None,
     no_score_display: bool | None,
     log_format: Literal["eval", "json"] | None,
@@ -556,6 +573,7 @@ def eval_command(
         no_log_samples=no_log_samples,
         log_images=log_images,
         log_buffer=log_buffer,
+        log_shared=log_shared,
         no_score=no_score,
         no_score_display=no_score_display,
         is_eval_set=False,
@@ -670,6 +688,7 @@ def eval_set_command(
     no_log_samples: bool | None,
     log_images: bool | None,
     log_buffer: int | None,
+    log_shared: int | None,
     no_score: bool | None,
     no_score_display: bool | None,
     bundle_dir: str | None,
@@ -728,6 +747,7 @@ def eval_set_command(
         no_log_samples=no_log_samples,
         log_images=log_images,
         log_buffer=log_buffer,
+        log_shared=log_shared,
         no_score=no_score,
         no_score_display=no_score_display,
         is_eval_set=True,
@@ -783,6 +803,7 @@ def eval_exec(
     no_log_samples: bool | None,
     log_images: bool | None,
     log_buffer: int | None,
+    log_shared: int | None,
     no_score: bool | None,
     no_score_display: bool | None,
     is_eval_set: bool = False,
@@ -865,6 +886,7 @@ def eval_exec(
             log_samples=log_samples,
             log_images=log_images,
             log_buffer=log_buffer,
+            log_shared=log_shared,
             score=score,
             score_display=score_display,
         )
@@ -1004,6 +1026,15 @@ def parse_comma_separated(value: str | None) -> list[str] | None:
 @click.option(
     "--log-buffer", type=int, help=LOG_BUFFER_HELP, envvar="INSPECT_EVAL_LOG_BUFFER"
 )
+@click.option(
+    "--log-shared",
+    is_flag=False,
+    flag_value="true",
+    default=None,
+    callback=int_or_bool_flag_callback(DEFAULT_LOG_SHARED),
+    help=LOG_SHARED_HELP,
+    envvar=["INSPECT_LOG_SHARED", "INSPECT_EVAL_LOG_SHARED"],
+)
 @click.option(
     "--no-score",
     type=bool,
@@ -1052,6 +1083,7 @@ def eval_retry_command(
     no_log_samples: bool | None,
     log_images: bool | None,
     log_buffer: int | None,
+    log_shared: int | None,
     no_score: bool | None,
     no_score_display: bool | None,
     max_connections: int | None,
@@ -1099,6 +1131,7 @@ def eval_retry_command(
         log_samples=log_samples,
         log_images=log_images,
         log_buffer=log_buffer,
+        log_shared=log_shared,
         score=score,
         score_display=score_display,
         max_retries=max_retries,

inspect_ai/_cli/util.py

@@ -1,11 +1,54 @@
-from typing import Any
+from typing import Any, Callable
 
+import click
 import yaml
 
 from inspect_ai._util.config import resolve_args
 from inspect_ai.util._sandbox.environment import SandboxEnvironmentSpec
 
 
+def int_or_bool_flag_callback(
+    true_value: int, false_value: int = 0
+) -> Callable[[click.Context, click.Parameter, Any], int]:
+    def callback(ctx: click.Context, param: click.Parameter, value: Any) -> int:
+        """Callback to parse the an option that can either be a boolean flag or integer.
+
+        Desired behavior:
+        - Not specified at all -> false_value
+        - Specified with no value -> true_value
+        - Specified with "true"/"false" -> true_value or false_value respectively
+        - Specified with an integer -> that integer
+        """
+        # 1. If this parameter was never given on the command line,
+        #    then we return 0.
+        source = ctx.get_parameter_source(param.name) if param.name else ""
+        if source == click.core.ParameterSource.DEFAULT:
+            # Means the user did NOT specify the flag at all
+            return false_value
+
+        # 2. The user did specify the flag. If value is None,
+        #    that means they used the flag with no argument, e.g. --my-flag
+        if value is None:
+            return true_value
+
+        # 3. If there is a value, try to parse booleans or an integer.
+        lower_val = value.lower()
+        if lower_val in ("true", "yes", "1"):
+            return true_value
+        elif lower_val in ("false", "no", "0"):
+            return false_value
+        else:
+            # 4. Otherwise, assume it is an integer
+            try:
+                return int(value)
+            except ValueError:
+                raise click.BadParameter(
+                    f"Expected 'true', 'false', or an integer for --{param.name}. Got: {value}"
+                )
+
+    return callback
+
+
 def parse_cli_config(
     args: tuple[str] | list[str] | None, config: str | None
 ) -> dict[str, Any]:

inspect_ai/_display/core/config.py

@@ -36,7 +36,7 @@ def task_config(
             value = value if isinstance(value, list) else [value]
             value = [str(v) for v in value]
             config_print.append(f"{name}: {','.join(value)}")
-        elif name not in ["limit", "model", "response_schema"]:
+        elif name not in ["limit", "model", "response_schema", "log_shared"]:
             if isinstance(value, list):
                 value = ",".join([str(v) for v in value])
             if isinstance(value, str):

inspect_ai/_display/core/display.py

@@ -15,6 +15,7 @@ from typing import (
 )
 
 import rich
+from pydantic import BaseModel, Field, field_validator
 from rich.console import Console
 
 from inspect_ai.log import EvalConfig, EvalResults, EvalStats
@@ -104,12 +105,20 @@ class TaskScreen(contextlib.AbstractContextManager["TaskScreen"]):
         raise NotImplementedError("input_panel not implemented by current display")
 
 
-@dataclass
-class TaskDisplayMetric:
+class TaskDisplayMetric(BaseModel):
     scorer: str
     name: str
-    value: float | int
-    reducer: str | None
+    value: float | int | None = Field(default=None)
+    reducer: str | None = Field(default=None)
+
+    @field_validator("value", mode="before")
+    @classmethod
+    def handle_null_value(cls, v: Any) -> Union[float, int, None]:
+        if v is None:
+            return None
+        if isinstance(v, float | int):
+            return v
+        raise ValueError(f"Expected float, int, or None, got {type(v)}")
 
 
 @runtime_checkable

inspect_ai/_display/core/results.py

@@ -180,7 +180,7 @@ def task_metric(metrics: list[TaskDisplayMetric], width: int | None = None) -> s
     )
 
     metric = metrics[0]
-    if np.isnan(metric.value):
+    if metric.value is None or np.isnan(metric.value):
         value = " n/a"
     else:
         value = f"{metric.value:.2f}"

inspect_ai/_display/textual/app.py

@@ -58,10 +58,12 @@ class TaskScreenResult(Generic[TR]):
         value: TR | BaseException,
         tasks: list[TaskWithResult],
         output: list[str],
+        warnings: list[str],
     ) -> None:
         self.value = value
         self.tasks = tasks
         self.output = output
+        self.warnings = warnings
 
 
 class TaskScreenApp(App[TR]):
@@ -86,6 +88,7 @@ class TaskScreenApp(App[TR]):
         self._worker: Worker[TR] | None = None
         self._error: BaseException | None = None
         self._output: list[str] = []
+        self._warnings: list[str] = []
 
         # task screen
         self._total_tasks = 0
@@ -120,7 +123,12 @@ class TaskScreenApp(App[TR]):
             value = CancelledError()
 
         # return result w/ output
-        return TaskScreenResult(value=value, tasks=self._app_tasks, output=self._output)
+        return TaskScreenResult(
+            value=value,
+            tasks=self._app_tasks,
+            output=self._output,
+            warnings=self._warnings,
+        )
 
     async def on_load(self) -> None:
         # events used to synchronise loading
@@ -349,8 +357,11 @@ class TaskScreenApp(App[TR]):
         if text.endswith("\n"):
             text = text[:-1]
 
-        # track output (for printing at the end)
-        self._output.append(text)
+        # track output and warnings (for printing at the end)
+        if "WARNING" in text:
+            self._warnings.append(text)
+        else:
+            self._output.append(text)
 
         # write to console view
         self.query_one(ConsoleView).write_ansi(text)

inspect_ai/_display/textual/display.py

@@ -42,6 +42,10 @@ class TextualDisplay(Display):
         # print tasks
         rich.print(tasks_results(result.tasks))
 
+        # print warnings
+        if result.warnings:
+            print("\n".join(result.warnings))
+
         # raise error as required
         if isinstance(result.value, BaseException):
             raise result.value

inspect_ai/_display/textual/widgets/samples.py

@@ -17,7 +17,7 @@ from textual.widgets import (
     OptionList,
     Static,
 )
-from textual.widgets.option_list import Option
+from textual.widgets.option_list import Option, OptionDoesNotExist
 
 from inspect_ai._display.textual.widgets.port_mappings import get_url
 from inspect_ai._util.format import format_progress_time
@@ -124,7 +124,7 @@ class SamplesList(OptionList):
     def set_samples(self, samples: list[ActiveSample]) -> None:
         # check for a highlighted sample (make sure we don't remove it)
         highlighted_id = (
-            self.get_option_at_index(self.highlighted).id
+            self.get_id_at_index(self.highlighted)
             if self.highlighted is not None
             else None
         )
@@ -179,12 +179,18 @@ class SamplesList(OptionList):
             self.scroll_to_highlight()
 
     def sample_for_highlighted(self, highlighted: int) -> ActiveSample | None:
-        highlighted_id = self.get_option_at_index(highlighted).id
+        highlighted_id = self.get_id_at_index(highlighted)
         if highlighted_id is not None:
             return sample_for_id(self.samples, highlighted_id)
         else:
             return None
 
+    def get_id_at_index(self, index: int) -> str | None:
+        try:
+            return self.get_option_at_index(index).id
+        except OptionDoesNotExist:
+            return None
+
 
 class SampleVNC(Horizontal):
     DEFAULT_CSS = """

inspect_ai/_display/textual/widgets/task_detail.py

@@ -14,7 +14,7 @@ from inspect_ai._display.core.display import TaskDisplayMetric
 @dataclass
 class TaskMetric:
     name: str
-    value: float
+    value: float | int | None
 
 
 class TaskDetail(Widget):
@@ -221,21 +221,21 @@ class TaskMetrics(Widget):
             self.recompute_grid()
 
     def on_mount(self) -> None:
-        self.recompute_grid()
+        self.recompute_grid(True)
 
-    def recompute_grid(self) -> None:
-        if not self.is_mounted:
+    def recompute_grid(self, force: bool = False) -> None:
+        if not self.is_mounted and not force:
             return
-
         grid = self.query_one(f"#{self.grid_id()}")
 
         grid.remove_children()
         for metric in self.metrics:
             # Add the value static but keep it around
             # for future updates
-            self.value_widgets[metric.name] = Static(
-                self._metric_value(metric.value), markup=False
-            )
+            if metric.value is not None:
+                self.value_widgets[metric.name] = Static(
+                    self._metric_value(metric.value), markup=False
+                )
 
             grid.mount(Static(metric.name, markup=False))
             grid.mount(self.value_widgets[metric.name])

inspect_ai/_display/textual/widgets/tasks.py

@@ -17,6 +17,11 @@ from inspect_ai._display.core.results import task_metric
 from inspect_ai._display.textual.widgets.clock import Clock
 from inspect_ai._display.textual.widgets.task_detail import TaskDetail
 from inspect_ai._display.textual.widgets.toggle import Toggle
+from inspect_ai._display.textual.widgets.vscode import conditional_vscode_link
+from inspect_ai._util.file import to_uri
+from inspect_ai._util.vscode import (
+    VSCodeCommand,
+)
 
 from ...core.display import (
     Progress,
@@ -151,7 +156,7 @@ class TaskProgressView(Widget):
         height: auto;
         width: 1fr;
         layout: grid;
-        grid-size: 8 2;
+        grid-size: 9 2;
         grid-columns: auto auto auto auto 1fr auto auto auto;
         grid-rows: auto auto;
         grid-gutter: 0 1;
@@ -200,6 +205,15 @@ class TaskProgressView(Widget):
 
         self.sample_count_width: int = sample_count_width
         self.display_metrics = display_metrics
+        self.view_log_link = conditional_vscode_link(
+            "[View Log]",
+            VSCodeCommand(
+                command="inspect.openLogViewer",
+                args=[to_uri(task.profile.log_location)]
+                if task.profile.log_location
+                else [],
+            ),
+        )
 
     metrics: reactive[list[TaskDisplayMetric] | None] = reactive(None)
     metrics_width: reactive[int | None] = reactive(None)
@@ -222,6 +236,8 @@ class TaskProgressView(Widget):
         yield self.count_display
         yield self.metrics_display
         yield Clock()
+        yield self.view_log_link
+
         yield self.task_detail
 
     @on(Toggle.Toggled)

inspect_ai/_display/textual/widgets/vscode.py

@@ -0,0 +1,44 @@
+from textual.widget import Widget
+from textual.widgets import Link, Static
+
+from inspect_ai._util.vscode import (
+    VSCodeCommand,
+    can_execute_vscode_command,
+    execute_vscode_commands,
+)
+
+
+def conditional_vscode_link(text: str, command: VSCodeCommand) -> Widget:
+    if can_execute_vscode_command(command.command):
+        vscode_link = VSCodeLink(text)
+        vscode_link.commands = [command]
+        return vscode_link
+    else:
+        return Static()
+
+
+class VSCodeLink(Link):
+    def __init__(
+        self,
+        text: str,
+        *,
+        url: str | None = None,
+        tooltip: str | None = None,
+        name: str | None = None,
+        id: str | None = None,
+        classes: str | None = None,
+        disabled: bool = False,
+    ) -> None:
+        super().__init__(
+            text,
+            url=url,
+            tooltip=tooltip,
+            name=name,
+            id=id,
+            classes=classes,
+            disabled=disabled,
+        )
+        self.commands: list[VSCodeCommand] = []
+
+    def on_click(self) -> None:
+        execute_vscode_commands(self.commands)