inspect-ai 0.3.57__py3-none-any.whl → 0.3.59__py3-none-any.whl

inspect_ai/_display/textual/widgets/samples.py

@@ -5,28 +5,28 @@ from rich.console import RenderableType
 from rich.table import Table
 from rich.text import Text
 from textual.app import ComposeResult
-from textual.containers import (
-    Horizontal,
-    HorizontalGroup,
-    Vertical,
-    VerticalGroup,
-)
+from textual.containers import Horizontal, HorizontalGroup, Vertical, VerticalGroup
 from textual.reactive import reactive
 from textual.widget import Widget
 from textual.widgets import (
     Button,
     Collapsible,
+    Link,
     LoadingIndicator,
     OptionList,
     Static,
 )
 from textual.widgets.option_list import Option, Separator
 
+from inspect_ai._display.textual.widgets.port_mappings import get_url
 from inspect_ai._util.format import format_progress_time
+from inspect_ai._util.port_names import get_service_by_port
 from inspect_ai._util.registry import registry_unqualified_name
 from inspect_ai.log._samples import ActiveSample
+from inspect_ai.log._transcript import ToolEvent
 
 from .clock import Clock
+from .sandbox import SandboxView
 from .transcript import TranscriptView
 
 
@@ -73,6 +73,7 @@ class SamplesView(Widget):
 
     async def set_highlighted_sample(self, highlighted: int | None) -> None:
         sample_info = self.query_one(SampleInfo)
+        sample_vnc = self.query_one(SampleVNC)
         transcript_view = self.query_one(TranscriptView)
         sample_toolbar = self.query_one(SampleToolbar)
         if highlighted is not None:
@@ -82,12 +83,14 @@ class SamplesView(Widget):
                 transcript_view.display = True
                 sample_toolbar.display = True
                 await sample_info.sync_sample(sample)
+                await sample_vnc.sync_sample(sample)
                 await transcript_view.sync_sample(sample)
                 await sample_toolbar.sync_sample(sample)
                 return
 
         # otherwise hide ui
         sample_info.display = False
+        sample_vnc.display = False
         transcript_view.display = False
         sample_toolbar.display = False
 
@@ -181,10 +184,59 @@ class SamplesList(OptionList):
             return None
 
 
-class SampleInfo(Horizontal):
+class SampleVNC(Horizontal):
+    DEFAULT_CSS = """
+    SampleVNC {
+        layout: grid;
+        grid-size: 2 1;
+        grid-columns: auto 1fr;
+    }
+    SampleVNC Static {
+        color: $secondary;
+    }
+    SampleVNC Link {
+        color: $accent;
+    }
+    """
+
+    def __init__(self) -> None:
+        super().__init__()
+        self._sample: ActiveSample | None = None
+
+    def compose(self) -> ComposeResult:
+        yield Static("VNC: ")
+        yield Link("")
+
+    async def sync_sample(self, sample: ActiveSample) -> None:
+        if sample == self._sample:
+            return
+
+        # defult to hidden (show if we find a vnc connection)
+        self.display = False
+
+        # is there a vnc connection? if so populate
+        for connection in [c for c in sample.sandboxes.values() if c.ports]:
+            for port in connection.ports or []:
+                service = get_service_by_port(port.container_port, port.protocol)
+                if service == "noVNC" and port.mappings:
+                    host_mappings = port.mappings
+                    link = self.query_one(Link)
+                    vnc_url = get_url(host_mappings[0].host_port, service)
+                    if vnc_url:
+                        link.text = vnc_url
+                        link.url = link.text
+                        self.display = True
+                        break
+
+
+class SampleInfo(Vertical):
     DEFAULT_CSS = """
     SampleInfo {
         color: $text-muted;
+        layout: grid;
+        grid-size: 1 2;
+        grid-rows: auto 1;
+        grid-gutter: 1;
     }
     SampleInfo Collapsible {
         padding: 0;
@@ -217,11 +269,13 @@ class SampleInfo(Horizontal):
     def __init__(self) -> None:
         super().__init__()
         self._sample: ActiveSample | None = None
+        self._sandbox_count: int | None = None
 
     def compose(self) -> ComposeResult:
         with Collapsible(title=""):
             yield SampleLimits()
             yield SandboxesView()
+        yield SampleVNC()
 
     async def sync_sample(self, sample: ActiveSample | None) -> None:
         if sample is None:
@@ -232,12 +286,14 @@ class SampleInfo(Horizontal):
             limits = self.query_one(SampleLimits)
             await limits.sync_sample(sample)
 
+            new_sandbox_count = len(sample.sandboxes)
             # bail if we've already processed this sample
-            if self._sample == sample:
+            if self._sample == sample and self._sandbox_count == new_sandbox_count:
                 return
 
             # set sample
             self._sample = sample
+            self._sandbox_count = new_sandbox_count
 
             # update UI
             self.display = True
@@ -245,6 +301,7 @@ class SampleInfo(Horizontal):
             self.query_one(Collapsible).title = title
             sandboxes = self.query_one(SandboxesView)
             await sandboxes.sync_sample(sample)
+            await self.query_one(SampleVNC).sync_sample(sample)
 
 
 class SampleLimits(Widget):
@@ -294,6 +351,9 @@ class SandboxesView(Vertical):
         background: transparent;
         height: auto;
     }
+    #sandboxes-list {
+        height: auto;
+    }
     SandboxesView Static {
         background: transparent;
     }
@@ -311,16 +371,24 @@ class SandboxesView(Vertical):
 
     async def sync_sample(self, sample: ActiveSample) -> None:
         if len(sample.sandboxes) > 0:
+            multiple_sandboxes = len(sample.sandboxes) > 1
             self.display = True
             sandboxes_caption = cast(Static, self.query_one("#sandboxes-caption"))
-            sandboxes_caption.update("[bold]sandbox containers:[/bold]")
+            sandboxes_caption.update(
+                f"[bold]sandbox container{'s' if multiple_sandboxes else ''}:[/bold]"
+            )
 
             sandboxes_list = self.query_one("#sandboxes-list")
             await sandboxes_list.remove_children()
+
             await sandboxes_list.mount_all(
-                [Static(sandbox.command) for sandbox in sample.sandboxes.values()]
+                [
+                    SandboxView(connection, name if multiple_sandboxes else None)
+                    for name, connection in sample.sandboxes.items()
+                ]
             )
-            sandboxes_list.mount(
+
+            await sandboxes_list.mount(
                 Static(
                     "[italic]Hold down Alt (or Option) to select text for copying[/italic]",
                     classes="clipboard-message",
@@ -332,16 +400,29 @@ class SandboxesView(Vertical):
 
 
 class SampleToolbar(Horizontal):
+    STATUS_GROUP = "status_group"
+    TIMEOUT_TOOL_CALL = "timeout_tool_call"
     CANCEL_SCORE_OUTPUT = "cancel_score_output"
     CANCEL_RAISE_ERROR = "cancel_raise_error"
     PENDING_STATUS = "pending_status"
     PENDING_CAPTION = "pending_caption"
 
     DEFAULT_CSS = f"""
+    SampleToolbar {{
+        grid-size: 5 1;
+        grid-columns: auto auto 1fr auto auto;
+    }}
+    SampleToolbar #{STATUS_GROUP} {{
+        min-width: 20;
+    }}
     SampleToolbar Button {{
         margin-bottom: 1;
         margin-right: 2;
-        min-width: 20;
+        min-width: 18;
+    }}
+    SampleToolbar #{TIMEOUT_TOOL_CALL} {{
+        color: $secondary-darken-3;
+        min-width: 16;
     }}
     SampleToolbar #{CANCEL_SCORE_OUTPUT} {{
         color: $primary-darken-3;
@@ -356,9 +437,16 @@ class SampleToolbar(Horizontal):
         self.sample: ActiveSample | None = None
 
     def compose(self) -> ComposeResult:
-        with VerticalGroup(id=self.PENDING_STATUS):
-            yield Static("Executing...", id=self.PENDING_CAPTION)
-            yield HorizontalGroup(EventLoadingIndicator(), Clock())
+        with HorizontalGroup(id=self.STATUS_GROUP):
+            with VerticalGroup(id=self.PENDING_STATUS):
+                yield Static("Executing...", id=self.PENDING_CAPTION)
+                yield HorizontalGroup(EventLoadingIndicator(), Clock())
+        yield Button(
+            Text("Timeout Tool"),
+            id=self.TIMEOUT_TOOL_CALL,
+            tooltip="Cancel the tool call and report a timeout to the model.",
+        )
+        yield Horizontal()
         yield Button(
             Text("Cancel (Score)"),
             id=self.CANCEL_SCORE_OUTPUT,
@@ -372,12 +460,21 @@ class SampleToolbar(Horizontal):
 
     def on_mount(self) -> None:
         self.query_one("#" + self.PENDING_STATUS).visible = False
+        self.query_one("#" + self.TIMEOUT_TOOL_CALL).display = False
         self.query_one("#" + self.CANCEL_SCORE_OUTPUT).display = False
         self.query_one("#" + self.CANCEL_RAISE_ERROR).display = False
 
     def on_button_pressed(self, event: Button.Pressed) -> None:
         if self.sample:
-            if event.button.id == self.CANCEL_SCORE_OUTPUT:
+            if event.button.id == self.TIMEOUT_TOOL_CALL:
+                last_event = (
+                    self.sample.transcript.events[-1]
+                    if self.sample.transcript.events
+                    else None
+                )
+                if isinstance(last_event, ToolEvent):
+                    last_event.cancel()
+            elif event.button.id == self.CANCEL_SCORE_OUTPUT:
                 self.sample.interrupt("score")
             elif event.button.id == self.CANCEL_RAISE_ERROR:
                 self.sample.interrupt("error")
@@ -389,6 +486,7 @@ class SampleToolbar(Horizontal):
         self.sample = sample
 
         pending_status = self.query_one("#" + self.PENDING_STATUS)
+        timeout_tool = self.query_one("#" + self.TIMEOUT_TOOL_CALL)
         clock = self.query_one(Clock)
         cancel_score_output = cast(
             Button, self.query_one("#" + self.CANCEL_SCORE_OUTPUT)
@@ -419,14 +517,19 @@ class SampleToolbar(Horizontal):
                 pending_caption.update(
                     Text.from_markup(f"[italic]{pending_caption_text}[/italic]")
                 )
+
+                timeout_tool.display = isinstance(last_event, ToolEvent)
+
                 clock.start(last_event.timestamp.timestamp())
             else:
                 pending_status.visible = False
+                timeout_tool.display = False
                 clock.stop()
 
         else:
             self.display = False
             pending_status.visible = False
+            timeout_tool.display = False
             clock.stop()
 
 

inspect_ai/_display/textual/widgets/sandbox.py

@@ -0,0 +1,37 @@
+from textual.app import ComposeResult
+from textual.containers import Horizontal, Vertical
+from textual.widgets import Static
+
+from inspect_ai.util._sandbox.environment import SandboxConnection
+
+from .port_mappings import PortMappingsView
+
+
+class SandboxView(Vertical):
+    DEFAULT_CSS = """
+    .indent {
+        width: 2;
+    }
+    .no_indent {
+        width: 0;
+    }
+    """
+
+    def __init__(
+        self,
+        connection: SandboxConnection,
+        name: str | None,  # if None, no header or indent
+    ) -> None:
+        super().__init__()
+        self.sandbox_name = name
+        self.connection = connection
+
+    def compose(self) -> ComposeResult:
+        if self.sandbox_name:
+            yield Static(self.sandbox_name)
+        with Horizontal():
+            yield Static("", classes="indent" if self.sandbox_name else "no_indent")
+            with Vertical():
+                yield Static(self.connection.command)
+                if self.connection.ports:
+                    yield PortMappingsView(self.connection.ports)

inspect_ai/_eval/eval.py

@@ -7,11 +7,12 @@ from shortuuid import uuid
 from typing_extensions import Unpack
 
 from inspect_ai._cli.util import parse_cli_args
-from inspect_ai._display.core.active import display
+from inspect_ai._display.core.active import display as task_display
 from inspect_ai._util.config import resolve_args
 from inspect_ai._util.constants import DEFAULT_LOG_FORMAT
 from inspect_ai._util.error import PrerequisiteError
 from inspect_ai._util.file import absolute_file_path
+from inspect_ai._util.logger import warn_once
 from inspect_ai._util.platform import platform_init
 from inspect_ai._util.registry import registry_lookup
 from inspect_ai.approval._apply import init_tool_approval
@@ -34,7 +35,7 @@ from inspect_ai.scorer._reducer import reducer_log_names
 from inspect_ai.solver._chain import chain
 from inspect_ai.solver._solver import Solver, SolverSpec
 from inspect_ai.util import SandboxEnvironmentType
-from inspect_ai.util._trace import init_trace
+from inspect_ai.util._display import DisplayType, display_type, init_display_type
 
 from .context import init_eval_context
 from .loader import ResolvedTask, resolve_tasks
@@ -55,6 +56,7 @@ def eval(
     solver: Solver | list[Solver] | SolverSpec | None = None,
     tags: list[str] | None = None,
     trace: bool | None = None,
+    display: DisplayType | None = None,
     approval: str | list[ApprovalPolicy] | None = None,
     log_level: str | None = None,
     log_level_transcript: str | None = None,
@@ -100,7 +102,8 @@ def eval(
         solver (Solver | list[Solver] | SolverSpec | None): Alternative solver for task(s).
           Optional (uses task solver by default).
         tags (list[str] | None): Tags to associate with this evaluation run.
-        trace: (bool | None): Trace message interactions with evaluated model to terminal.
+        trace (bool | None): Trace message interactions with evaluated model to terminal.
+        display (DisplayType | None): Task display type (defaults to 'full').
         approval: (str | list[ApprovalPolicy] | None): Tool use approval policies.
           Either a path to an approval policy config file or a list of approval policies.
           Defaults to no approval policy.
@@ -150,9 +153,11 @@ def eval(
     platform_init()
 
     # resolve eval trace
-    max_tasks, max_samples = init_eval_trace(trace, max_tasks, max_samples, model)
+    max_tasks, max_samples = init_eval_display(
+        display, trace, max_tasks, max_samples, model
+    )
 
-    return display().run_task_app(
+    return task_display().run_task_app(
         main=eval_async(
             tasks=tasks,
             model=model,
@@ -163,7 +168,6 @@ def eval(
             sandbox_cleanup=sandbox_cleanup,
             solver=solver,
             tags=tags,
-            trace=trace,
             approval=approval,
             log_level=log_level,
             log_level_transcript=log_level_transcript,
@@ -201,7 +205,6 @@ async def eval_async(
     sandbox_cleanup: bool | None = None,
     solver: Solver | list[Solver] | SolverSpec | None = None,
     tags: list[str] | None = None,
-    trace: bool | None = None,
     approval: str | list[ApprovalPolicy] | ApprovalPolicyConfig | None = None,
     log_level: str | None = None,
     log_level_transcript: str | None = None,
@@ -247,7 +250,6 @@ async def eval_async(
         solver (Solver | list[Solver] | SolverSpec | None): Alternative solver for task(s).
           Optional (uses task solver by default).
         tags (list[str] | None): Tags to associate with this evaluation run.
-        trace: (bool | None): Trace message interactions with evaluated model to terminal.
         approval: (str | list[ApprovalPolicy] | None): Tool use approval policies.
           Either a path to an approval policy config file or a list of approval policies.
           Defaults to no approval policy.
@@ -329,8 +331,8 @@ async def eval_async(
             log.warning("No inspect tasks were found at the specified paths.")
             return []
 
-        # apply trace mode constraints
-        if trace:
+        # apply conversation display constraints
+        if display_type() == "conversation":
             # single task at a time
             if max_tasks is not None:
                 max_tasks = 1
@@ -371,7 +373,6 @@ async def eval_async(
             epochs_reducer=reducer_log_names(epochs_reducer)
             if epochs_reducer
             else None,
-            trace=trace,
             approval=config_from_approval_policies(approval) if approval else None,
             fail_on_error=fail_on_error,
             message_limit=message_limit,
@@ -467,6 +468,7 @@ def eval_retry(
     max_sandboxes: int | None = None,
     sandbox_cleanup: bool | None = None,
     trace: bool | None = None,
+    display: DisplayType | None = None,
     fail_on_error: bool | float | None = None,
     debug_errors: bool | None = None,
     log_samples: bool | None = None,
@@ -501,6 +503,7 @@ def eval_retry(
         sandbox_cleanup (bool | None): Cleanup sandbox environments after task completes
            (defaults to True)
         trace (bool | None): Trace message interactions with evaluated model to terminal.
+        display (DisplayType | None): Task display type (defaults to 'full').
         fail_on_error (bool | float | None): `True` to fail on first sample error
            (default); `False` to never fail on sample errors; Value between 0 and 1
            to fail if a proportion of total samples fails. Value greater than 1 to fail
@@ -529,9 +532,9 @@ def eval_retry(
     platform_init()
 
     # resolve eval trace
-    max_tasks, max_samples = init_eval_trace(trace, max_tasks, max_samples)
+    max_tasks, max_samples = init_eval_display(display, trace, max_tasks, max_samples)
 
-    return display().run_task_app(
+    return task_display().run_task_app(
         main=eval_retry_async(
             tasks=tasks,
             log_level=log_level,
@@ -800,9 +803,8 @@ def eval_init(
 
     # resolve tasks (set active model to resolve uses of the
     # 'default' model in tools, solvers, and scorers)
-    from inspect_ai._display.core.active import display
 
-    with display().suspend_task_app():
+    with task_display().suspend_task_app():
         resolved_tasks: list[ResolvedTask] = []
         for m in models:
             init_active_model(m, generate_config)
@@ -816,17 +818,27 @@ def eval_init(
     return models, approval, resolved_tasks
 
 
-def init_eval_trace(
+def init_eval_display(
+    display: DisplayType | None,
     trace: bool | None,
     max_tasks: int | None,
     max_samples: int | None,
     model: Any = None,
 ) -> tuple[int | None, int | None]:
-    # init trace setting
-    init_trace(trace)
-
-    # adapt task/samples as required
+    # propagate any trace value to display_type
     if trace:
+        warn_once(
+            log,
+            "WARNING: The --trace flag is deprecated (use --display=conversation instead)",
+        )
+        display = "conversation"
+
+    # apply default and init
+    display = display or display_type()
+    init_display_type(display)
+
+    # adapt task/samples as required if we are in conversation mode
+    if display_type() == "conversation":
         # single task at a time
         if max_tasks is not None:
             max_tasks = 1

inspect_ai/_eval/evalset.py

@@ -33,7 +33,7 @@ from inspect_ai.model import (
 )
 from inspect_ai.model._generate_config import GenerateConfig
 from inspect_ai.solver._solver import Solver, SolverSpec
-from inspect_ai.util import SandboxEnvironmentType
+from inspect_ai.util import DisplayType, SandboxEnvironmentType
 
 from .eval import eval, eval_init
 from .loader import ResolvedTask, resolve_task_args
@@ -59,6 +59,7 @@ def eval_set(
     solver: Solver | list[Solver] | SolverSpec | None = None,
     tags: list[str] | None = None,
     trace: bool | None = None,
+    display: DisplayType | None = None,
     approval: str | list[ApprovalPolicy] | None = None,
     score: bool = True,
     log_level: str | None = None,
@@ -116,6 +117,7 @@ def eval_set(
            evaluating task(s). ptional (uses task solver by default).
         tags (list[str] | None): Tags to associate with this evaluation run.
         trace: (bool | None): Trace message interactions with evaluated model to terminal.
+        display (DisplayType | None): Task display type (defaults to 'full').
         approval: (str | list[ApprovalPolicy] | None): Tool use approval policies.
           Either a path to an approval policy config file or a list of approval policies.
           Defaults to no approval policy.
@@ -180,6 +182,7 @@ def eval_set(
             solver=solver,
             tags=tags,
             trace=trace,
+            display=display,
             approval=approval,
             log_level=log_level,
             log_level_transcript=log_level_transcript,
@@ -501,9 +504,6 @@ def latest_completed_task_eval_logs(
     # take the most recent completed log for each id
     latest_completed_logs: list[Log] = []
     for id, id_logs in logs_by_id.items():
-        # filter on completed
-        id_logs = [id_log for id_log in id_logs if id_log[1].status != "started"]
-
         # continue if there are no target logs
         if len(id_logs) == 0:
             continue
@@ -517,11 +517,13 @@ def latest_completed_task_eval_logs(
         latest_completed_logs.append(id_logs[0])
 
         # remove the rest if requested
+        # (don't remove 'started' in case its needed for post-mortum debugging)
         if cleanup_older:
             fs = filesystem(id_logs[0][0].name)
             for id_log in id_logs[1:]:
                 try:
-                    fs.rm(id_log[0].name)
+                    if id_log.header.status != "started":
+                        fs.rm(id_log.info.name)
                 except Exception as ex:
                     logger.warning(f"Error attempt to remove '{id_log[0].name}': {ex}")
 

inspect_ai/_eval/score.py

@@ -85,6 +85,7 @@ async def score_async(
             sample_id=sample.id,
             epoch=sample.epoch,
             input=sample.input,
+            target=Target(sample.target),
             choices=sample.choices,
             messages=sample.messages,
             output=sample.output,

inspect_ai/_eval/task/__init__.py

@@ -1,4 +1,4 @@
-from .task import Task, TaskInfo, PreviousTask, Tasks  # noqa: I001, F401
+from .task import Task, TaskInfo, PreviousTask, Tasks, task_with  # noqa: I001, F401
 from .epochs import Epochs
 
-__all__ = ["Epochs", "Task", "TaskInfo", "PreviousTask", "Tasks"]
+__all__ = ["Epochs", "Task", "TaskInfo", "PreviousTask", "Tasks", "task_with"]