inspect-ai 0.3.99__py3-none-any.whl → 0.3.100__py3-none-any.whl

inspect_ai/_display/core/config.py

@@ -1,4 +1,8 @@
+from rich.console import RenderableType
+from rich.text import Text
+
 from inspect_ai._util.registry import is_model_dict, is_registry_dict
+from inspect_ai._util.text import truncate_text
 from inspect_ai.log._log import eval_config_defaults
 
 from .display import TaskProfile
@@ -6,7 +10,7 @@ from .display import TaskProfile
 
 def task_config(
     profile: TaskProfile, generate_config: bool = True, style: str = ""
-) -> str:
+) -> RenderableType:
     # merge config
     # wind params back for display
     task_args = dict(profile.task_args)
@@ -39,15 +43,17 @@ def task_config(
         elif name not in ["limit", "model", "response_schema", "log_shared"]:
             if isinstance(value, list):
                 value = ",".join([str(v) for v in value])
+            elif isinstance(value, dict):
+                value = "{...}"
             if isinstance(value, str):
+                value = truncate_text(value, 50)
                 value = value.replace("[", "\\[")
             config_print.append(f"{name}: {value}")
     values = ", ".join(config_print)
     if values:
-        if style:
-            return f"[{style}]{values}[/{style}]"
-        else:
-            return values
+        values_text = Text(values, style=style)
+        values_text.truncate(500, overflow="ellipsis")
+        return values_text
     else:
         return ""
 

inspect_ai/_display/core/panel.py

@@ -9,6 +9,7 @@ from rich.text import Text
 from inspect_ai._util.constants import CONSOLE_DISPLAY_WIDTH
 from inspect_ai._util.path import cwd_relative_path
 from inspect_ai._util.registry import registry_unqualified_name
+from inspect_ai.util._display import display_type
 
 from .display import TaskProfile
 from .rich import is_vscode_notebook, rich_theme
@@ -24,7 +25,13 @@ def task_panel(
     | None,
     footer: RenderableType | tuple[RenderableType, RenderableType] | None,
     log_location: str | None,
-) -> Panel:
+) -> RenderableType:
+    # dispatch to plain handler if we are in plain mode
+    if display_type() == "plain":
+        return task_panel_plain(
+            profile, show_model, body, subtitle, footer, log_location
+        )
+
     # rendering context
     theme = rich_theme()
     console = rich.get_console()
@@ -93,7 +100,7 @@ def task_panel(
     # create panel w/ title
     panel = Panel(
         root,
-        title=f"[bold][{theme.meta}]{task_title(profile, show_model)}[/{theme.meta}][/bold]",
+        title=task_panel_title(profile, show_model),
         title_align="left",
         width=width,
         expand=True,
@@ -101,6 +108,63 @@ def task_panel(
     return panel
 
 
+def task_panel_plain(
+    profile: TaskProfile,
+    show_model: bool,
+    body: RenderableType,
+    subtitle: RenderableType
+    | str
+    | Tuple[RenderableType | str, RenderableType | str]
+    | None,
+    footer: RenderableType | tuple[RenderableType, RenderableType] | None,
+    log_location: str | None,
+) -> RenderableType:
+    # delimiter text
+    delimeter = "---------------------------------------------------------"
+
+    # root table for output
+    table = Table.grid(expand=False)
+    table.add_column()
+    table.add_row(delimeter)
+
+    # title and subtitle
+    table.add_row(task_panel_title(profile, show_model))
+    if isinstance(subtitle, tuple):
+        subtitle = subtitle[0]
+    table.add_row(subtitle)
+
+    # task info
+    if body:
+        table.add_row(body)
+
+    # footer
+    if isinstance(footer, tuple):
+        footer = footer[0]
+    if footer:
+        table.add_row(footer)
+
+    # log location
+    if log_location:
+        # Print a cwd relative path
+        try:
+            log_location_relative = cwd_relative_path(log_location, walk_up=True)
+        except ValueError:
+            log_location_relative = log_location
+        table.add_row(f"Log: {log_location_relative}")
+
+    table.add_row(delimeter)
+    table.add_row("")
+
+    return table
+
+
+def task_panel_title(profile: TaskProfile, show_model: bool) -> str:
+    theme = rich_theme()
+    return (
+        f"[bold][{theme.meta}]{task_title(profile, show_model)}[/{theme.meta}][/bold]"
+    )
+
+
 def to_renderable(item: RenderableType | str, style: str = "") -> RenderableType:
     if isinstance(item, str):
         return Text.from_markup(item, style=style)

inspect_ai/_display/core/textual.py

@@ -8,8 +8,6 @@ logger = getLogger(__name__)
 # force mouse support for textual -- this works around an issue where
 # mouse events are disabled after a reload of the vs code ide, see:
 #   https://github.com/Textualize/textual/issues/5380
-# ansi codes for enabling mouse support are idempotent so it is fine
-# to do this even in cases where mouse support is already enabled.
 # we try/catch since we aren't 100% sure there aren't cases where doing
 # this won't raise and we'd rather not fail hard in in these case
 def textual_enable_mouse_support(driver: Driver) -> None:
@@ -17,5 +15,10 @@ def textual_enable_mouse_support(driver: Driver) -> None:
     if enable_mouse_support:
         try:
             enable_mouse_support()
+            # Re-enable SGR-Pixels format if it was previously enabled.
+            # See #1943.
+            enable_mouse_pixels = getattr(driver, "_enable_mouse_pixels", None)
+            if enable_mouse_pixels and getattr(driver, "_mouse_pixels", False):
+                enable_mouse_pixels()
         except Exception as ex:
             logger.warning(f"Error enabling mouse support: {ex}")

inspect_ai/_display/plain/display.py

@@ -208,3 +208,4 @@ class PlainTaskDisplay(TaskDisplay):
     def complete(self, result: TaskResult) -> None:
         self.task.result = result
         self._print_status()
+        print("")

inspect_ai/_display/rich/display.py

@@ -341,8 +341,6 @@ def tasks_live_status(
 
     # get config
     config = task_config(tasks[0].profile, generate_config=False, style=theme.light)
-    if config:
-        config += "\n"
 
     # build footer table
     footer_table = Table.grid(expand=True)
@@ -356,6 +354,8 @@ def tasks_live_status(
     layout_table = Table.grid(expand=True)
     layout_table.add_column()
     layout_table.add_row(config)
+    if config:
+        layout_table.add_row("")
     layout_table.add_row(progress)
     layout_table.add_row(footer_table)
 

inspect_ai/_display/textual/widgets/transcript.py

@@ -105,9 +105,28 @@ class TranscriptView(ScrollableContainer):
         self, events: Sequence[Event], limit: int = 10
     ) -> list[Widget]:
         widgets: list[Widget] = []
-        widget_count = 0
-        # reverse the events so that the N most recents events are displayed
-        for event in events[::-1]:
+
+        # filter the events to the <limit> most recent
+        filtered_events = events
+        if len(events) > limit:
+            filtered_events = filtered_events[-limit:]
+
+        # find the sample init event
+        sample_init: SampleInitEvent | None = None
+        for event in events:
+            if isinstance(event, SampleInitEvent):
+                sample_init = event
+                break
+
+        # add the sample init event if it isn't already in the event list
+        if sample_init and sample_init not in filtered_events:
+            filtered_events = [sample_init] + list(filtered_events)
+
+        # compute how many events we filtered out
+        filtered_count = len(events) - len(filtered_events)
+        showed_filtered_count = False
+
+        for event in filtered_events:
             display = render_event(event)
             if display:
                 for d in display:
@@ -123,14 +142,23 @@ class TranscriptView(ScrollableContainer):
                             set_transcript_markdown_options(d.content)
                         widgets.append(Static(d.content, markup=False))
                         widgets.append(Static(Text(" ")))
-                        widget_count += 1
 
-            # only render the N most recent events
-            if widget_count >= limit:
-                break
+                        if not showed_filtered_count and filtered_count > 0:
+                            showed_filtered_count = True
+
+                            widgets.append(
+                                Static(
+                                    transcript_separator(
+                                        f"{filtered_count} events..."
+                                        if filtered_count > 1
+                                        else "1 event...",
+                                        self.app.current_theme.primary,
+                                    )
+                                )
+                            )
+                            widgets.append(Static(Text(" ")))
 
-        # reverse the list since we added the events in reverse order
-        return widgets[::-1]
+        return widgets
 
 
 class EventDisplay(NamedTuple):

inspect_ai/_eval/score.py

@@ -1,7 +1,7 @@
 import functools
 from copy import deepcopy
 from pathlib import Path
-from typing import Any, Callable, Literal, cast
+from typing import Any, Callable, Literal
 
 import anyio
 
@@ -270,9 +270,7 @@ def metrics_from_log(log: EvalLog) -> list[Metric] | dict[str, list[Metric]] | N
 
 
 def metric_from_log(metric: EvalMetricDefinition) -> Metric:
-    return cast(
-        Metric, registry_create("metric", metric.name, **(metric.options or {}))
-    )
+    return registry_create("metric", metric.name, **(metric.options or {}))
 
 
 def reducers_from_log(log: EvalLog) -> list[ScoreReducer] | None:

inspect_ai/_eval/task/run.py

@@ -35,11 +35,7 @@ from inspect_ai._util.registry import (
     registry_log_name,
     registry_unqualified_name,
 )
-from inspect_ai._util.working import (
-    end_sample_working_limit,
-    init_sample_working_limit,
-    sample_waiting_time,
-)
+from inspect_ai._util.working import init_sample_working_time, sample_waiting_time
 from inspect_ai._view.notify import view_notify_eval
 from inspect_ai.dataset import Dataset, Sample
 from inspect_ai.log import (
@@ -90,6 +86,8 @@ from inspect_ai.solver._fork import set_task_generate
 from inspect_ai.solver._solver import Solver
 from inspect_ai.solver._task_state import sample_state, set_sample_state, state_jsonable
 from inspect_ai.util._limit import LimitExceededError
+from inspect_ai.util._limit import time_limit as create_time_limit
+from inspect_ai.util._limit import working_limit as create_working_limit
 from inspect_ai.util._sandbox.context import sandbox_connections
 from inspect_ai.util._sandbox.environment import SandboxEnvironmentSpec
 from inspect_ai.util._span import span
@@ -635,10 +633,6 @@ async def task_run_sample(
             )
 
             async with sandboxenv_cm:
-                timeout_cm: (
-                    contextlib._GeneratorContextManager[anyio.CancelScope]
-                    | contextlib.nullcontext[None]
-                ) = contextlib.nullcontext()
                 try:
                     # update active sample wth sandboxes now that we are initialised
                     # (ensure that we still exit init context in presence of sandbox error)
@@ -647,19 +641,17 @@ async def task_run_sample(
                     finally:
                         await init_span.__aexit__(None, None, None)
 
-                    # initialise timeout context manager
-                    timeout_cm = (
-                        anyio.fail_after(time_limit)
-                        if time_limit is not None
-                        else contextlib.nullcontext()
-                    )
-
                     # record start time
                     start_time = time.monotonic()
-                    init_sample_working_limit(start_time, working_limit)
-
-                    # run sample w/ optional timeout
-                    with timeout_cm, state._token_limit, state._message_limit:
+                    init_sample_working_time(start_time)
+
+                    # run sample w/ optional limits
+                    with (
+                        state._token_limit,
+                        state._message_limit,
+                        create_time_limit(time_limit),
+                        create_working_limit(working_limit),
+                    ):
                         # mark started
                         active.started = datetime.now().timestamp()
 
@@ -675,24 +667,15 @@ async def task_run_sample(
                             )
 
                         # set progress for plan then run it
-                        state = await plan(state, generate)
-
-                    # disable sample working limit after execution
-                    end_sample_working_limit()
+                        async with span("solvers"):
+                            state = await plan(state, generate)
 
                 except TimeoutError:
-                    if time_limit is not None:
-                        transcript()._event(
-                            SampleLimitEvent(
-                                type="time",
-                                message=f"Sample completed: exceeded time limit ({time_limit:,} seconds)",
-                                limit=time_limit,
-                            )
-                        )
-                    else:
-                        py_logger.warning(
-                            "Unexpected timeout error reached top of sample stack. Are you handling TimeoutError when applying timeouts?"
-                        )
+                    # Scoped time limits manifest themselves as LimitExceededError, not
+                    # TimeoutError.
+                    py_logger.warning(
+                        "Unexpected timeout error reached top of sample stack. Are you handling TimeoutError when applying timeouts?"
+                    )
 
                     # capture most recent state for scoring
                     state = sample_state() or state
@@ -737,54 +720,59 @@ async def task_run_sample(
                 # the cause of the timeout is a hung container and scoring requires
                 # interacting with the container). as a middle ground we use half
                 # of the original timeout value for scoring.
-                if time_limit is not None:
-                    timeout_cm = anyio.fail_after(time_limit / 2)
+                scoring_time_limit = time_limit / 2 if time_limit else None
 
                 set_sample_state(state)
 
                 # scoring
                 try:
                     # timeout during scoring will result in an ordinary sample error
-                    with timeout_cm:
+                    with create_time_limit(scoring_time_limit):
                         if error is None:
-                            for scorer in scorers or []:
-                                scorer_name = unique_scorer_name(
-                                    scorer, list(results.keys())
-                                )
-                                async with span(name=scorer_name, type="scorer"):
-                                    score_result = (
-                                        await scorer(state, Target(sample.target))
-                                        if scorer
-                                        else None
+                            async with span(name="scorers"):
+                                for scorer in scorers or []:
+                                    scorer_name = unique_scorer_name(
+                                        scorer, list(results.keys())
                                     )
-                                    if score_result is not None:
-                                        sample_score = SampleScore(
-                                            score=score_result,
-                                            sample_id=sample.id,
-                                            sample_metadata=sample.metadata,
-                                            scorer=registry_unqualified_name(scorer),
+                                    async with span(name=scorer_name, type="scorer"):
+                                        score_result = (
+                                            await scorer(state, Target(sample.target))
+                                            if scorer
+                                            else None
+                                        )
+                                        if score_result is not None:
+                                            sample_score = SampleScore(
+                                                score=score_result,
+                                                sample_id=sample.id,
+                                                sample_metadata=sample.metadata,
+                                                scorer=registry_unqualified_name(
+                                                    scorer
+                                                ),
+                                            )
+                                            transcript()._event(
+                                                ScoreEvent(
+                                                    score=score_result,
+                                                    target=sample.target,
+                                                )
+                                            )
+                                            results[scorer_name] = sample_score
+
+                                # add scores returned by solvers
+                                if state.scores is not None:
+                                    for name, score in state.scores.items():
+                                        results[name] = SampleScore(
+                                            score=score,
+                                            sample_id=state.sample_id,
+                                            sample_metadata=state.metadata,
                                         )
                                         transcript()._event(
                                             ScoreEvent(
-                                                score=score_result, target=sample.target
+                                                score=score, target=sample.target
                                             )
                                         )
-                                        results[scorer_name] = sample_score
-
-                            # add scores returned by solvers
-                            if state.scores is not None:
-                                for name, score in state.scores.items():
-                                    results[name] = SampleScore(
-                                        score=score,
-                                        sample_id=state.sample_id,
-                                        sample_metadata=state.metadata,
-                                    )
-                                    transcript()._event(
-                                        ScoreEvent(score=score, target=sample.target)
-                                    )
 
-                            # propagate results into scores
-                            state.scores = {k: v.score for k, v in results.items()}
+                                # propagate results into scores
+                                state.scores = {k: v.score for k, v in results.items()}
 
                 except anyio.get_cancelled_exc_class():
                     if active.interrupt_action:
@@ -798,17 +786,7 @@ async def task_run_sample(
                     raise
 
                 except BaseException as ex:
-                    # note timeout
-                    if isinstance(ex, TimeoutError):
-                        transcript()._event(
-                            SampleLimitEvent(
-                                type="time",
-                                message=f"Unable to score sample due to exceeded time limit ({time_limit:,} seconds)",
-                                limit=time_limit,
-                            )
-                        )
-
-                    # handle error (this will throw if we've exceeded the limit)
+                    # handle error
                     error, raise_error = handle_error(ex)
 
         except Exception as ex:

inspect_ai/_util/content.py

@@ -1,9 +1,14 @@
 from typing import Literal, Union
 
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, Field, JsonValue
 
 
-class ContentText(BaseModel):
+class ContentBase(BaseModel):
+    internal: JsonValue | None = Field(default=None)
+    """Model provider specific payload - typically used to aid transformation back to model types."""
+
+
+class ContentText(ContentBase):
     """Text content."""
 
     type: Literal["text"] = Field(default="text")
@@ -16,7 +21,7 @@ class ContentText(BaseModel):
     """Was this a refusal message?"""
 
 
-class ContentReasoning(BaseModel):
+class ContentReasoning(ContentBase):
     """Reasoning content.
 
     See the specification for [thinking blocks](https://docs.anthropic.com/en/docs/build-with-claude/extended-thinking#understanding-thinking-blocks) for Claude models.
@@ -35,7 +40,7 @@ class ContentReasoning(BaseModel):
     """Indicates that the explicit content of this reasoning block has been redacted."""
 
 
-class ContentImage(BaseModel):
+class ContentImage(ContentBase):
     """Image content."""
 
     type: Literal["image"] = Field(default="image")
@@ -51,7 +56,7 @@ class ContentImage(BaseModel):
     """
 
 
-class ContentAudio(BaseModel):
+class ContentAudio(ContentBase):
     """Audio content."""
 
     type: Literal["audio"] = Field(default="audio")
@@ -64,7 +69,7 @@ class ContentAudio(BaseModel):
     """Format of audio data ('mp3' or 'wav')"""
 
 
-class ContentVideo(BaseModel):
+class ContentVideo(ContentBase):
     """Video content."""
 
     type: Literal["video"] = Field(default="video")

inspect_ai/_util/interrupt.py

@@ -1,6 +1,6 @@
 import anyio
 
-from .working import check_sample_working_limit
+from inspect_ai.util._limit import check_working_limit
 
 
 def check_sample_interrupt() -> None:
@@ -12,4 +12,4 @@ def check_sample_interrupt() -> None:
         raise anyio.get_cancelled_exc_class()
 
     # check for working_limit
-    check_sample_working_limit()
+    check_working_limit()

inspect_ai/_util/text.py

@@ -1,12 +1,19 @@
 import random
 import re
 import string
+import textwrap
 from logging import getLogger
 from typing import List, NamedTuple
 
 logger = getLogger(__name__)
 
 
+def truncate_text(text: str, max_length: int) -> str:
+    if len(text) <= max_length:
+        return text
+    return textwrap.shorten(text, width=max_length, placeholder="...")
+
+
 def strip_punctuation(s: str) -> str:
     return s.strip(string.whitespace + string.punctuation)
 

inspect_ai/_util/working.py

@@ -1,19 +1,14 @@
 import time
 from contextvars import ContextVar
 
-from inspect_ai.util._limit import LimitExceededError
+from inspect_ai.util._limit import check_working_limit, record_waiting_time
 
 
-def init_sample_working_limit(start_time: float, working_limit: float | None) -> None:
-    _sample_working_limit.set(working_limit)
+def init_sample_working_time(start_time: float) -> None:
     _sample_start_time.set(start_time)
     _sample_waiting_time.set(0)
 
 
-def end_sample_working_limit() -> None:
-    _sample_working_limit.set(None)
-
-
 def sample_waiting_time() -> float:
     return _sample_waiting_time.get()
 
@@ -23,37 +18,13 @@ def sample_working_time() -> float:
 
 
 def report_sample_waiting_time(waiting_time: float) -> None:
+    # record and check for scoped limits
+    record_waiting_time(waiting_time)
+    check_working_limit()
+
+    # record sample-level limits
     _sample_waiting_time.set(_sample_waiting_time.get() + waiting_time)
-    check_sample_working_limit()
-
-
-def check_sample_working_limit() -> None:
-    from inspect_ai.log._transcript import SampleLimitEvent, transcript
-
-    # no check if we don't have a limit
-    working_limit = _sample_working_limit.get()
-    if working_limit is None:
-        return
-
-    # are we over the limit?
-    running_time = time.monotonic() - _sample_start_time.get()
-    working_time = running_time - sample_waiting_time()
-    if working_time > working_limit:
-        message = f"Exceeded working time limit ({working_limit:,} seconds)"
-        transcript()._event(
-            SampleLimitEvent(type="working", limit=int(working_limit), message=message)
-        )
-        raise LimitExceededError(
-            type="working",
-            value=int(working_time),
-            limit=int(working_limit),
-            message=message,
-        )
-
-
-_sample_working_limit: ContextVar[float | None] = ContextVar(
-    "sample_working_limit", default=None
-)
+
 
 _sample_start_time: ContextVar[float] = ContextVar("sample_start_time", default=0)
 

inspect_ai/_view/schema.py

@@ -45,9 +45,10 @@ def sync_view_schema() -> None:
                 "false",
             ],
             cwd=WWW_DIR,
+            check=True,
         )
 
-        subprocess.run(["yarn", "prettier:write"], cwd=types_path.parent)
+        subprocess.run(["yarn", "prettier:write"], cwd=types_path.parent, check=True)
 
         shutil.copyfile(types_path, vs_code_types_path)
 

inspect_ai/_view/www/CLAUDE.md

@@ -0,0 +1,15 @@
+# CLAUDE.md
+
+This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
+
+## Build/Lint/Test Commands
+
+- Run type checker, linter, and tests: `yarn check-all`
+- Run type checker: `yarn tsc`
+- Run linter: `yarn lint`
+- Run tests: `yarn test`
+
+## Other Information
+
+- The code in this project is typescript, learn more about the configuration by inspecting package.json.
+- Respect existing code patterns when modifying files. Run linting before committing changes.