inspect-ai 0.3.72__py3-none-any.whl → 0.3.73__py3-none-any.whl

inspect_ai/_cli/eval.py

@@ -11,12 +11,12 @@ from inspect_ai._util.constants import (
     DEFAULT_EPOCHS,
     DEFAULT_LOG_LEVEL_TRANSCRIPT,
     DEFAULT_MAX_CONNECTIONS,
-    DEFAULT_MAX_RETRIES,
 )
 from inspect_ai._util.file import filesystem
 from inspect_ai._util.samples import parse_sample_id, parse_samples_limit
 from inspect_ai.log._file import log_file_info
 from inspect_ai.model import GenerateConfigArgs
+from inspect_ai.model._generate_config import ResponseSchema
 from inspect_ai.scorer._reducer import create_reducers
 from inspect_ai.solver._solver import SolverSpec
 
@@ -47,9 +47,9 @@ NO_SCORE_HELP = (
 NO_SCORE_DISPLAY = "Do not display scoring metrics in realtime."
 MAX_CONNECTIONS_HELP = f"Maximum number of concurrent connections to Model API (defaults to {DEFAULT_MAX_CONNECTIONS})"
 MAX_RETRIES_HELP = (
-    f"Maximum number of times to retry request (defaults to {DEFAULT_MAX_RETRIES})"
+    "Maximum number of times to retry model API requests (defaults to unlimited)"
 )
-TIMEOUT_HELP = "Request timeout (in seconds)."
+TIMEOUT_HELP = "Model API request timeout in seconds (defaults to no timeout)"
 
 
 def eval_options(func: Callable[..., Any]) -> Callable[..., click.Context]:
@@ -405,6 +405,12 @@ def eval_options(func: Callable[..., Any]) -> Callable[..., click.Context]:
         help='Include reasoning in chat message history sent to generate (defaults to "auto", which uses the recommended default for each provider)',
         envvar="INSPECT_EVAL_REASONING_HISTORY",
     )
+    @click.option(
+        "--response-schema",
+        type=str,
+        help="JSON schema for desired response format (output should still be validated). OpenAI, Google, and Mistral only.",
+        envvar="INSPECT_EVAL_RESPONSE_SCHEMA",
+    )
     @click.option(
         "--log-format",
         type=click.Choice(["eval", "json"], case_sensitive=False),
@@ -476,6 +482,7 @@ def eval_command(
     reasoning_effort: str | None,
     reasoning_tokens: int | None,
     reasoning_history: Literal["none", "all", "last", "auto"] | None,
+    response_schema: ResponseSchema | None,
     message_limit: int | None,
     token_limit: int | None,
     time_limit: int | None,
@@ -640,6 +647,7 @@ def eval_set_command(
     reasoning_effort: str | None,
     reasoning_tokens: int | None,
     reasoning_history: Literal["none", "all", "last", "auto"] | None,
+    response_schema: ResponseSchema | None,
     message_limit: int | None,
     token_limit: int | None,
     time_limit: int | None,
@@ -889,6 +897,9 @@ def config_from_locals(locals: dict[str, Any]) -> GenerateConfigArgs:
             if key == "reasoning_history":
                 if value is not False:
                     value = None
+            if key == "response_schema":
+                if value is not None:
+                    value = ResponseSchema.model_validate_json(value)
             config[key] = value  # type: ignore
     return config
 

inspect_ai/_cli/sandbox.py

@@ -1,7 +1,7 @@
-import asyncio
-
+import anyio
 import click
 
+from inspect_ai._util._async import configured_async_backend
 from inspect_ai.util._sandbox.registry import registry_find_sandboxenv
 
 
@@ -27,4 +27,4 @@ def sandbox_cleanup(type: str, environment_id: str | None) -> None:
     """
     sandboxenv_type = registry_find_sandboxenv(type)
     cli_cleanup = getattr(sandboxenv_type, "cli_cleanup")
-    asyncio.run(cli_cleanup(environment_id))
+    anyio.run(cli_cleanup, environment_id, backend=configured_async_backend())

inspect_ai/_cli/score.py

@@ -1,6 +1,6 @@
-import asyncio
 import os
 
+import anyio
 import click
 import rich
 from rich.panel import Panel
@@ -13,6 +13,7 @@ from inspect_ai._display import display
 from inspect_ai._display.core.rich import rich_theme
 from inspect_ai._eval.context import init_eval_context, init_task_context
 from inspect_ai._eval.score import ScoreAction, task_score
+from inspect_ai._util._async import configured_async_backend
 from inspect_ai._util.file import basename, dirname, exists
 from inspect_ai.log._log import EvalLog
 from inspect_ai.log._recorders import create_recorder_for_location
@@ -64,8 +65,8 @@ def score_command(
     process_common_options(common)
 
     # score
-    asyncio.run(
-        score(
+    async def run_score() -> None:
+        return await score(
             log_dir=common["log_dir"],
             log_file=log_file,
             scorer=scorer,
@@ -74,7 +75,8 @@ def score_command(
             action=action,
             log_level=common["log_level"],
         )
-    )
+
+    anyio.run(run_score, backend=configured_async_backend())
 
 
 async def score(

inspect_ai/_cli/trace.py

@@ -15,6 +15,7 @@ from rich.table import Column, Table
 from inspect_ai._util.error import PrerequisiteError
 from inspect_ai._util.trace import (
     ActionTraceRecord,
+    TraceRecord,
     inspect_trace_dir,
     list_trace_files,
     read_trace_file,
@@ -84,6 +85,41 @@ def dump_command(trace_file: str | None, filter: str | None) -> None:
     )
 
 
+@trace_command.command("http")
+@click.argument("trace-file", type=str, required=False)
+@click.option(
+    "--filter",
+    type=str,
+    help="Filter (applied to trace message field).",
+)
+@click.option(
+    "--failed",
+    type=bool,
+    is_flag=True,
+    default=False,
+    help="Show only failed HTTP requests (non-200 status)",
+)
+def http_command(trace_file: str | None, filter: str | None, failed: bool) -> None:
+    """View all HTTP requests in the trace log."""
+    _, traces = _read_traces(trace_file, "HTTP", filter)
+
+    last_timestamp = ""
+    table = Table(Column(), Column(), box=None)
+    for trace in traces:
+        if failed and "200 OK" in trace.message:
+            continue
+        timestamp = trace.timestamp.split(".")[0]
+        if timestamp == last_timestamp:
+            timestamp = ""
+        else:
+            last_timestamp = timestamp
+            timestamp = f"[{timestamp}]"
+        table.add_row(timestamp, trace.message)
+
+    if table.row_count > 0:
+        r_print(table)
+
+
 @trace_command.command("anomalies")
 @click.argument("trace-file", type=str, required=False)
 @click.option(
@@ -99,12 +135,7 @@ def dump_command(trace_file: str | None, filter: str | None) -> None:
 )
 def anomolies_command(trace_file: str | None, filter: str | None, all: bool) -> None:
     """Look for anomalies in a trace file (never completed or cancelled actions)."""
-    trace_file_path = _resolve_trace_file_path(trace_file)
-    traces = read_trace_file(trace_file_path)
-
-    if filter:
-        filter = filter.lower()
-        traces = [trace for trace in traces if filter in trace.message.lower()]
+    trace_file_path, traces = _read_traces(trace_file, None, filter)
 
     # Track started actions
     running_actions: dict[str, ActionTraceRecord] = {}
@@ -199,6 +230,22 @@ def anomolies_command(trace_file: str | None, filter: str | None, all: bool) ->
         print(console.export_text(styles=True).strip())
 
 
+def _read_traces(
+    trace_file: str | None, level: str | None = None, filter: str | None = None
+) -> tuple[Path, list[TraceRecord]]:
+    trace_file_path = _resolve_trace_file_path(trace_file)
+    traces = read_trace_file(trace_file_path)
+
+    if level:
+        traces = [trace for trace in traces if trace.level == level]
+
+    if filter:
+        filter = filter.lower()
+        traces = [trace for trace in traces if filter in trace.message.lower()]
+
+    return (trace_file_path, traces)
+
+
 def _print_bucket(
     print_fn: Callable[[RenderableType], None],
     label: str,

inspect_ai/_display/core/config.py

@@ -34,7 +34,7 @@ def task_config(
             value = value if isinstance(value, list) else [value]
             value = [str(v) for v in value]
             config_print.append(f"{name}: {','.join(value)}")
-        elif name not in ["limit", "model"]:
+        elif name not in ["limit", "model", "response_schema"]:
             if isinstance(value, list):
                 value = ",".join([str(v) for v in value])
             if isinstance(value, str):

inspect_ai/_display/core/display.py

@@ -4,6 +4,7 @@ from types import TracebackType
 from typing import (
     Any,
     AsyncIterator,
+    Callable,
     Coroutine,
     Iterator,
     Protocol,
@@ -130,7 +131,7 @@ class Display(Protocol):
     @contextlib.contextmanager
     def progress(self, total: int) -> Iterator[Progress]: ...
 
-    def run_task_app(self, main: Coroutine[Any, Any, TR]) -> TR: ...
+    def run_task_app(self, main: Callable[[], Coroutine[None, None, TR]]) -> TR: ...
 
     @contextlib.contextmanager
     def suspend_task_app(self) -> Iterator[None]: ...

inspect_ai/_display/core/footer.py

@@ -1,7 +1,7 @@
 from rich.console import RenderableType
 from rich.text import Text
 
-from inspect_ai._util.logger import http_rate_limit_count
+from inspect_ai._util.retry import http_retries_count
 from inspect_ai.util._concurrency import concurrency_status
 from inspect_ai.util._throttle import throttle
 
@@ -26,12 +26,12 @@ def task_resources() -> str:
 
 
 def task_counters(counters: dict[str, str]) -> str:
-    return task_dict(counters | task_http_rate_limits())
+    return task_dict(counters | task_http_retries())
 
 
-def task_http_rate_limits() -> dict[str, str]:
-    return {"HTTP rate limits": f"{http_rate_limit_count():,}"}
+def task_http_retries() -> dict[str, str]:
+    return {"HTTP retries": f"{http_retries_count():,}"}
 
 
-def task_http_rate_limits_str() -> str:
-    return f"HTTP rate limits: {http_rate_limit_count():,}"
+def task_http_retries_str() -> str:
+    return f"HTTP retries: {http_retries_count():,}"

inspect_ai/_display/plain/display.py

@@ -1,10 +1,12 @@
-import asyncio
 import contextlib
-from typing import Any, AsyncIterator, Coroutine, Iterator
+from typing import AsyncIterator, Callable, Coroutine, Iterator
 
+import anyio
 import rich
 
 from inspect_ai._display.core.rich import rich_initialise
+from inspect_ai._util._async import configured_async_backend, run_coroutine
+from inspect_ai._util.platform import running_in_notebook
 from inspect_ai._util.text import truncate
 from inspect_ai._util.throttle import throttle
 
@@ -22,7 +24,7 @@ from ..core.display import (
     TaskSpec,
     TaskWithResult,
 )
-from ..core.footer import task_http_rate_limits_str
+from ..core.footer import task_http_retries_str
 from ..core.panel import task_panel, task_targets
 from ..core.results import task_metric, tasks_results
 
@@ -41,8 +43,11 @@ class PlainDisplay(Display):
     def progress(self, total: int) -> Iterator[Progress]:
         yield PlainProgress(total)
 
-    def run_task_app(self, main: Coroutine[Any, Any, TR]) -> TR:
-        return asyncio.run(main)
+    def run_task_app(self, main: Callable[[], Coroutine[None, None, TR]]) -> TR:
+        if running_in_notebook():
+            return run_coroutine(main())
+        else:
+            return anyio.run(main, backend=configured_async_backend())
 
     @contextlib.contextmanager
     def suspend_task_app(self) -> Iterator[None]:
@@ -182,7 +187,7 @@ class PlainTaskDisplay(TaskDisplay):
             status_parts.append(resources)
 
             # Add rate limits
-            rate_limits = task_http_rate_limits_str()
+            rate_limits = task_http_retries_str()
             if rate_limits:
                 status_parts.append(rate_limits)
 

inspect_ai/_display/rich/display.py

@@ -1,8 +1,8 @@
-import asyncio
 import contextlib
 from dataclasses import dataclass
 from typing import Any, AsyncIterator, Callable, Coroutine, Iterator
 
+import anyio
 import rich
 from rich.console import Console, Group, RenderableType
 from rich.live import Live
@@ -11,7 +11,9 @@ from rich.progress import Progress as RProgress
 from rich.table import Table
 from typing_extensions import override
 
+from inspect_ai._util._async import configured_async_backend, run_coroutine
 from inspect_ai._util.constants import CONSOLE_DISPLAY_WIDTH
+from inspect_ai._util.platform import running_in_notebook
 from inspect_ai.log._transcript import InputEvent, transcript
 from inspect_ai.util._display import display_type
 from inspect_ai.util._throttle import throttle
@@ -59,7 +61,6 @@ class RichDisplay(Display):
         self.progress_ui: RProgress | None = None
         self.parallel = False
         self.live: Live | None = None
-        self.timer_handle: asyncio.TimerHandle | None = None
         self.counters: dict[str, str] = {}
         rich_initialise()
 
@@ -74,8 +75,11 @@ class RichDisplay(Display):
             yield RichProgress(total, progress)
 
     @override
-    def run_task_app(self, main: Coroutine[Any, Any, TR]) -> TR:
-        return asyncio.run(main)
+    def run_task_app(self, main: Callable[[], Coroutine[None, None, TR]]) -> TR:
+        if running_in_notebook():
+            return run_coroutine(main())
+        else:
+            return anyio.run(main, backend=configured_async_backend())
 
     @override
     @contextlib.contextmanager
@@ -104,13 +108,15 @@ class RichDisplay(Display):
                 with RichTaskScreen(live) as task_screen:
                     self.live = live
 
-                    # enque a display update
-                    self.timer_handle = asyncio.get_event_loop().call_later(
-                        1, self._update_display
-                    )
+                    async with anyio.create_task_group() as tg:
+                        # update display every second while running
+                        tg.start_soon(self._update_display_loop)
 
-                    # yield
-                    yield task_screen
+                        # let the task screen run
+                        try:
+                            yield task_screen
+                        finally:
+                            tg.cancel_scope.cancel()
 
                 # render task results (re-enable live if necessary)
                 if not live.is_started:
@@ -124,8 +130,6 @@ class RichDisplay(Display):
             self.progress_ui = None
             self.parallel = False
             self.live = None
-            if self.timer_handle:
-                self.timer_handle.cancel()
 
     @override
     @contextlib.contextmanager
@@ -161,7 +165,13 @@ class RichDisplay(Display):
                 r = task_live_status(self.tasks, self.progress_ui, self.counters)
             self.live.update(r, refresh=True)
 
-        self.timer_handle = asyncio.get_event_loop().call_later(1, self._update_display)
+    async def _update_display_loop(self) -> None:
+        try:
+            while True:
+                await anyio.sleep(1)
+                self._update_display()
+        except Exception:
+            pass
 
     @override
     def display_counter(self, caption: str, value: str) -> None:

inspect_ai/_display/textual/app.py

@@ -1,16 +1,18 @@
-import asyncio
 import contextlib
 from asyncio import CancelledError
 from typing import (
     Any,
     AsyncIterator,
+    Awaitable,
+    Callable,
     ClassVar,
-    Coroutine,
     Generic,
     Iterator,
     cast,
 )
 
+import anyio
+import anyio.from_thread
 import rich
 from rich.console import Console
 from textual.app import App, ComposeResult
@@ -103,9 +105,8 @@ class TaskScreenApp(App[TR]):
         if focus and self.app._driver:
             textual_enable_mouse_support(self.app._driver)
 
-    def run_app(self, main: Coroutine[Any, Any, TR]) -> TaskScreenResult[TR]:
-        # create the worker
-        self._worker = self.run_worker(main, start=False, exit_on_error=False)
+    def run_app(self, main: Callable[[], Awaitable[TR]]) -> TaskScreenResult[TR]:
+        self._worker = self.run_worker(main(), start=False, exit_on_error=False)
 
         # run the app
         self.run()
@@ -123,8 +124,8 @@ class TaskScreenApp(App[TR]):
 
     async def on_load(self) -> None:
         # events used to synchronise loading
-        self._on_load_app = asyncio.Event()
-        self._on_app_loaded = asyncio.Event()
+        self._on_load_app = anyio.Event()
+        self._on_app_loaded = anyio.Event()
 
         # run the workers
         self.workers.start_all()
@@ -136,7 +137,7 @@ class TaskScreenApp(App[TR]):
         while not self._on_load_app.is_set():
             if len(self.workers._workers) == 0:
                 return
-            await asyncio.sleep(0.1)
+            await anyio.sleep(0.1)
 
     @contextlib.contextmanager
     def suspend_app(self) -> Iterator[None]:
@@ -422,7 +423,7 @@ class TaskScreenApp(App[TR]):
 class TextualTaskScreen(TaskScreen, Generic[TR]):
     def __init__(self, app: TaskScreenApp[TR]) -> None:
         self.app = app
-        self.lock = asyncio.Lock()
+        self.lock = anyio.Lock()
 
     def __exit__(self, *excinfo: Any) -> None:
         pass

inspect_ai/_display/textual/display.py

@@ -1,5 +1,5 @@
 import contextlib
-from typing import Any, AsyncIterator, Coroutine, Iterator
+from typing import AsyncIterator, Callable, Coroutine, Iterator
 
 import rich
 from typing_extensions import override
@@ -30,7 +30,7 @@ class TextualDisplay(Display):
             yield RichProgress(total, progress)
 
     @override
-    def run_task_app(self, main: Coroutine[Any, Any, TR]) -> TR:
+    def run_task_app(self, main: Callable[[], Coroutine[None, None, TR]]) -> TR:
         # create and run the app
         self.app = TaskScreenApp[TR]()
         result = self.app.run_app(main)

inspect_ai/_display/textual/widgets/footer.py

@@ -36,3 +36,7 @@ class AppFooter(Widget):
     def watch_right(self, new_right: RenderableType) -> None:
         footer_right = cast(Static, self.query_one("#footer-right"))
         footer_right.update(new_right)
+        if footer_right.tooltip is None:
+            footer_right.tooltip = (
+                "Execute 'inspect trace http' for a log of all HTTP requests."
+            )

inspect_ai/_display/textual/widgets/samples.py

@@ -506,6 +506,7 @@ class SampleToolbar(Horizontal):
         # track the sample
         self.sample = sample
 
+        status_group = self.query_one("#" + self.STATUS_GROUP)
         pending_status = self.query_one("#" + self.PENDING_STATUS)
         timeout_tool = self.query_one("#" + self.TIMEOUT_TOOL_CALL)
         clock = self.query_one(Clock)
@@ -537,11 +538,19 @@ class SampleToolbar(Horizontal):
                 pending_caption = cast(
                     Static, self.query_one("#" + self.PENDING_CAPTION)
                 )
-                pending_caption_text = (
-                    "Generating..."
-                    if isinstance(last_event, ModelEvent)
-                    else "Executing..."
-                )
+                if isinstance(last_event, ModelEvent):
+                    # see if there are retries in play
+                    if sample.retry_count > 0:
+                        suffix = "retry" if sample.retry_count == 1 else "retries"
+                        pending_caption_text = (
+                            f"Generating ({sample.retry_count:,} {suffix})..."
+                        )
+                    else:
+                        pending_caption_text = "Generating..."
+                else:
+                    pending_caption_text = "Executing..."
+                status_group.styles.width = max(22, len(pending_caption_text))
+
                 pending_caption.update(
                     Text.from_markup(f"[italic]{pending_caption_text}[/italic]")
                 )

inspect_ai/_eval/context.py

@@ -1,6 +1,6 @@
 from inspect_ai._util.dotenv import init_dotenv
 from inspect_ai._util.hooks import init_hooks
-from inspect_ai._util.logger import init_http_rate_limit_count, init_logger
+from inspect_ai._util.logger import init_logger
 from inspect_ai.approval._apply import have_tool_approval, init_tool_approval
 from inspect_ai.approval._human.manager import init_human_approval_manager
 from inspect_ai.approval._policy import ApprovalPolicy
@@ -20,7 +20,6 @@ def init_eval_context(
     init_logger(log_level, log_level_transcript)
     init_concurrency()
     init_max_subprocesses(max_subprocesses)
-    init_http_rate_limit_count()
     init_hooks()
     init_active_samples()
     init_human_approval_manager()

inspect_ai/_eval/eval.py

@@ -1,8 +1,12 @@
 import logging
 import os
+import sys
 from pathlib import Path
 from typing import Any, Literal
 
+if sys.version_info < (3, 11):
+    from exceptiongroup import ExceptionGroup
+
 from shortuuid import uuid
 from typing_extensions import Unpack
 
@@ -166,43 +170,51 @@ def eval(
         display, trace, max_tasks, max_samples, model
     )
 
-    return task_display().run_task_app(
-        main=eval_async(
-            tasks=tasks,
-            model=model,
-            model_base_url=model_base_url,
-            model_args=model_args,
-            task_args=task_args,
-            sandbox=sandbox,
-            sandbox_cleanup=sandbox_cleanup,
-            solver=solver,
-            tags=tags,
-            approval=approval,
-            log_level=log_level,
-            log_level_transcript=log_level_transcript,
-            log_dir=log_dir,
-            log_format=log_format,
-            limit=limit,
-            sample_id=sample_id,
-            epochs=epochs,
-            fail_on_error=fail_on_error,
-            debug_errors=debug_errors,
-            message_limit=message_limit,
-            token_limit=token_limit,
-            time_limit=time_limit,
-            working_limit=working_limit,
-            max_samples=max_samples,
-            max_tasks=max_tasks,
-            max_subprocesses=max_subprocesses,
-            max_sandboxes=max_sandboxes,
-            log_samples=log_samples,
-            log_images=log_images,
-            log_buffer=log_buffer,
-            score=score,
-            score_display=score_display,
-            **kwargs,
-        )
-    )
+    async def run_task_app() -> list[EvalLog]:
+        try:
+            return await eval_async(
+                tasks=tasks,
+                model=model,
+                model_base_url=model_base_url,
+                model_args=model_args,
+                task_args=task_args,
+                sandbox=sandbox,
+                sandbox_cleanup=sandbox_cleanup,
+                solver=solver,
+                tags=tags,
+                approval=approval,
+                log_level=log_level,
+                log_level_transcript=log_level_transcript,
+                log_dir=log_dir,
+                log_format=log_format,
+                limit=limit,
+                sample_id=sample_id,
+                epochs=epochs,
+                fail_on_error=fail_on_error,
+                debug_errors=debug_errors,
+                message_limit=message_limit,
+                token_limit=token_limit,
+                time_limit=time_limit,
+                working_limit=working_limit,
+                max_samples=max_samples,
+                max_tasks=max_tasks,
+                max_subprocesses=max_subprocesses,
+                max_sandboxes=max_sandboxes,
+                log_samples=log_samples,
+                log_images=log_images,
+                log_buffer=log_buffer,
+                score=score,
+                score_display=score_display,
+                **kwargs,
+            )
+        # exceptions can escape when debug_errors is True and that's okay
+        except ExceptionGroup as ex:
+            if debug_errors:
+                raise ex.exceptions[0] from None
+            else:
+                raise
+
+    return task_display().run_task_app(run_task_app)
 
 
 # single call to eval_async at a time
@@ -556,8 +568,8 @@ def eval_retry(
     # resolve eval trace
     max_tasks, max_samples = init_eval_display(display, trace, max_tasks, max_samples)
 
-    return task_display().run_task_app(
-        main=eval_retry_async(
+    async def run_task_app() -> list[EvalLog]:
+        return await eval_retry_async(
             tasks=tasks,
             log_level=log_level,
             log_level_transcript=log_level_transcript,
@@ -578,8 +590,9 @@ def eval_retry(
             max_retries=max_retries,
             timeout=timeout,
             max_connections=max_connections,
-        ),
-    )
+        )
+
+    return task_display().run_task_app(run_task_app)
 
 
 async def eval_retry_async(