inspect-ai 0.3.49__py3-none-any.whl → 0.3.51__py3-none-any.whl

inspect_ai/approval/_human/manager.py

@@ -0,0 +1,108 @@
+import asyncio
+import uuid
+from asyncio import Future
+from contextvars import ContextVar
+from typing import Callable, Literal, NamedTuple, cast
+
+from inspect_ai.solver._task_state import TaskState
+from inspect_ai.tool._tool_call import ToolCall, ToolCallView
+
+from .._approval import Approval, ApprovalDecision
+
+
+class ApprovalRequest(NamedTuple):
+    message: str
+    call: ToolCall
+    view: ToolCallView
+    state: TaskState | None
+    choices: list[ApprovalDecision]
+
+
+class PendingApprovalRequest(NamedTuple):
+    request: ApprovalRequest
+    task: str
+    model: str
+    id: int | str
+    epoch: int
+
+
+class HumanApprovalManager:
+    def __init__(self) -> None:
+        self._approval_requests: dict[
+            str, tuple[PendingApprovalRequest, Future[Approval]]
+        ] = {}
+        self._change_callbacks: list[Callable[[Literal["add", "remove"]], None]] = []
+
+    def request_approval(self, request: ApprovalRequest) -> str:
+        from inspect_ai.log._samples import sample_active
+
+        id = str(uuid.uuid4())
+        future = cast(Future[Approval], asyncio.get_event_loop().create_future())
+        sample = sample_active()
+        assert sample
+        assert sample.sample.id
+        pending = PendingApprovalRequest(
+            request=request,
+            task=sample.task,
+            model=sample.model,
+            id=sample.sample.id,
+            epoch=sample.epoch,
+        )
+        self._approval_requests[id] = (pending, future)
+        self._notify_change("add")
+        return id
+
+    def withdraw_request(self, id: str) -> None:
+        del self._approval_requests[id]
+        self._notify_change("remove")
+
+    async def wait_for_approval(self, id: str) -> Approval:
+        _, future = self._approval_requests[id]
+        return await future
+
+    def on_change(
+        self, callback: Callable[[Literal["add", "remove"]], None]
+    ) -> Callable[[], None]:
+        self._change_callbacks.append(callback)
+
+        def unsubscribe() -> None:
+            if callback in self._change_callbacks:
+                self._change_callbacks.remove(callback)
+
+        return unsubscribe
+
+    def approval_requests(self) -> list[tuple[str, PendingApprovalRequest]]:
+        return [(aid, data) for aid, (data, _) in self._approval_requests.items()]
+
+    def complete_approval(self, id: str, result: Approval) -> None:
+        if id in self._approval_requests:
+            _, future = self._approval_requests[id]
+            if not future.done():
+                future.set_result(result)
+            del self._approval_requests[id]
+            self._notify_change("remove")
+
+    def fail_approval(self, id: str, error: Exception) -> None:
+        if id in self._approval_requests:
+            _, future = self._approval_requests[id]
+            if not future.done():
+                future.set_exception(error)
+            del self._approval_requests[id]
+            self._notify_change("remove")
+
+    def _notify_change(self, action: Literal["add", "remove"]) -> None:
+        for callback in self._change_callbacks:
+            callback(action)
+
+
+def human_approval_manager() -> HumanApprovalManager:
+    return _human_approval_manager.get()
+
+
+def init_human_approval_manager() -> None:
+    _human_approval_manager.set(HumanApprovalManager())
+
+
+_human_approval_manager: ContextVar[HumanApprovalManager] = ContextVar(
+    "_human_approval_manager"
+)

inspect_ai/approval/_human/panel.py

@@ -0,0 +1,233 @@
+from asyncio import CancelledError
+from typing import Callable, Literal
+
+from rich.console import RenderableType
+from rich.text import Text
+from textual.app import ComposeResult
+from textual.containers import Horizontal, ScrollableContainer
+from textual.reactive import reactive
+from textual.widgets import Button, Static
+from typing_extensions import override
+
+from inspect_ai._util.registry import registry_unqualified_name
+from inspect_ai.solver._task_state import TaskState
+from inspect_ai.tool._tool_call import ToolCall, ToolCallView
+from inspect_ai.util._panel import InputPanel, input_panel
+
+from .._approval import Approval, ApprovalDecision
+from .manager import ApprovalRequest, PendingApprovalRequest, human_approval_manager
+from .util import (
+    HUMAN_APPROVED,
+    HUMAN_ESCALATED,
+    HUMAN_REJECTED,
+    HUMAN_TERMINATED,
+    render_tool_approval,
+)
+
+PANEL_TITLE = "Approvals"
+
+
+async def panel_approval(
+    message: str,
+    call: ToolCall,
+    view: ToolCallView,
+    state: TaskState | None,
+    choices: list[ApprovalDecision],
+) -> Approval:
+    # ensure the approvals panel is shown
+    await input_panel(PANEL_TITLE, ApprovalInputPanel)
+
+    # submit to human approval manager (will be picked up by panel)
+    approvals = human_approval_manager()
+    id = approvals.request_approval(
+        ApprovalRequest(
+            message=message, call=call, view=view, state=state, choices=choices
+        )
+    )
+    try:
+        return await approvals.wait_for_approval(id)
+    except CancelledError:
+        approvals.withdraw_request(id)
+        raise
+
+
+class ApprovalInputPanel(InputPanel):
+    DEFAULT_CSS = """
+    ApprovalInputPanel {
+        width: 1fr;
+        height: 1fr;
+        padding: 0 1 1 1;
+        layout: grid;
+        grid-size: 1 3;
+        grid-rows: auto 1fr auto;
+    }
+    """
+
+    _approvals: list[tuple[str, PendingApprovalRequest]] = []
+    _unsubscribe: Callable[[], None] | None = None
+
+    @override
+    def compose(self) -> ComposeResult:
+        yield ApprovalRequestHeading()
+        yield ApprovalRequestContent()
+        yield ApprovalRequestActions()
+
+    def on_mount(self) -> None:
+        self._unsubscribe = human_approval_manager().on_change(
+            self.on_approvals_changed
+        )
+
+    def on_unmount(self) -> None:
+        if self._unsubscribe is not None:
+            self._unsubscribe()
+
+    def on_approvals_changed(self, action: Literal["add", "remove"]) -> None:
+        heading = self.query_one(ApprovalRequestHeading)
+        content = self.query_one(ApprovalRequestContent)
+        actions = self.query_one(ApprovalRequestActions)
+        self._approvals = human_approval_manager().approval_requests()
+        if len(self._approvals) > 0:
+            approval_id, approval_request = self._approvals[0]
+            self.title = f"{PANEL_TITLE} ({len(self._approvals):,})"
+            heading.request = approval_request
+            content.approval = approval_request.request
+            actions.approval_request = approval_id, approval_request
+            if action == "add":
+                self.activate()
+                actions.activate()
+            self.visible = True
+        else:
+            self.title = PANEL_TITLE
+            heading.request = None
+            content.approval = None
+            actions.approval_request = None
+            self.deactivate()
+            self.visible = False
+
+
+class ApprovalRequestHeading(Static):
+    DEFAULT_CSS = """
+    ApprovalRequestHeading {
+        width: 1fr;
+        background: $surface;
+        color: $secondary;
+        margin-left: 1;
+    }
+    """
+
+    request: reactive[PendingApprovalRequest | None] = reactive(None)
+
+    def render(self) -> RenderableType:
+        if self.request is not None:
+            return f"{registry_unqualified_name(self.request.task)} (id: {self.request.id}, epoch {self.request.epoch}): {self.request.model}"
+        else:
+            return ""
+
+
+class ApprovalRequestContent(ScrollableContainer):
+    DEFAULT_CSS = """
+    ApprovalRequestContent {
+        scrollbar-size-vertical: 1;
+        scrollbar-gutter: stable;
+        border: solid $foreground 20%;
+        padding: 0 1 0 1;
+    }
+    """
+
+    approval: reactive[ApprovalRequest | None] = reactive(None)
+
+    async def watch_approval(self, approval: ApprovalRequest | None) -> None:
+        await self.remove_children()
+        if approval:
+            self.mount_all(
+                Static(r) for r in render_tool_approval(approval.message, approval.view)
+            )
+            self.scroll_end(animate=False)
+
+
+class ApprovalRequestActions(Horizontal):
+    APPROVE_TOOL_CALL = "approve-tool-call"
+    REJECT_TOOL_CALL = "reject-tool-call"
+    ESCALATE_TOOL_CALL = "escalate-tool-call"
+    TERMINATE_TOOL_CALL_SAMPLE = "terminate-tool-call-sample"
+
+    DEFAULT_CSS = f"""
+    ApprovalRequestActions Button {{
+        margin-right: 1;
+        min-width: 20;
+    }}
+    ApprovalRequestActions #{APPROVE_TOOL_CALL} {{
+        color: $success;
+    }}
+    ApprovalRequestActions #{REJECT_TOOL_CALL} {{
+        color: $warning-darken-3;
+    }}
+    ApprovalRequestActions #{ESCALATE_TOOL_CALL} {{
+        color: $primary-darken-3;
+        margin-left: 3;
+    }}
+    ApprovalRequestActions #{TERMINATE_TOOL_CALL_SAMPLE} {{
+        color: $error-darken-1;
+        margin-left: 3;
+    }}
+    """
+
+    approval_request: reactive[tuple[str, PendingApprovalRequest] | None] = reactive(
+        None
+    )
+
+    def compose(self) -> ComposeResult:
+        yield Button(
+            Text("Approve"),
+            id=self.APPROVE_TOOL_CALL,
+            tooltip="Approve the tool call.",
+        )
+        yield Button(
+            Text("Reject"),
+            id=self.REJECT_TOOL_CALL,
+            tooltip="Reject the tool call.",
+        )
+        yield Button(
+            Text("Escalate"),
+            id=self.ESCALATE_TOOL_CALL,
+            tooltip="Escalate the tool call to another approver.",
+        )
+        yield Button(
+            Text("Terminate"),
+            id=self.TERMINATE_TOOL_CALL_SAMPLE,
+            tooltip="Terminate the sample.",
+        )
+
+    def activate(self) -> None:
+        approve = self.query_one(f"#{self.APPROVE_TOOL_CALL}")
+        approve.focus()
+
+    def on_button_pressed(self, event: Button.Pressed) -> None:
+        if self.approval_request is not None:
+            id, _ = self.approval_request
+            if event.button.id == self.APPROVE_TOOL_CALL:
+                approval = Approval(decision="approve", explanation=HUMAN_APPROVED)
+            elif event.button.id == self.REJECT_TOOL_CALL:
+                approval = Approval(decision="reject", explanation=HUMAN_REJECTED)
+            elif event.button.id == self.ESCALATE_TOOL_CALL:
+                approval = Approval(decision="escalate", explanation=HUMAN_ESCALATED)
+            elif event.button.id == self.TERMINATE_TOOL_CALL_SAMPLE:
+                approval = Approval(decision="terminate", explanation=HUMAN_TERMINATED)
+            else:
+                raise ValueError(f"Unexpected button id: {event.button.id}")
+            human_approval_manager().complete_approval(id, approval)
+
+    def watch_approval_request(
+        self, approval_request: tuple[str, PendingApprovalRequest] | None
+    ) -> None:
+        choices = (
+            approval_request[1].request.choices if approval_request is not None else []
+        )
+
+        def update_visible(id: str, choice: ApprovalDecision) -> None:
+            self.query_one(f"#{id}").display = choice in choices
+
+        update_visible(self.APPROVE_TOOL_CALL, "approve")
+        update_visible(self.REJECT_TOOL_CALL, "reject")
+        update_visible(self.ESCALATE_TOOL_CALL, "escalate")
+        update_visible(self.TERMINATE_TOOL_CALL_SAMPLE, "terminate")

inspect_ai/approval/_human/util.py

@@ -0,0 +1,51 @@
+from rich.console import RenderableType
+from rich.highlighter import ReprHighlighter
+from rich.rule import Rule
+from rich.text import Text
+
+from inspect_ai._util.transcript import transcript_markdown
+from inspect_ai.tool._tool_call import ToolCallContent, ToolCallView
+from inspect_ai.util._trace import trace_enabled
+
+HUMAN_APPROVED = "Human operator approved tool call."
+HUMAN_REJECTED = "Human operator rejected the tool call."
+HUMAN_TERMINATED = "Human operator asked that the sample be terminated."
+HUMAN_ESCALATED = "Human operator escalated the tool call approval."
+
+
+def render_tool_approval(message: str, view: ToolCallView) -> list[RenderableType]:
+    renderables: list[RenderableType] = []
+    text_highlighter = ReprHighlighter()
+
+    # ignore content if trace enabled
+    message = message.strip() if not trace_enabled() else ""
+
+    def add_view_content(view_content: ToolCallContent) -> None:
+        if view_content.title:
+            renderables.append(Text.from_markup(f"[bold]{view_content.title}[/bold]\n"))
+        if view_content.format == "markdown":
+            renderables.append(transcript_markdown(view_content.content))
+        else:
+            text_content = text_highlighter(Text(view_content.content))
+            renderables.append(text_content)
+
+    # assistant content (don't add if trace_enabled as we already have it in that case)
+    if message:
+        renderables.append(Text.from_markup("[bold]Assistant[/bold]\n"))
+        renderables.append(Text(f"{message.strip()}"))
+
+    # extra context provided by tool view
+    if view.context:
+        renderables.append(Text())
+        add_view_content(view.context)
+        renderables.append(Text())
+
+    # tool call view
+    if view.call:
+        if message or view.context:
+            renderables.append(Rule("", style="#282c34", align="left", characters="․"))
+        renderables.append(Text())
+        add_view_content(view.call)
+        renderables.append(Text())
+
+    return renderables

inspect_ai/dataset/_sources/hf.py

@@ -21,9 +21,9 @@ from .._util import data_to_samples, record_to_sample_fn
 
 def hf_dataset(
     path: str,
+    split: str,
     name: str | None = None,
     data_dir: str | None = None,
-    split: str | None = None,
     revision: str | None = None,
     sample_fields: FieldSpec | RecordToSample | None = None,
     auto_id: bool = False,
@@ -44,10 +44,10 @@ def hf_dataset(
           builder that is used comes from a generic dataset script (JSON, CSV,
           Parquet, text etc.) or from the dataset script (a python file) inside
           the dataset directory.
+        split (str): Which split of the data to load.
         name (str | None): Name of the dataset configuration.
         data_dir (str | None): data_dir of the dataset configuration
           to read data from.
-        split (str | None): Which split of the data to load.
         revision (str | None): Specific revision to load (e.g. "main", a branch
           name, or a specific commit SHA). When using `revision` the `cached` option
           is ignored and datasets are revalidated on Hugging Face before loading.

inspect_ai/dataset/_sources/util.py

@@ -34,7 +34,7 @@ def resolve_sample_files(dataset: Dataset) -> None:
     # for each sample
     for sample in dataset:
         # check for sandbox config file
-        if sample.sandbox and sample.sandbox.config is not None:
+        if sample.sandbox and isinstance(sample.sandbox.config, str):
             sample.sandbox = SandboxEnvironmentSpec(
                 sample.sandbox.type, resolve_file(sample.sandbox.config)
             )

inspect_ai/log/_file.py

@@ -1,17 +1,15 @@
-import asyncio
 import os
 import re
 from logging import getLogger
 from typing import Any, Callable, Generator, Literal, cast
 
-import fsspec  # type: ignore
-from fsspec.asyn import AsyncFileSystem  # type: ignore
-from fsspec.core import split_protocol  # type: ignore
 from pydantic_core import to_json
 
+from inspect_ai._util._async import run_coroutine
 from inspect_ai._util.constants import ALL_LOG_FORMATS, EVAL_LOG_FORMAT
 from inspect_ai._util.file import (
     FileInfo,
+    async_fileystem,
     file,
     filesystem,
 )
@@ -110,25 +108,25 @@ async def list_eval_logs_async(
     # async filesystem if we can
     fs = filesystem(log_dir, fs_options)
     if fs.is_async():
-        async_fs = async_fileystem(log_dir, fs_options=fs_options)
-        if await async_fs._exists(log_dir):
-            # prevent caching of listings
-            async_fs.invalidate_cache(log_dir)
-            # list logs
-            if recursive:
-                files: list[dict[str, Any]] = []
-                async for _, _, filenames in async_fs._walk(log_dir, detail=True):
-                    files.extend(filenames.values())
+        async with async_fileystem(log_dir, fs_options=fs_options) as async_fs:
+            if await async_fs._exists(log_dir):
+                # prevent caching of listings
+                async_fs.invalidate_cache(log_dir)
+                # list logs
+                if recursive:
+                    files: list[dict[str, Any]] = []
+                    async for _, _, filenames in async_fs._walk(log_dir, detail=True):
+                        files.extend(filenames.values())
+                else:
+                    files = cast(
+                        list[dict[str, Any]],
+                        await async_fs._ls(log_dir, detail=True),
+                    )
+                logs = [fs._file_info(file) for file in files]
+                # resolve to eval logs
+                return log_files_from_ls(logs, formats, descending)
             else:
-                files = cast(
-                    list[dict[str, Any]],
-                    async_fs._ls(log_dir, detail=True),
-                )
-            logs = [fs._file_info(file) for file in files]
-            # resolve to eval logs
-            return log_files_from_ls(logs, formats, descending)
-        else:
-            return []
+                return []
     else:
         return list_eval_logs(
             log_dir=log_dir,
@@ -146,6 +144,22 @@ def write_eval_log(
 ) -> None:
     """Write an evaluation log.
 
+    Args:
+       log (EvalLog): Evaluation log to write.
+       location (str | FileInfo): Location to write log to.
+       format (Literal["eval", "json", "auto"]): Write to format
+          (defaults to 'auto' based on `log_file` extension)
+    """
+    run_coroutine(write_eval_log_async(log, location, format))
+
+
+async def write_eval_log_async(
+    log: EvalLog,
+    location: str | FileInfo | None = None,
+    format: Literal["eval", "json", "auto"] = "auto",
+) -> None:
+    """Write an evaluation log.
+
     Args:
        log (EvalLog): Evaluation log to write.
        location (str | FileInfo): Location to write log to.
@@ -169,7 +183,7 @@ def write_eval_log(
         recorder_type = recorder_type_for_location(location)
     else:
         recorder_type = recorder_type_for_format(format)
-    recorder_type.write_log(location, log)
+    await recorder_type.write_log(location, log)
 
     logger.debug(f"Writing eval log to {location} completed")
 
@@ -224,6 +238,31 @@ def read_eval_log(
 ) -> EvalLog:
     """Read an evaluation log.
 
+    Args:
+       log_file (str | FileInfo): Log file to read.
+       header_only (bool): Read only the header (i.e. exclude
+         the "samples" and "logging" fields). Defaults to False.
+       resolve_attachments (bool): Resolve attachments (e.g. images)
+          to their full content.
+       format (Literal["eval", "json", "auto"]): Read from format
+          (defaults to 'auto' based on `log_file` extension)
+
+    Returns:
+       EvalLog object read from file.
+    """
+    return run_coroutine(
+        read_eval_log_async(log_file, header_only, resolve_attachments, format)
+    )
+
+
+async def read_eval_log_async(
+    log_file: str | FileInfo,
+    header_only: bool = False,
+    resolve_attachments: bool = False,
+    format: Literal["eval", "json", "auto"] = "auto",
+) -> EvalLog:
+    """Read an evaluation log.
+
     Args:
        log_file (str | FileInfo): Log file to read.
        header_only (bool): Read only the header (i.e. exclude
@@ -245,7 +284,7 @@ def read_eval_log(
         recorder_type = recorder_type_for_location(log_file)
     else:
         recorder_type = recorder_type_for_format(format)
-    log = recorder_type.read_log(log_file, header_only)
+    log = await recorder_type.read_log(log_file, header_only)
 
     # resolve attachement if requested
     if resolve_attachments and log.samples:
@@ -267,7 +306,15 @@ def read_eval_log(
 def read_eval_log_headers(
     log_files: list[str] | list[FileInfo] | list[EvalLogInfo],
 ) -> list[EvalLog]:
-    return [read_eval_log(log_file, header_only=True) for log_file in log_files]
+    return run_coroutine(read_eval_log_headers_async(log_files))
+
+
+async def read_eval_log_headers_async(
+    log_files: list[str] | list[FileInfo] | list[EvalLogInfo],
+) -> list[EvalLog]:
+    return [
+        await read_eval_log_async(log_file, header_only=True) for log_file in log_files
+    ]
 
 
 def read_eval_log_sample(
@@ -279,6 +326,35 @@ def read_eval_log_sample(
 ) -> EvalSample:
     """Read a sample from an evaluation log.
 
+    Args:
+       log_file (str | FileInfo): Log file to read.
+       id (int | str): Sample id to read.
+       epoch (int): Epoch for sample id (defaults to 1)
+       resolve_attachments (bool): Resolve attachments (e.g. images)
+          to their full content.
+       format (Literal["eval", "json", "auto"]): Read from format
+          (defaults to 'auto' based on `log_file` extension)
+
+    Returns:
+       EvalSample object read from file.
+
+    Raises:
+       IndexError: If the passed id and epoch are not found.
+    """
+    return run_coroutine(
+        read_eval_log_sample_async(log_file, id, epoch, resolve_attachments, format)
+    )
+
+
+async def read_eval_log_sample_async(
+    log_file: str | FileInfo,
+    id: int | str,
+    epoch: int = 1,
+    resolve_attachments: bool = False,
+    format: Literal["eval", "json", "auto"] = "auto",
+) -> EvalSample:
+    """Read a sample from an evaluation log.
+
     Args:
        log_file (str | FileInfo): Log file to read.
        id (int | str): Sample id to read.
@@ -301,7 +377,7 @@ def read_eval_log_sample(
         recorder_type = recorder_type_for_location(log_file)
     else:
         recorder_type = recorder_type_for_format(format)
-    sample = recorder_type.read_log_sample(log_file, id, epoch)
+    sample = await recorder_type.read_log_sample(log_file, id, epoch)
 
     if resolve_attachments:
         sample = resolve_sample_attachments(sample)
@@ -442,7 +518,7 @@ def log_file_info(info: FileInfo) -> "EvalLogInfo":
     )
 
 
-def eval_log_json(log: EvalLog) -> str:
+def eval_log_json(log: EvalLog) -> bytes:
     # serialize to json (ignore values that are unserializable)
     # these values often result from solvers using metadata to
     # pass around 'live' objects -- this is fine to do and we
@@ -452,14 +528,8 @@ def eval_log_json(log: EvalLog) -> str:
         indent=2,
         exclude_none=True,
         fallback=lambda _x: None,
-    ).decode()
+    )
 
 
-def async_fileystem(log_file: str, fs_options: dict[str, Any] = {}) -> AsyncFileSystem:
-    # determine protocol
-    protocol, _ = split_protocol(log_file)
-    protocol = protocol or "file"
-    # create filesystem
-    fs_options = fs_options.copy()
-    fs_options.update({"asynchronous": True, "loop": asyncio.get_event_loop()})
-    return fsspec.filesystem(protocol, **fs_options)
+def eval_log_json_str(log: EvalLog) -> str:
+    return eval_log_json(log).decode()