inspect-ai 0.3.48__py3-none-any.whl → 0.3.50__py3-none-any.whl

inspect_ai/approval/_human/console.py

@@ -0,0 +1,62 @@
+from rich.prompt import Prompt
+
+from inspect_ai._util.transcript import transcript_panel
+from inspect_ai.tool._tool_call import ToolCallView
+from inspect_ai.util._console import input_screen
+
+from .._approval import Approval, ApprovalDecision
+from .util import (
+    HUMAN_APPROVED,
+    HUMAN_ESCALATED,
+    HUMAN_REJECTED,
+    HUMAN_TERMINATED,
+    render_tool_approval,
+)
+
+
+def console_approval(
+    message: str, view: ToolCallView, choices: list[ApprovalDecision]
+) -> Approval:
+    with input_screen(width=None) as console:
+        console.print(
+            transcript_panel(
+                title="Approve Tool", content=render_tool_approval(message, view)
+            )
+        )
+
+        # provide choices
+        prompts: dict[str, str] = {}
+        for choice in choices:
+            prompts[choice[0]] = f"{choice.capitalize()} ({choice[0]})"
+        values = list(prompts.values())
+        prompt = ", ".join(values[:-1])
+        prompt = f"{prompt}, or {values[-1]}"
+
+        def render_approval(approval: Approval) -> Approval:
+            console.print(f"Decision: {approval.decision.capitalize()}")
+            return approval
+
+        while True:
+            decision = Prompt.ask(
+                prompt=prompt,
+                console=console,
+                choices=list(prompts.keys()),
+                default="a",
+            ).lower()
+
+            if decision == "a":
+                return render_approval(
+                    Approval(decision="approve", explanation=HUMAN_APPROVED)
+                )
+            elif decision == "r":
+                return render_approval(
+                    Approval(decision="reject", explanation=HUMAN_REJECTED)
+                )
+            elif decision == "t":
+                return render_approval(
+                    Approval(decision="terminate", explanation=HUMAN_TERMINATED)
+                )
+            elif decision == "e":
+                return render_approval(
+                    Approval(decision="escalate", explanation=HUMAN_ESCALATED)
+                )

inspect_ai/approval/_human/manager.py

@@ -0,0 +1,108 @@
+import asyncio
+import uuid
+from asyncio import Future
+from contextvars import ContextVar
+from typing import Callable, Literal, NamedTuple, cast
+
+from inspect_ai.solver._task_state import TaskState
+from inspect_ai.tool._tool_call import ToolCall, ToolCallView
+
+from .._approval import Approval, ApprovalDecision
+
+
+class ApprovalRequest(NamedTuple):
+    message: str
+    call: ToolCall
+    view: ToolCallView
+    state: TaskState | None
+    choices: list[ApprovalDecision]
+
+
+class PendingApprovalRequest(NamedTuple):
+    request: ApprovalRequest
+    task: str
+    model: str
+    id: int | str
+    epoch: int
+
+
+class HumanApprovalManager:
+    def __init__(self) -> None:
+        self._approval_requests: dict[
+            str, tuple[PendingApprovalRequest, Future[Approval]]
+        ] = {}
+        self._change_callbacks: list[Callable[[Literal["add", "remove"]], None]] = []
+
+    def request_approval(self, request: ApprovalRequest) -> str:
+        from inspect_ai.log._samples import sample_active
+
+        id = str(uuid.uuid4())
+        future = cast(Future[Approval], asyncio.get_event_loop().create_future())
+        sample = sample_active()
+        assert sample
+        assert sample.sample.id
+        pending = PendingApprovalRequest(
+            request=request,
+            task=sample.task,
+            model=sample.model,
+            id=sample.sample.id,
+            epoch=sample.epoch,
+        )
+        self._approval_requests[id] = (pending, future)
+        self._notify_change("add")
+        return id
+
+    def withdraw_request(self, id: str) -> None:
+        del self._approval_requests[id]
+        self._notify_change("remove")
+
+    async def wait_for_approval(self, id: str) -> Approval:
+        _, future = self._approval_requests[id]
+        return await future
+
+    def on_change(
+        self, callback: Callable[[Literal["add", "remove"]], None]
+    ) -> Callable[[], None]:
+        self._change_callbacks.append(callback)
+
+        def unsubscribe() -> None:
+            if callback in self._change_callbacks:
+                self._change_callbacks.remove(callback)
+
+        return unsubscribe
+
+    def approval_requests(self) -> list[tuple[str, PendingApprovalRequest]]:
+        return [(aid, data) for aid, (data, _) in self._approval_requests.items()]
+
+    def complete_approval(self, id: str, result: Approval) -> None:
+        if id in self._approval_requests:
+            _, future = self._approval_requests[id]
+            if not future.done():
+                future.set_result(result)
+            del self._approval_requests[id]
+            self._notify_change("remove")
+
+    def fail_approval(self, id: str, error: Exception) -> None:
+        if id in self._approval_requests:
+            _, future = self._approval_requests[id]
+            if not future.done():
+                future.set_exception(error)
+            del self._approval_requests[id]
+            self._notify_change("remove")
+
+    def _notify_change(self, action: Literal["add", "remove"]) -> None:
+        for callback in self._change_callbacks:
+            callback(action)
+
+
+def human_approval_manager() -> HumanApprovalManager:
+    return _human_approval_manager.get()
+
+
+def init_human_approval_manager() -> None:
+    _human_approval_manager.set(HumanApprovalManager())
+
+
+_human_approval_manager: ContextVar[HumanApprovalManager] = ContextVar(
+    "_human_approval_manager"
+)

inspect_ai/approval/_human/panel.py

@@ -0,0 +1,233 @@
+from asyncio import CancelledError
+from typing import Callable, Literal
+
+from rich.console import RenderableType
+from rich.text import Text
+from textual.app import ComposeResult
+from textual.containers import Horizontal, ScrollableContainer
+from textual.reactive import reactive
+from textual.widgets import Button, Static
+from typing_extensions import override
+
+from inspect_ai._util.registry import registry_unqualified_name
+from inspect_ai.solver._task_state import TaskState
+from inspect_ai.tool._tool_call import ToolCall, ToolCallView
+from inspect_ai.util._panel import InputPanel, input_panel
+
+from .._approval import Approval, ApprovalDecision
+from .manager import ApprovalRequest, PendingApprovalRequest, human_approval_manager
+from .util import (
+    HUMAN_APPROVED,
+    HUMAN_ESCALATED,
+    HUMAN_REJECTED,
+    HUMAN_TERMINATED,
+    render_tool_approval,
+)
+
+PANEL_TITLE = "Approvals"
+
+
+async def panel_approval(
+    message: str,
+    call: ToolCall,
+    view: ToolCallView,
+    state: TaskState | None,
+    choices: list[ApprovalDecision],
+) -> Approval:
+    # ensure the approvals panel is shown
+    await input_panel(PANEL_TITLE, ApprovalInputPanel)
+
+    # submit to human approval manager (will be picked up by panel)
+    approvals = human_approval_manager()
+    id = approvals.request_approval(
+        ApprovalRequest(
+            message=message, call=call, view=view, state=state, choices=choices
+        )
+    )
+    try:
+        return await approvals.wait_for_approval(id)
+    except CancelledError:
+        approvals.withdraw_request(id)
+        raise
+
+
+class ApprovalInputPanel(InputPanel):
+    DEFAULT_CSS = """
+    ApprovalInputPanel {
+        width: 1fr;
+        height: 1fr;
+        padding: 0 1 1 1;
+        layout: grid;
+        grid-size: 1 3;
+        grid-rows: auto 1fr auto;
+    }
+    """
+
+    _approvals: list[tuple[str, PendingApprovalRequest]] = []
+    _unsubscribe: Callable[[], None] | None = None
+
+    @override
+    def compose(self) -> ComposeResult:
+        yield ApprovalRequestHeading()
+        yield ApprovalRequestContent()
+        yield ApprovalRequestActions()
+
+    def on_mount(self) -> None:
+        self._unsubscribe = human_approval_manager().on_change(
+            self.on_approvals_changed
+        )
+
+    def on_unmount(self) -> None:
+        if self._unsubscribe is not None:
+            self._unsubscribe()
+
+    def on_approvals_changed(self, action: Literal["add", "remove"]) -> None:
+        heading = self.query_one(ApprovalRequestHeading)
+        content = self.query_one(ApprovalRequestContent)
+        actions = self.query_one(ApprovalRequestActions)
+        self._approvals = human_approval_manager().approval_requests()
+        if len(self._approvals) > 0:
+            approval_id, approval_request = self._approvals[0]
+            self.title = f"{PANEL_TITLE} ({len(self._approvals):,})"
+            heading.request = approval_request
+            content.approval = approval_request.request
+            actions.approval_request = approval_id, approval_request
+            if action == "add":
+                self.activate()
+                actions.activate()
+            self.visible = True
+        else:
+            self.title = PANEL_TITLE
+            heading.request = None
+            content.approval = None
+            actions.approval_request = None
+            self.deactivate()
+            self.visible = False
+
+
+class ApprovalRequestHeading(Static):
+    DEFAULT_CSS = """
+    ApprovalRequestHeading {
+        width: 1fr;
+        background: $surface;
+        color: $secondary;
+        margin-left: 1;
+    }
+    """
+
+    request: reactive[PendingApprovalRequest | None] = reactive(None)
+
+    def render(self) -> RenderableType:
+        if self.request is not None:
+            return f"{registry_unqualified_name(self.request.task)} (id: {self.request.id}, epoch {self.request.epoch}): {self.request.model}"
+        else:
+            return ""
+
+
+class ApprovalRequestContent(ScrollableContainer):
+    DEFAULT_CSS = """
+    ApprovalRequestContent {
+        scrollbar-size-vertical: 1;
+        scrollbar-gutter: stable;
+        border: solid $foreground 20%;
+        padding: 0 1 0 1;
+    }
+    """
+
+    approval: reactive[ApprovalRequest | None] = reactive(None)
+
+    async def watch_approval(self, approval: ApprovalRequest | None) -> None:
+        await self.remove_children()
+        if approval:
+            self.mount_all(
+                Static(r) for r in render_tool_approval(approval.message, approval.view)
+            )
+            self.scroll_end(animate=False)
+
+
+class ApprovalRequestActions(Horizontal):
+    APPROVE_TOOL_CALL = "approve-tool-call"
+    REJECT_TOOL_CALL = "reject-tool-call"
+    ESCALATE_TOOL_CALL = "escalate-tool-call"
+    TERMINATE_TOOL_CALL_SAMPLE = "terminate-tool-call-sample"
+
+    DEFAULT_CSS = f"""
+    ApprovalRequestActions Button {{
+        margin-right: 1;
+        min-width: 20;
+    }}
+    ApprovalRequestActions #{APPROVE_TOOL_CALL} {{
+        color: $success;
+    }}
+    ApprovalRequestActions #{REJECT_TOOL_CALL} {{
+        color: $warning-darken-3;
+    }}
+    ApprovalRequestActions #{ESCALATE_TOOL_CALL} {{
+        color: $primary-darken-3;
+        margin-left: 3;
+    }}
+    ApprovalRequestActions #{TERMINATE_TOOL_CALL_SAMPLE} {{
+        color: $error-darken-1;
+        margin-left: 3;
+    }}
+    """
+
+    approval_request: reactive[tuple[str, PendingApprovalRequest] | None] = reactive(
+        None
+    )
+
+    def compose(self) -> ComposeResult:
+        yield Button(
+            Text("Approve"),
+            id=self.APPROVE_TOOL_CALL,
+            tooltip="Approve the tool call.",
+        )
+        yield Button(
+            Text("Reject"),
+            id=self.REJECT_TOOL_CALL,
+            tooltip="Reject the tool call.",
+        )
+        yield Button(
+            Text("Escalate"),
+            id=self.ESCALATE_TOOL_CALL,
+            tooltip="Escalate the tool call to another approver.",
+        )
+        yield Button(
+            Text("Terminate"),
+            id=self.TERMINATE_TOOL_CALL_SAMPLE,
+            tooltip="Terminate the sample.",
+        )
+
+    def activate(self) -> None:
+        approve = self.query_one(f"#{self.APPROVE_TOOL_CALL}")
+        approve.focus()
+
+    def on_button_pressed(self, event: Button.Pressed) -> None:
+        if self.approval_request is not None:
+            id, _ = self.approval_request
+            if event.button.id == self.APPROVE_TOOL_CALL:
+                approval = Approval(decision="approve", explanation=HUMAN_APPROVED)
+            elif event.button.id == self.REJECT_TOOL_CALL:
+                approval = Approval(decision="reject", explanation=HUMAN_REJECTED)
+            elif event.button.id == self.ESCALATE_TOOL_CALL:
+                approval = Approval(decision="escalate", explanation=HUMAN_ESCALATED)
+            elif event.button.id == self.TERMINATE_TOOL_CALL_SAMPLE:
+                approval = Approval(decision="terminate", explanation=HUMAN_TERMINATED)
+            else:
+                raise ValueError(f"Unexpected button id: {event.button.id}")
+            human_approval_manager().complete_approval(id, approval)
+
+    def watch_approval_request(
+        self, approval_request: tuple[str, PendingApprovalRequest] | None
+    ) -> None:
+        choices = (
+            approval_request[1].request.choices if approval_request is not None else []
+        )
+
+        def update_visible(id: str, choice: ApprovalDecision) -> None:
+            self.query_one(f"#{id}").display = choice in choices
+
+        update_visible(self.APPROVE_TOOL_CALL, "approve")
+        update_visible(self.REJECT_TOOL_CALL, "reject")
+        update_visible(self.ESCALATE_TOOL_CALL, "escalate")
+        update_visible(self.TERMINATE_TOOL_CALL_SAMPLE, "terminate")

inspect_ai/approval/_human/util.py

@@ -0,0 +1,51 @@
+from rich.console import RenderableType
+from rich.highlighter import ReprHighlighter
+from rich.rule import Rule
+from rich.text import Text
+
+from inspect_ai._util.transcript import transcript_markdown
+from inspect_ai.tool._tool_call import ToolCallContent, ToolCallView
+from inspect_ai.util._trace import trace_enabled
+
+HUMAN_APPROVED = "Human operator approved tool call."
+HUMAN_REJECTED = "Human operator rejected the tool call."
+HUMAN_TERMINATED = "Human operator asked that the sample be terminated."
+HUMAN_ESCALATED = "Human operator escalated the tool call approval."
+
+
+def render_tool_approval(message: str, view: ToolCallView) -> list[RenderableType]:
+    renderables: list[RenderableType] = []
+    text_highlighter = ReprHighlighter()
+
+    # ignore content if trace enabled
+    message = message.strip() if not trace_enabled() else ""
+
+    def add_view_content(view_content: ToolCallContent) -> None:
+        if view_content.title:
+            renderables.append(Text.from_markup(f"[bold]{view_content.title}[/bold]\n"))
+        if view_content.format == "markdown":
+            renderables.append(transcript_markdown(view_content.content))
+        else:
+            text_content = text_highlighter(Text(view_content.content))
+            renderables.append(text_content)
+
+    # assistant content (don't add if trace_enabled as we already have it in that case)
+    if message:
+        renderables.append(Text.from_markup("[bold]Assistant[/bold]\n"))
+        renderables.append(Text(f"{message.strip()}"))
+
+    # extra context provided by tool view
+    if view.context:
+        renderables.append(Text())
+        add_view_content(view.context)
+        renderables.append(Text())
+
+    # tool call view
+    if view.call:
+        if message or view.context:
+            renderables.append(Rule("", style="#282c34", align="left", characters="․"))
+        renderables.append(Text())
+        add_view_content(view.call)
+        renderables.append(Text())
+
+    return renderables

inspect_ai/dataset/_sources/hf.py

@@ -21,9 +21,9 @@ from .._util import data_to_samples, record_to_sample_fn
 
 def hf_dataset(
     path: str,
+    split: str,
     name: str | None = None,
     data_dir: str | None = None,
-    split: str | None = None,
     revision: str | None = None,
     sample_fields: FieldSpec | RecordToSample | None = None,
     auto_id: bool = False,
@@ -44,10 +44,10 @@ def hf_dataset(
           builder that is used comes from a generic dataset script (JSON, CSV,
           Parquet, text etc.) or from the dataset script (a python file) inside
           the dataset directory.
+        split (str): Which split of the data to load.
         name (str | None): Name of the dataset configuration.
         data_dir (str | None): data_dir of the dataset configuration
           to read data from.
-        split (str | None): Which split of the data to load.
         revision (str | None): Specific revision to load (e.g. "main", a branch
           name, or a specific commit SHA). When using `revision` the `cached` option
           is ignored and datasets are revalidated on Hugging Face before loading.

inspect_ai/dataset/_sources/util.py

@@ -34,7 +34,7 @@ def resolve_sample_files(dataset: Dataset) -> None:
     # for each sample
     for sample in dataset:
         # check for sandbox config file
-        if sample.sandbox and sample.sandbox.config is not None:
+        if sample.sandbox and isinstance(sample.sandbox.config, str):
             sample.sandbox = SandboxEnvironmentSpec(
                 sample.sandbox.type, resolve_file(sample.sandbox.config)
             )