inspect-ai 0.3.55__py3-none-any.whl → 0.3.56__py3-none-any.whl

inspect_ai/solver/_human_agent/install.py

@@ -0,0 +1,222 @@
+import inspect
+from textwrap import dedent
+
+from inspect_ai.util import sandbox
+
+from .._task_state import TaskState
+from .commands.command import HumanAgentCommand
+
+INSTALL_DIR = "human_agent_install"
+HUMAN_AGENT_DIR = "/opt/human_agent"
+TASK_PY = "task.py"
+INSTALL_SH = "install.sh"
+BASHRC = ".bashrc"
+WELCOME_FILE = "welcome.txt"
+WELCOME_LOGIN_FILE = "welcome_login.txt"
+INSTRUCTIONS_FILE = "instructions.txt"
+RECORD_SESSION_DIR = "/var/tmp/user-sessions"
+
+
+async def install_human_agent(
+    state: TaskState, commands: list[HumanAgentCommand], record_session: bool
+) -> None:
+    # see if we have already installed
+    if not (await sandbox().exec(["mkdir", HUMAN_AGENT_DIR])).success:
+        return
+
+    # setup installation directory
+    await checked_exec(["mkdir", "-p", INSTALL_DIR])
+
+    # generate task.py
+    task_py = human_agent_commands(commands)
+    await checked_write_file(f"{INSTALL_DIR}/{TASK_PY}", task_py, executable=True)
+
+    # generate .bashrc
+    bash_rc = human_agent_bashrc(commands, record_session)
+    await checked_write_file(f"{INSTALL_DIR}/{BASHRC}", bash_rc, executable=True)
+
+    # write and run installation script
+    install_sh = human_agent_install_sh()
+    await checked_write_file(f"{INSTALL_DIR}/{INSTALL_SH}", install_sh, executable=True)
+    await checked_exec(["bash", f"./{INSTALL_SH}"], cwd=INSTALL_DIR)
+    await checked_exec(["rm", "-rf", INSTALL_DIR])
+
+
+def human_agent_commands(commands: list[HumanAgentCommand]) -> str:
+    # filter out hidden commands
+    commands = [command for command in commands if "cli" in command.contexts]
+
+    # standard imports (including any dependencies that call methods carry)
+    imports = dedent("""
+    import argparse
+    import sys
+    from argparse import Namespace
+    from pathlib import Path
+
+    sys.path.append("/var/tmp/sandbox-services/human_agent")
+    from human_agent import call_human_agent
+
+    def format_time(t):
+        minutes, seconds = divmod(t, 60)
+        hours, minutes = divmod(minutes, 60)
+        return f"{hours:.0f}:{minutes:02.0f}:{seconds:02.0f}"
+    """)
+
+    # command handler source code (extracted from call methods)
+    command_handlers = "\n\n".join(
+        dedent(
+            inspect.getsource(command.cli).replace("cli(self, ", f"{command.name}(", 1)
+        )
+        for command in commands
+    )
+
+    # parse commands
+    command_parsers: list[str] = []
+    for command in commands:
+        command_parsers.append(
+            dedent(f"""
+        {command.name}_parser = subparsers.add_parser("{command.name}", help="{command.description}")
+        """).lstrip()
+        )
+        for arg in command.cli_args:
+            if arg.name.startswith("--"):
+                extras = 'action="store_true", default=False'
+            else:
+                extras = f"""nargs={1 if arg.required else '"?"'}"""
+            command_parsers.append(
+                dedent(f"""
+                {command.name}_parser.add_argument("{arg.name}", {extras}, help="{arg.description}")
+                """).strip()
+            )
+
+    parse = (
+        dedent("""
+    parser = argparse.ArgumentParser(description="Human agent tools.")
+    subparsers = parser.add_subparsers(dest="command")
+    """)
+        + "\n"
+        + "\n".join(command_parsers)
+    )
+
+    # dispatch commands
+    command_dispatchers: list[str] = []
+    for i, command in enumerate(commands):
+        conditional = "if" if i == 0 else "elif"
+        command_dispatchers.append(
+            f'{conditional} command == "{command.name}": {command.name}(args)'
+        )
+    command_dispatchers.append("else: parser.print_help()")
+
+    dispatch = dedent("""
+    args = parser.parse_args()
+    command = args.command
+    delattr(args, 'command')
+    """) + "\n".join(command_dispatchers)
+
+    return "\n".join([imports, command_handlers, parse, dispatch]) + "\n"
+
+
+def human_agent_bashrc(commands: list[HumanAgentCommand], record_session: bool) -> str:
+    # only run in interative terminals
+    TERMINAL_CHECK = dedent("""
+
+    ### Inspect Human Agent Setup #########################################=
+
+    # only run if shell is interactive
+    case $- in
+        *i*) ;;
+        *) return ;;
+    esac
+
+    # only run if attached to a terminal
+    if ! tty -s; then
+        return
+    fi
+    """)
+
+    # shell alias and completions
+    command_names = " ".join(
+        [f"{command.name}" for command in commands if "cli" in command.contexts]
+    )
+    COMMANDS = dedent(f"""
+    # shell alias for human agent commands
+    alias task='python3 {HUMAN_AGENT_DIR}/{TASK_PY}'
+
+    # completion handler
+    _task_completion() {{
+        local cur
+        cur="${{COMP_WORDS[COMP_CWORD]}}"
+        if [ "$COMP_CWORD" -eq 1 ]; then
+            local commands="{command_names}"
+
+            # Generate completion matches
+            COMPREPLY=($(compgen -W "${{commands}}" -- ${{cur}}))
+        fi
+    }}
+    complete -F _task_completion task
+    """)
+
+    # session recording
+    if record_session:
+        RECORDING = dedent(f"""
+        # record human agent session transcript
+        if [ -z "$SCRIPT_RUNNING" ]; then
+            export SCRIPT_RUNNING=1
+            LOGDIR={RECORD_SESSION_DIR}
+            mkdir -p "$LOGDIR"
+            TIMESTAMP=$(date +%Y%m%d_%H%M%S)
+            INPUTFILE="$LOGDIR/$(whoami)_$TIMESTAMP.input"
+            OUTPUTFILE="$LOGDIR/$(whoami)_$TIMESTAMP.output"
+            TIMINGFILE="$LOGDIR/$(whoami)_$TIMESTAMP.timing"
+            exec script -q -f -m advanced -I "$INPUTFILE" -O "$OUTPUTFILE" -T "$TIMINGFILE" -c "bash --login -i"
+        fi
+        """)
+    else:
+        RECORDING = ""
+
+    # display task instructions
+    INSTRUCTIONS = dedent("""
+    if [ -z "$INSTRUCTIONS_SHOWN" ]; then
+        export INSTRUCTIONS_SHOWN=1
+        task instructions > instructions.txt
+        cat instructions.txt
+    fi
+    """).lstrip()
+
+    # return .bashrc
+    return "\n".join([TERMINAL_CHECK, COMMANDS, RECORDING, INSTRUCTIONS])
+
+
+def human_agent_install_sh() -> str:
+    return dedent(f"""
+    #!/usr/bin/env bash
+
+    # create installation directory
+    HUMAN_AGENT="{HUMAN_AGENT_DIR}"
+    mkdir -p $HUMAN_AGENT
+
+    # copy command script
+    cp {TASK_PY} $HUMAN_AGENT
+
+    # append to .bashrc
+    cat {BASHRC} >> ~/{BASHRC}
+    """)
+
+
+async def checked_exec(
+    cmd: list[str],
+    input: str | bytes | None = None,
+    cwd: str | None = None,
+) -> str:
+    result = await sandbox().exec(cmd, input=input, cwd=cwd)
+    if not result.success:
+        raise RuntimeError(f"Error executing command {' '.join(cmd)}: {result.stderr}")
+    return result.stdout
+
+
+async def checked_write_file(
+    file: str, contents: str, executable: bool = False
+) -> None:
+    await checked_exec(["tee", "--", file], input=contents)
+    if executable:
+        await checked_exec(["chmod", "+x", file])

inspect_ai/solver/_human_agent/panel.py

@@ -0,0 +1,252 @@
+from typing import cast
+
+from textual.app import ComposeResult
+from textual.containers import (
+    Container,
+    Horizontal,
+    VerticalScroll,
+)
+from textual.reactive import reactive
+from textual.widgets import (
+    Button,
+    ContentSwitcher,
+    Label,
+    Link,
+    LoadingIndicator,
+    Static,
+)
+
+from inspect_ai._util.format import format_progress_time
+from inspect_ai._util.vscode import (
+    VSCodeCommand,
+    can_execute_vscode_commands,
+    execute_vscode_commands,
+)
+from inspect_ai.util import InputPanel, SandboxConnection, throttle
+
+from .state import HumanAgentState
+
+
+class HumanAgentPanel(InputPanel):
+    DEFAULT_TITLE = "Human Agent"
+
+    SANDBOX_VIEW_ID = "human-agent-sandbox-view"
+    SANDBOX_INSTRUCTIONS_ID = "sandbox-instructions"
+    VSCODE_LINKS_ID = "vscode-links"
+    LOGIN_VSCODE_TERMINAL_ID = "login-vscode-terminal"
+    LOGIN_VSCODE_WINDOW_ID = "login-vscode-window"
+    COMMAND_INSTRUCTIONS_ID = "command-instructions"
+    SANDBOX_COMMAND_ID = "sandbox-command"
+
+    INSTRUCTIONS_CLASS = "instructions"
+    LINK_LABEL_CLASS = "link-label"
+
+    DEFAULT_CSS = f"""
+    #{SANDBOX_VIEW_ID} {{
+        scrollbar-size-vertical: 1;
+    }}
+    HumanAgentPanel .{INSTRUCTIONS_CLASS} {{
+        color: $text-muted;
+        margin-bottom: 1;
+    }}
+    #{SANDBOX_COMMAND_ID} {{
+        color: $secondary;
+    }}
+    HumanAgentPanel .{LINK_LABEL_CLASS} {{
+        color: $text-muted;
+    }}
+    HumanAgentPanel VSCodeLink {{
+        margin-left: 1;
+        margin-right: 2;
+    }}
+    HumanAgentPanel #{VSCODE_LINKS_ID} {{
+        height: 1;
+        margin-bottom: 1;
+    }}
+    """
+
+    connection: reactive[SandboxConnection | None] = reactive(None)
+
+    # implement HumanAgentView
+    def connect(self, connection: SandboxConnection) -> None:
+        self.connection = connection
+
+    @throttle(1)
+    def update_state(self, state: HumanAgentState) -> None:
+        status_bar = self.query_one(StatusBar)
+        status_bar.running = state.running
+        status_bar.time = state.time
+
+    def compose(self) -> ComposeResult:
+        with ContentSwitcher(initial=LoadingView.ID):
+            yield LoadingView()
+            with VerticalScroll(id=self.SANDBOX_VIEW_ID):
+                yield StatusBar()
+                yield Static(
+                    id=self.SANDBOX_INSTRUCTIONS_ID, classes=self.INSTRUCTIONS_CLASS
+                )
+                with Horizontal(id=self.VSCODE_LINKS_ID):
+                    yield Label("Login:", classes=self.LINK_LABEL_CLASS)
+                    yield VSCodeLink(
+                        "VS Code Window",
+                        id=self.LOGIN_VSCODE_WINDOW_ID,
+                    )
+                    yield Label("Login:", classes=self.LINK_LABEL_CLASS)
+                    yield VSCodeLink(
+                        "VS Code Terminal",
+                        id=self.LOGIN_VSCODE_TERMINAL_ID,
+                    )
+                yield Static(
+                    id=self.COMMAND_INSTRUCTIONS_ID, classes=self.INSTRUCTIONS_CLASS
+                )
+                yield Static(id=self.SANDBOX_COMMAND_ID)
+
+    def watch_connection(self, connection: SandboxConnection | None) -> None:
+        if connection:
+            # switch to sandbox view
+            self.query_one(ContentSwitcher).current = self.SANDBOX_VIEW_ID
+
+            # note whether we are in vscode
+            vscode = can_execute_vscode_commands()
+
+            # suffix for instructions based on whether we are in vscode
+            instructions_command = "Login to the system with the following command (hold down Alt or Option to select text for copying):"
+            instructions_vscode = (
+                "Use the links below to login to the system within VS Code:"
+            )
+
+            # update instructions
+            instructions_lbl = cast(
+                Static, self.query_one(f"#{self.SANDBOX_INSTRUCTIONS_ID}")
+            )
+            instructions = f"""You are completing a task on a Linux system (task instructions will be presented when you login). {instructions_vscode if vscode else instructions_command}"""
+            instructions_lbl.update(instructions)
+
+            # login: vscode terminal
+            vscode_links = self.query_one(f"#{self.VSCODE_LINKS_ID}")
+            vscode_links.display = vscode
+            terminal_btn = cast(
+                VSCodeLink, self.query_one(f"#{self.LOGIN_VSCODE_TERMINAL_ID}")
+            )
+            terminal_btn.commands = [
+                VSCodeCommand(
+                    command="workbench.action.terminal.new", args=[{"location": 2}]
+                ),
+                VSCodeCommand(
+                    command="workbench.action.terminal.sendSequence",
+                    args=[{"text": f"{connection.command}\n"}],
+                ),
+            ]
+
+            # login: vscode window
+            window_btn = cast(
+                VSCodeLink, self.query_one(f"#{self.LOGIN_VSCODE_WINDOW_ID}")
+            )
+            if connection.vscode_command is not None:
+                window_btn.commands = [
+                    VSCodeCommand(
+                        command=connection.vscode_command[0],
+                        args=connection.vscode_command[1:],
+                    )
+                ]
+
+            # command (always available)
+            command_instructions_lbl = cast(
+                Static, self.query_one(f"#{self.COMMAND_INSTRUCTIONS_ID}")
+            )
+            command_instructions_lbl.display = vscode
+            command_instructions_lbl.update(
+                instructions_command.replace("Login", "Alternatively, login", 1)
+            )
+            command_lbl = cast(Static, self.query_one(f"#{self.SANDBOX_COMMAND_ID}"))
+            command_lbl.update(connection.command)
+
+
+class StatusBar(Horizontal):
+    STATUS_ID = "task-status"
+    TIME_ID = "task-time"
+
+    LABEL_CLASS = "status-label"
+    VALUE_CLASS = "status-value"
+
+    DEFAULT_CSS = f"""
+    StatusBar {{
+        width: 1fr;
+        height: 1;
+        background: $surface;
+        margin-bottom: 1;
+        layout: grid;
+        grid-size: 4 1;
+        grid-columns: auto auto auto auto;
+        grid-gutter: 1;
+    }}
+    .{LABEL_CLASS} {{
+        color: $primary;
+    }}
+    .{VALUE_CLASS} {{
+        color: $foreground;
+    }}
+    StatusBar Link {{
+        dock: right;
+        margin-right: 1;
+    }}
+    """
+
+    running: reactive[bool] = reactive(True)
+    time: reactive[float] = reactive(0)
+
+    def __init__(self) -> None:
+        super().__init__()
+
+    def compose(self) -> ComposeResult:
+        yield Label("Status:", classes=self.LABEL_CLASS)
+        yield Static("Running", id=self.STATUS_ID, classes=self.VALUE_CLASS)
+        yield Label(" Time:", classes=self.LABEL_CLASS)
+        yield Static("0:00:00", id=self.TIME_ID, classes=self.VALUE_CLASS)
+
+    def watch_running(self, running: bool) -> None:
+        cast(Static, self.query_one(f"#{self.STATUS_ID}")).update(
+            "Running" if running else "Stopped"
+        )
+
+    def watch_time(self, time: float) -> None:
+        time_display = format_progress_time(time)
+        cast(Static, self.query_one(f"#{self.TIME_ID}")).update(time_display)
+
+
+class LoadingView(Container):
+    ID = "human-agent-loading-view"
+
+    def __init__(self) -> None:
+        super().__init__(id=self.ID)
+
+    def compose(self) -> ComposeResult:
+        yield LoadingIndicator()
+        yield Button()  # add focusable widget so the tab can activate
+
+
+class VSCodeLink(Link):
+    def __init__(
+        self,
+        text: str,
+        *,
+        url: str | None = None,
+        tooltip: str | None = None,
+        name: str | None = None,
+        id: str | None = None,
+        classes: str | None = None,
+        disabled: bool = False,
+    ) -> None:
+        super().__init__(
+            text,
+            url=url,
+            tooltip=tooltip,
+            name=name,
+            id=id,
+            classes=classes,
+            disabled=disabled,
+        )
+        self.commands: list[VSCodeCommand] = []
+
+    def on_click(self) -> None:
+        execute_vscode_commands(self.commands)

inspect_ai/solver/_human_agent/service.py

@@ -0,0 +1,45 @@
+from inspect_ai.model import ModelOutput
+from inspect_ai.util._sandbox import sandbox
+from inspect_ai.util._sandbox.service import sandbox_service
+
+from .._task_state import TaskState
+from .commands.command import HumanAgentCommand
+from .state import HumanAgentState
+from .view import HumanAgentView
+
+
+async def run_human_agent_service(
+    state: TaskState, commands: list[HumanAgentCommand], view: HumanAgentView | None
+) -> TaskState:
+    # initialise agent state
+    instructions = "\n\n".join([message.text for message in state.messages]).strip()
+    agent_state = HumanAgentState(instructions=instructions)
+
+    # extract service methods from commands
+    methods = {
+        command.name: command.service(agent_state)
+        for command in commands
+        if "service" in command.contexts
+    }
+
+    # callback to check if task is completed (use this to periodically
+    # update the view with the current state)
+    def task_is_completed() -> bool:
+        if view:
+            view.update_state(agent_state)
+        return agent_state.answer is not None
+
+    # run the service
+    await sandbox_service(
+        name="human_agent",
+        methods=methods,
+        until=task_is_completed,
+        sandbox=sandbox(),
+    )
+
+    # set the answer if we have one
+    if agent_state.answer is not None:
+        state.output = ModelOutput.from_content("human_agent", agent_state.answer)
+
+    # return state
+    return state

inspect_ai/solver/_human_agent/state.py

@@ -0,0 +1,55 @@
+from time import time as current_time
+
+from pydantic import BaseModel, Field
+
+from inspect_ai.scorer._metric import Score
+from inspect_ai.util._store_model import StoreModel
+
+
+class IntermediateScoring(BaseModel):
+    time: float
+    scores: list[Score]
+
+
+class HumanAgentState(StoreModel):
+    instructions: str
+    """Task instructions."""
+
+    @property
+    def running(self) -> bool:
+        """Is the task currently running?"""
+        return self.running_state
+
+    @running.setter
+    def running(self, running: bool) -> None:
+        """Set current running state."""
+        # if we are flipping to running mode then update started running
+        if not self.running_state and running:
+            self.started_running = current_time()
+
+        # if we are exiting running mode then update accumulated time
+        if self.running_state and not running:
+            self.accumulated_time = self.time
+
+        # update running
+        self.running_state = running
+
+    @property
+    def time(self) -> float:
+        """Total time spend on task."""
+        running_time = current_time() - self.started_running if self.running else 0
+        return self.accumulated_time + running_time
+
+    scorings: list[IntermediateScoring] = Field(default_factory=list)
+    """Intermediate scorings yielded by `task score`"""
+
+    answer: str | None = Field(default=None)
+    """Final answer provided in `task submit`"""
+
+    logs: dict[str, str] = Field(default_factory=dict)
+    """Session logs generated by `script` """
+
+    # internal state variables used by running and time properties
+    running_state: bool = Field(default=True)
+    started_running: float = Field(default_factory=current_time)
+    accumulated_time: float = Field(default=0.0)

inspect_ai/solver/_human_agent/view.py

@@ -0,0 +1,24 @@
+from typing import Protocol
+
+from inspect_ai.util import SandboxConnection
+
+from .state import HumanAgentState
+
+
+class HumanAgentView(Protocol):
+    def connect(self, connection: SandboxConnection) -> None: ...
+    def update_state(self, state: HumanAgentState) -> None: ...
+
+
+class ConsoleView(HumanAgentView):
+    """Fallback view for when we aren't running fullscreen UI."""
+
+    def connect(self, connection: SandboxConnection) -> None:
+        print(
+            "You are completing a task on a Linux system (task instructions will be presented "
+            + "when you login). Login to the system with the following command:\n"
+        )
+        print(f"{connection.command}\n")
+
+    def update_state(self, state: HumanAgentState) -> None:
+        pass

inspect_ai/solver/_task_state.py

@@ -3,11 +3,11 @@ from contextvars import ContextVar
 from copy import deepcopy
 from dataclasses import dataclass
 from random import Random
-from typing import Any, Union, cast, overload
+from typing import Any, Type, Union, cast, overload
 
 from pydantic_core import to_jsonable_python
 
-from inspect_ai.dataset._dataset import Sample
+from inspect_ai.dataset._dataset import MT, Sample, metadata_as
 from inspect_ai.model import (
     ChatMessage,
     ChatMessageUser,
@@ -19,6 +19,7 @@ from inspect_ai.model._model import sample_total_tokens
 from inspect_ai.tool import Tool, ToolChoice
 from inspect_ai.tool._tool_def import ToolDef
 from inspect_ai.util._store import Store, store_jsonable
+from inspect_ai.util._store_model import SMT
 
 
 @dataclass
@@ -349,6 +350,31 @@ class TaskState:
         for tool in tools:
             self._tools.append(tool if isinstance(tool, Tool) else tool.as_tool())
 
+    def metadata_as(self, metadata_cls: Type[MT]) -> MT:
+        """Pydantic model interface to metadata.
+
+        Args:
+          metadata_cls: Pydantic model type
+
+        Returns:
+          BaseModel: Instance of metadata_cls bound to current Store.
+        """
+        if not self.metadata:
+            raise ValueError("Sample does not have metadata")
+
+        return metadata_as(self.metadata, metadata_cls)
+
+    def store_as(self, model_cls: Type[SMT]) -> SMT:
+        """Pydantic model interface to the store.
+
+        Args:
+          model_cls: Pydantic model type (must derive from StoreModel)
+
+        Returns:
+          StoreModel: Instance of model_cls bound to current Store.
+        """
+        return model_cls(store=self.store)
+
 
 def sample_state() -> TaskState | None:
     return _sample_state.get(None)

inspect_ai/tool/_tool.py

@@ -11,6 +11,7 @@ from typing import (
     runtime_checkable,
 )
 
+from inspect_ai._util.content import ContentImage, ContentText
 from inspect_ai._util.registry import (
     RegistryInfo,
     registry_add,
@@ -18,13 +19,20 @@ from inspect_ai._util.registry import (
     registry_tag,
 )
 
-from . import Content
 from ._tool_call import ToolCallViewer
 
 logger = getLogger(__name__)
 
 
-ToolResult = str | int | float | bool | list[Content]
+ToolResult = (
+    str
+    | int
+    | float
+    | bool
+    | ContentText
+    | ContentImage
+    | list[ContentText | ContentImage]
+)
 
 
 class ToolError(Exception):