inspect-ai 0.3.54__py3-none-any.whl → 0.3.56__py3-none-any.whl

inspect_ai/solver/_human_agent/commands/submit.py

@@ -0,0 +1,151 @@
+from argparse import Namespace
+from logging import getLogger
+from pathlib import PurePosixPath
+from re import Pattern, compile, match
+from typing import Awaitable, Callable, Literal
+
+from pydantic import JsonValue
+
+from inspect_ai._util.ansi import render_text
+from inspect_ai.util._sandbox import sandbox
+
+from ..install import RECORD_SESSION_DIR
+from ..state import HumanAgentState
+from .command import HumanAgentCommand, call_human_agent
+
+logger = getLogger(__name__)
+
+
+class SubmitCommand(HumanAgentCommand):
+    def __init__(self, record_session: bool):
+        super().__init__()
+        self._record_session = record_session
+
+    @property
+    def name(self) -> str:
+        return "submit"
+
+    @property
+    def description(self) -> str:
+        return "Submit your final answer for the task."
+
+    @property
+    def group(self) -> Literal[1, 2, 3]:
+        return 1
+
+    @property
+    def cli_args(self) -> list[HumanAgentCommand.CLIArg]:
+        return [
+            HumanAgentCommand.CLIArg(
+                name="answer",
+                description="Answer to submit for scoring (optional, not required for all tasks)",
+            )
+        ]
+
+    def cli(self, args: Namespace) -> None:
+        # read cli args
+        call_args = vars(args)
+
+        # first validate (print and exit if we get a str back)
+        error = call_human_agent("validate", **call_args)
+        if error:
+            print(error)
+            return
+
+        # verify that the user wants to proceed
+        answer = call_args.get("answer", None)
+        answer_text = f" '{answer}'" if answer else ""
+        while True:
+            response = (
+                input(
+                    f"\nDo you definitely want to end the task and submit{answer_text}?\n\nThis will disconnect you from the task environment and you won't be able to reconnect.\n\nYes (y) or No (n): "
+                )
+                .lower()
+                .strip()
+            )
+            if response in ["yes", "y"]:
+                break
+            elif response in ["no", "n"]:
+                return
+            else:
+                print("Please enter yes or no.")
+
+        # thank the user!
+        print(
+            "\nThank you for working on this task!\n\n"
+            + "Your task will now be scored and you will be disconnected from this container.\n"
+        )
+
+        # submit the task
+        call_human_agent("submit", **call_args)
+
+    def service(self, state: HumanAgentState) -> Callable[..., Awaitable[JsonValue]]:
+        async def submit(
+            answer: str | None, session_logs: dict[str, str] | None = None
+        ) -> None:
+            if self._record_session:
+                state.logs = await self._read_session_logs()
+            state.running = False
+            state.answer = answer
+
+        return submit
+
+    async def _read_session_logs(self) -> dict[str, str]:
+        # retreive session logs (don't fail)
+        sessions_dir = PurePosixPath(RECORD_SESSION_DIR)
+        result = await sandbox().exec(["ls", "-1", sessions_dir.as_posix()])
+        if not result.success:
+            logger.warning(f"Error listing human agent session logs: {result.stderr}")
+            return {}
+
+        # read logs
+        session_logs: dict[str, str] = {}
+        for session_log in result.stdout.strip().splitlines():
+            try:
+                session_logs[session_log] = await sandbox().read_file(
+                    (sessions_dir / session_log).as_posix()
+                )
+            except Exception as ex:
+                logger.warning(f"Error reading human agent session log: {ex}")
+
+        return session_logs
+
+
+class ValidateCommand(HumanAgentCommand):
+    def __init__(self, answer: bool | str) -> None:
+        self._answer = compile(answer) if isinstance(answer, str) else answer
+
+    @property
+    def name(self) -> str:
+        return "validate"
+
+    @property
+    def description(self) -> str:
+        return "Validate a task submission."
+
+    @property
+    def contexts(self) -> list[Literal["cli", "service"]]:
+        return ["service"]
+
+    def service(self, state: HumanAgentState) -> Callable[..., Awaitable[JsonValue]]:
+        async def validate(answer: str | None) -> str | None:
+            def failed(reason: str) -> str:
+                return render_text(f"[bold]FAILED:[/bold] {reason}")
+
+            if not state.running:
+                return failed("Task is stopped (use 'task start' to start)")
+            if self._answer:
+                answer = answer.strip() if isinstance(answer, str) else answer
+                if not answer:
+                    return failed(
+                        "An explicit answer is required for scoring this task."
+                    )
+                elif isinstance(self._answer, Pattern) and not match(
+                    self._answer, answer
+                ):
+                    return failed(
+                        "Your answer was not in the required format (please review the task instructions)"
+                    )
+            return None  # made it through verification
+
+        return validate

inspect_ai/solver/_human_agent/install.py

@@ -0,0 +1,222 @@
+import inspect
+from textwrap import dedent
+
+from inspect_ai.util import sandbox
+
+from .._task_state import TaskState
+from .commands.command import HumanAgentCommand
+
+INSTALL_DIR = "human_agent_install"
+HUMAN_AGENT_DIR = "/opt/human_agent"
+TASK_PY = "task.py"
+INSTALL_SH = "install.sh"
+BASHRC = ".bashrc"
+WELCOME_FILE = "welcome.txt"
+WELCOME_LOGIN_FILE = "welcome_login.txt"
+INSTRUCTIONS_FILE = "instructions.txt"
+RECORD_SESSION_DIR = "/var/tmp/user-sessions"
+
+
+async def install_human_agent(
+    state: TaskState, commands: list[HumanAgentCommand], record_session: bool
+) -> None:
+    # see if we have already installed
+    if not (await sandbox().exec(["mkdir", HUMAN_AGENT_DIR])).success:
+        return
+
+    # setup installation directory
+    await checked_exec(["mkdir", "-p", INSTALL_DIR])
+
+    # generate task.py
+    task_py = human_agent_commands(commands)
+    await checked_write_file(f"{INSTALL_DIR}/{TASK_PY}", task_py, executable=True)
+
+    # generate .bashrc
+    bash_rc = human_agent_bashrc(commands, record_session)
+    await checked_write_file(f"{INSTALL_DIR}/{BASHRC}", bash_rc, executable=True)
+
+    # write and run installation script
+    install_sh = human_agent_install_sh()
+    await checked_write_file(f"{INSTALL_DIR}/{INSTALL_SH}", install_sh, executable=True)
+    await checked_exec(["bash", f"./{INSTALL_SH}"], cwd=INSTALL_DIR)
+    await checked_exec(["rm", "-rf", INSTALL_DIR])
+
+
+def human_agent_commands(commands: list[HumanAgentCommand]) -> str:
+    # filter out hidden commands
+    commands = [command for command in commands if "cli" in command.contexts]
+
+    # standard imports (including any dependencies that call methods carry)
+    imports = dedent("""
+    import argparse
+    import sys
+    from argparse import Namespace
+    from pathlib import Path
+
+    sys.path.append("/var/tmp/sandbox-services/human_agent")
+    from human_agent import call_human_agent
+
+    def format_time(t):
+        minutes, seconds = divmod(t, 60)
+        hours, minutes = divmod(minutes, 60)
+        return f"{hours:.0f}:{minutes:02.0f}:{seconds:02.0f}"
+    """)
+
+    # command handler source code (extracted from call methods)
+    command_handlers = "\n\n".join(
+        dedent(
+            inspect.getsource(command.cli).replace("cli(self, ", f"{command.name}(", 1)
+        )
+        for command in commands
+    )
+
+    # parse commands
+    command_parsers: list[str] = []
+    for command in commands:
+        command_parsers.append(
+            dedent(f"""
+        {command.name}_parser = subparsers.add_parser("{command.name}", help="{command.description}")
+        """).lstrip()
+        )
+        for arg in command.cli_args:
+            if arg.name.startswith("--"):
+                extras = 'action="store_true", default=False'
+            else:
+                extras = f"""nargs={1 if arg.required else '"?"'}"""
+            command_parsers.append(
+                dedent(f"""
+                {command.name}_parser.add_argument("{arg.name}", {extras}, help="{arg.description}")
+                """).strip()
+            )
+
+    parse = (
+        dedent("""
+    parser = argparse.ArgumentParser(description="Human agent tools.")
+    subparsers = parser.add_subparsers(dest="command")
+    """)
+        + "\n"
+        + "\n".join(command_parsers)
+    )
+
+    # dispatch commands
+    command_dispatchers: list[str] = []
+    for i, command in enumerate(commands):
+        conditional = "if" if i == 0 else "elif"
+        command_dispatchers.append(
+            f'{conditional} command == "{command.name}": {command.name}(args)'
+        )
+    command_dispatchers.append("else: parser.print_help()")
+
+    dispatch = dedent("""
+    args = parser.parse_args()
+    command = args.command
+    delattr(args, 'command')
+    """) + "\n".join(command_dispatchers)
+
+    return "\n".join([imports, command_handlers, parse, dispatch]) + "\n"
+
+
+def human_agent_bashrc(commands: list[HumanAgentCommand], record_session: bool) -> str:
+    # only run in interative terminals
+    TERMINAL_CHECK = dedent("""
+
+    ### Inspect Human Agent Setup #########################################=
+
+    # only run if shell is interactive
+    case $- in
+        *i*) ;;
+        *) return ;;
+    esac
+
+    # only run if attached to a terminal
+    if ! tty -s; then
+        return
+    fi
+    """)
+
+    # shell alias and completions
+    command_names = " ".join(
+        [f"{command.name}" for command in commands if "cli" in command.contexts]
+    )
+    COMMANDS = dedent(f"""
+    # shell alias for human agent commands
+    alias task='python3 {HUMAN_AGENT_DIR}/{TASK_PY}'
+
+    # completion handler
+    _task_completion() {{
+        local cur
+        cur="${{COMP_WORDS[COMP_CWORD]}}"
+        if [ "$COMP_CWORD" -eq 1 ]; then
+            local commands="{command_names}"
+
+            # Generate completion matches
+            COMPREPLY=($(compgen -W "${{commands}}" -- ${{cur}}))
+        fi
+    }}
+    complete -F _task_completion task
+    """)
+
+    # session recording
+    if record_session:
+        RECORDING = dedent(f"""
+        # record human agent session transcript
+        if [ -z "$SCRIPT_RUNNING" ]; then
+            export SCRIPT_RUNNING=1
+            LOGDIR={RECORD_SESSION_DIR}
+            mkdir -p "$LOGDIR"
+            TIMESTAMP=$(date +%Y%m%d_%H%M%S)
+            INPUTFILE="$LOGDIR/$(whoami)_$TIMESTAMP.input"
+            OUTPUTFILE="$LOGDIR/$(whoami)_$TIMESTAMP.output"
+            TIMINGFILE="$LOGDIR/$(whoami)_$TIMESTAMP.timing"
+            exec script -q -f -m advanced -I "$INPUTFILE" -O "$OUTPUTFILE" -T "$TIMINGFILE" -c "bash --login -i"
+        fi
+        """)
+    else:
+        RECORDING = ""
+
+    # display task instructions
+    INSTRUCTIONS = dedent("""
+    if [ -z "$INSTRUCTIONS_SHOWN" ]; then
+        export INSTRUCTIONS_SHOWN=1
+        task instructions > instructions.txt
+        cat instructions.txt
+    fi
+    """).lstrip()
+
+    # return .bashrc
+    return "\n".join([TERMINAL_CHECK, COMMANDS, RECORDING, INSTRUCTIONS])
+
+
+def human_agent_install_sh() -> str:
+    return dedent(f"""
+    #!/usr/bin/env bash
+
+    # create installation directory
+    HUMAN_AGENT="{HUMAN_AGENT_DIR}"
+    mkdir -p $HUMAN_AGENT
+
+    # copy command script
+    cp {TASK_PY} $HUMAN_AGENT
+
+    # append to .bashrc
+    cat {BASHRC} >> ~/{BASHRC}
+    """)
+
+
+async def checked_exec(
+    cmd: list[str],
+    input: str | bytes | None = None,
+    cwd: str | None = None,
+) -> str:
+    result = await sandbox().exec(cmd, input=input, cwd=cwd)
+    if not result.success:
+        raise RuntimeError(f"Error executing command {' '.join(cmd)}: {result.stderr}")
+    return result.stdout
+
+
+async def checked_write_file(
+    file: str, contents: str, executable: bool = False
+) -> None:
+    await checked_exec(["tee", "--", file], input=contents)
+    if executable:
+        await checked_exec(["chmod", "+x", file])

inspect_ai/solver/_human_agent/panel.py

@@ -0,0 +1,252 @@
+from typing import cast
+
+from textual.app import ComposeResult
+from textual.containers import (
+    Container,
+    Horizontal,
+    VerticalScroll,
+)
+from textual.reactive import reactive
+from textual.widgets import (
+    Button,
+    ContentSwitcher,
+    Label,
+    Link,
+    LoadingIndicator,
+    Static,
+)
+
+from inspect_ai._util.format import format_progress_time
+from inspect_ai._util.vscode import (
+    VSCodeCommand,
+    can_execute_vscode_commands,
+    execute_vscode_commands,
+)
+from inspect_ai.util import InputPanel, SandboxConnection, throttle
+
+from .state import HumanAgentState
+
+
+class HumanAgentPanel(InputPanel):
+    DEFAULT_TITLE = "Human Agent"
+
+    SANDBOX_VIEW_ID = "human-agent-sandbox-view"
+    SANDBOX_INSTRUCTIONS_ID = "sandbox-instructions"
+    VSCODE_LINKS_ID = "vscode-links"
+    LOGIN_VSCODE_TERMINAL_ID = "login-vscode-terminal"
+    LOGIN_VSCODE_WINDOW_ID = "login-vscode-window"
+    COMMAND_INSTRUCTIONS_ID = "command-instructions"
+    SANDBOX_COMMAND_ID = "sandbox-command"
+
+    INSTRUCTIONS_CLASS = "instructions"
+    LINK_LABEL_CLASS = "link-label"
+
+    DEFAULT_CSS = f"""
+    #{SANDBOX_VIEW_ID} {{
+        scrollbar-size-vertical: 1;
+    }}
+    HumanAgentPanel .{INSTRUCTIONS_CLASS} {{
+        color: $text-muted;
+        margin-bottom: 1;
+    }}
+    #{SANDBOX_COMMAND_ID} {{
+        color: $secondary;
+    }}
+    HumanAgentPanel .{LINK_LABEL_CLASS} {{
+        color: $text-muted;
+    }}
+    HumanAgentPanel VSCodeLink {{
+        margin-left: 1;
+        margin-right: 2;
+    }}
+    HumanAgentPanel #{VSCODE_LINKS_ID} {{
+        height: 1;
+        margin-bottom: 1;
+    }}
+    """
+
+    connection: reactive[SandboxConnection | None] = reactive(None)
+
+    # implement HumanAgentView
+    def connect(self, connection: SandboxConnection) -> None:
+        self.connection = connection
+
+    @throttle(1)
+    def update_state(self, state: HumanAgentState) -> None:
+        status_bar = self.query_one(StatusBar)
+        status_bar.running = state.running
+        status_bar.time = state.time
+
+    def compose(self) -> ComposeResult:
+        with ContentSwitcher(initial=LoadingView.ID):
+            yield LoadingView()
+            with VerticalScroll(id=self.SANDBOX_VIEW_ID):
+                yield StatusBar()
+                yield Static(
+                    id=self.SANDBOX_INSTRUCTIONS_ID, classes=self.INSTRUCTIONS_CLASS
+                )
+                with Horizontal(id=self.VSCODE_LINKS_ID):
+                    yield Label("Login:", classes=self.LINK_LABEL_CLASS)
+                    yield VSCodeLink(
+                        "VS Code Window",
+                        id=self.LOGIN_VSCODE_WINDOW_ID,
+                    )
+                    yield Label("Login:", classes=self.LINK_LABEL_CLASS)
+                    yield VSCodeLink(
+                        "VS Code Terminal",
+                        id=self.LOGIN_VSCODE_TERMINAL_ID,
+                    )
+                yield Static(
+                    id=self.COMMAND_INSTRUCTIONS_ID, classes=self.INSTRUCTIONS_CLASS
+                )
+                yield Static(id=self.SANDBOX_COMMAND_ID)
+
+    def watch_connection(self, connection: SandboxConnection | None) -> None:
+        if connection:
+            # switch to sandbox view
+            self.query_one(ContentSwitcher).current = self.SANDBOX_VIEW_ID
+
+            # note whether we are in vscode
+            vscode = can_execute_vscode_commands()
+
+            # suffix for instructions based on whether we are in vscode
+            instructions_command = "Login to the system with the following command (hold down Alt or Option to select text for copying):"
+            instructions_vscode = (
+                "Use the links below to login to the system within VS Code:"
+            )
+
+            # update instructions
+            instructions_lbl = cast(
+                Static, self.query_one(f"#{self.SANDBOX_INSTRUCTIONS_ID}")
+            )
+            instructions = f"""You are completing a task on a Linux system (task instructions will be presented when you login). {instructions_vscode if vscode else instructions_command}"""
+            instructions_lbl.update(instructions)
+
+            # login: vscode terminal
+            vscode_links = self.query_one(f"#{self.VSCODE_LINKS_ID}")
+            vscode_links.display = vscode
+            terminal_btn = cast(
+                VSCodeLink, self.query_one(f"#{self.LOGIN_VSCODE_TERMINAL_ID}")
+            )
+            terminal_btn.commands = [
+                VSCodeCommand(
+                    command="workbench.action.terminal.new", args=[{"location": 2}]
+                ),
+                VSCodeCommand(
+                    command="workbench.action.terminal.sendSequence",
+                    args=[{"text": f"{connection.command}\n"}],
+                ),
+            ]
+
+            # login: vscode window
+            window_btn = cast(
+                VSCodeLink, self.query_one(f"#{self.LOGIN_VSCODE_WINDOW_ID}")
+            )
+            if connection.vscode_command is not None:
+                window_btn.commands = [
+                    VSCodeCommand(
+                        command=connection.vscode_command[0],
+                        args=connection.vscode_command[1:],
+                    )
+                ]
+
+            # command (always available)
+            command_instructions_lbl = cast(
+                Static, self.query_one(f"#{self.COMMAND_INSTRUCTIONS_ID}")
+            )
+            command_instructions_lbl.display = vscode
+            command_instructions_lbl.update(
+                instructions_command.replace("Login", "Alternatively, login", 1)
+            )
+            command_lbl = cast(Static, self.query_one(f"#{self.SANDBOX_COMMAND_ID}"))
+            command_lbl.update(connection.command)
+
+
+class StatusBar(Horizontal):
+    STATUS_ID = "task-status"
+    TIME_ID = "task-time"
+
+    LABEL_CLASS = "status-label"
+    VALUE_CLASS = "status-value"
+
+    DEFAULT_CSS = f"""
+    StatusBar {{
+        width: 1fr;
+        height: 1;
+        background: $surface;
+        margin-bottom: 1;
+        layout: grid;
+        grid-size: 4 1;
+        grid-columns: auto auto auto auto;
+        grid-gutter: 1;
+    }}
+    .{LABEL_CLASS} {{
+        color: $primary;
+    }}
+    .{VALUE_CLASS} {{
+        color: $foreground;
+    }}
+    StatusBar Link {{
+        dock: right;
+        margin-right: 1;
+    }}
+    """
+
+    running: reactive[bool] = reactive(True)
+    time: reactive[float] = reactive(0)
+
+    def __init__(self) -> None:
+        super().__init__()
+
+    def compose(self) -> ComposeResult:
+        yield Label("Status:", classes=self.LABEL_CLASS)
+        yield Static("Running", id=self.STATUS_ID, classes=self.VALUE_CLASS)
+        yield Label(" Time:", classes=self.LABEL_CLASS)
+        yield Static("0:00:00", id=self.TIME_ID, classes=self.VALUE_CLASS)
+
+    def watch_running(self, running: bool) -> None:
+        cast(Static, self.query_one(f"#{self.STATUS_ID}")).update(
+            "Running" if running else "Stopped"
+        )
+
+    def watch_time(self, time: float) -> None:
+        time_display = format_progress_time(time)
+        cast(Static, self.query_one(f"#{self.TIME_ID}")).update(time_display)
+
+
+class LoadingView(Container):
+    ID = "human-agent-loading-view"
+
+    def __init__(self) -> None:
+        super().__init__(id=self.ID)
+
+    def compose(self) -> ComposeResult:
+        yield LoadingIndicator()
+        yield Button()  # add focusable widget so the tab can activate
+
+
+class VSCodeLink(Link):
+    def __init__(
+        self,
+        text: str,
+        *,
+        url: str | None = None,
+        tooltip: str | None = None,
+        name: str | None = None,
+        id: str | None = None,
+        classes: str | None = None,
+        disabled: bool = False,
+    ) -> None:
+        super().__init__(
+            text,
+            url=url,
+            tooltip=tooltip,
+            name=name,
+            id=id,
+            classes=classes,
+            disabled=disabled,
+        )
+        self.commands: list[VSCodeCommand] = []
+
+    def on_click(self) -> None:
+        execute_vscode_commands(self.commands)

inspect_ai/solver/_human_agent/service.py

@@ -0,0 +1,45 @@
+from inspect_ai.model import ModelOutput
+from inspect_ai.util._sandbox import sandbox
+from inspect_ai.util._sandbox.service import sandbox_service
+
+from .._task_state import TaskState
+from .commands.command import HumanAgentCommand
+from .state import HumanAgentState
+from .view import HumanAgentView
+
+
+async def run_human_agent_service(
+    state: TaskState, commands: list[HumanAgentCommand], view: HumanAgentView | None
+) -> TaskState:
+    # initialise agent state
+    instructions = "\n\n".join([message.text for message in state.messages]).strip()
+    agent_state = HumanAgentState(instructions=instructions)
+
+    # extract service methods from commands
+    methods = {
+        command.name: command.service(agent_state)
+        for command in commands
+        if "service" in command.contexts
+    }
+
+    # callback to check if task is completed (use this to periodically
+    # update the view with the current state)
+    def task_is_completed() -> bool:
+        if view:
+            view.update_state(agent_state)
+        return agent_state.answer is not None
+
+    # run the service
+    await sandbox_service(
+        name="human_agent",
+        methods=methods,
+        until=task_is_completed,
+        sandbox=sandbox(),
+    )
+
+    # set the answer if we have one
+    if agent_state.answer is not None:
+        state.output = ModelOutput.from_content("human_agent", agent_state.answer)
+
+    # return state
+    return state