inspect-ai 0.3.55__py3-none-any.whl → 0.3.56__py3-none-any.whl

inspect_ai/solver/__init__.py

@@ -4,6 +4,7 @@ from ._basic_agent import basic_agent
 from ._chain import chain
 from ._critique import self_critique
 from ._fork import fork
+from ._human_agent.agent import human_agent
 from ._multiple_choice import MultipleChoiceTemplate, multiple_choice
 from ._plan import Plan, plan
 from ._prompt import (
@@ -17,6 +18,7 @@ from ._use_tools import use_tools
 
 __all__ = [
     "basic_agent",
+    "human_agent",
     "chain",
     "fork",
     "generate",

inspect_ai/solver/_human_agent/agent.py

@@ -0,0 +1,83 @@
+import asyncio
+
+from inspect_ai.util import display_type, input_panel, sandbox
+
+from .._solver import Generate, Solver, solver
+from .._task_state import TaskState
+from .commands import human_agent_commands
+from .install import install_human_agent
+from .panel import HumanAgentPanel
+from .service import run_human_agent_service
+from .view import ConsoleView, HumanAgentView
+
+
+@solver
+def human_agent(
+    answer: bool | str = True,
+    intermediate_scoring: bool = False,
+    record_session: bool = True,
+) -> Solver:
+    """Human solver for agentic tasks that run in a Linux environment.
+
+    The Human agent solver installs agent task tools in the default
+    sandbox and presents the user with both task instructions and
+    documentation for the various tools (e.g. `task submit`,
+    `task start`, `task stop` `task instructions`, etc.). A human agent panel
+    is displayed with instructions for logging in to the sandbox.
+
+    If the user is running in VS Code with the Inspect extension,
+    they will also be presented with links to login to the sandbox
+    using a VS Code Window or Terminal.
+
+    Args:
+       answer (bool | str): Is an explicit answer required for this
+          task or is it scored based on files in the container? Pass a
+          `str` with a regex to validate that the answer matches
+          the expected format.
+       intermediate_scoring (bool): Allow the human agent to
+          check their score while working.
+       record_session (bool): Record all user commands and outputs in
+          the sandbox bash session.
+
+    Returns:
+       Solver: Human agent solver.
+    """
+    # we can only run one human agent interaction at a time (use lock to enforce)
+    agent_lock = asyncio.Lock()
+
+    async def solve(state: TaskState, generate: Generate) -> TaskState:
+        async with agent_lock:
+            # ensure that we have a sandbox to work with
+            try:
+                connection = await sandbox().connection()
+            except ProcessLookupError:
+                raise RuntimeError("Human agent must run in a task with a sandbox.")
+            except NotImplementedError:
+                raise RuntimeError(
+                    "Human agent must run with a sandbox that supports connections."
+                )
+
+            # helper function to run the agent (called for fullscreen vs. fallback below)
+            async def run_human_agent(view: HumanAgentView) -> TaskState:
+                # create agent commands
+                commands = human_agent_commands(
+                    state, answer, intermediate_scoring, record_session
+                )
+
+                # install agent tools
+                await install_human_agent(state, commands, record_session)
+
+                # hookup the view ui
+                view.connect(connection)
+
+                # run sandbox service
+                return await run_human_agent_service(state, commands, view)
+
+            # support both fullscreen ui and fallback
+            if display_type() == "full":
+                async with await input_panel(HumanAgentPanel) as panel:
+                    return await run_human_agent(panel)
+            else:
+                return await run_human_agent(ConsoleView())
+
+    return solve

inspect_ai/solver/_human_agent/commands/__init__.py

@@ -0,0 +1,36 @@
+from inspect_ai.solver._task_state import TaskState
+
+from .clock import StartCommand, StopCommand
+from .command import HumanAgentCommand
+from .instructions import InstructionsCommand
+from .note import NoteCommand
+from .score import ScoreCommand
+from .status import StatusCommand
+from .submit import SubmitCommand, ValidateCommand
+
+
+def human_agent_commands(
+    state: TaskState,
+    answer: bool | str,
+    intermediate_scoring: bool,
+    record_session: bool,
+) -> list[HumanAgentCommand]:
+    # base submit and validate
+    commands = [SubmitCommand(record_session), ValidateCommand(answer)]
+
+    # optional intermediate scoring
+    if intermediate_scoring:
+        commands.append(ScoreCommand(state))
+
+    # remaining commands
+    commands.extend(
+        [
+            NoteCommand(),
+            StatusCommand(),
+            StartCommand(),
+            StopCommand(),
+        ]
+    )
+
+    # with instructions (letting it see the other commands)
+    return commands + [InstructionsCommand(commands)]

inspect_ai/solver/_human_agent/commands/clock.py

@@ -0,0 +1,70 @@
+from argparse import Namespace
+from typing import Awaitable, Callable, Literal
+
+from pydantic import JsonValue
+
+from inspect_ai._util.format import format_progress_time
+
+from ..state import HumanAgentState
+from .command import HumanAgentCommand, call_human_agent
+from .status import render_status
+
+
+class StartCommand(HumanAgentCommand):
+    @property
+    def name(self) -> str:
+        return "start"
+
+    @property
+    def description(self) -> str:
+        return "Start the task clock (resume working)."
+
+    @property
+    def group(self) -> Literal[1, 2, 3]:
+        return 2
+
+    def cli(self, args: Namespace) -> None:
+        print(call_human_agent("start"))
+
+    def service(self, state: HumanAgentState) -> Callable[..., Awaitable[JsonValue]]:
+        from inspect_ai.log._transcript import transcript
+
+        async def start() -> str:
+            if not state.running:
+                state.running = True
+                transcript().info(
+                    f"Task started (total time: {format_progress_time(state.time)})"
+                )
+            return render_status(state)
+
+        return start
+
+
+class StopCommand(HumanAgentCommand):
+    @property
+    def name(self) -> str:
+        return "stop"
+
+    @property
+    def description(self) -> str:
+        return "Stop the task clock (pause working)."
+
+    @property
+    def group(self) -> Literal[1, 2, 3]:
+        return 2
+
+    def cli(self, args: Namespace) -> None:
+        print(call_human_agent("stop"))
+
+    def service(self, state: HumanAgentState) -> Callable[..., Awaitable[JsonValue]]:
+        from inspect_ai.log._transcript import transcript
+
+        async def stop() -> str:
+            if state.running:
+                state.running = False
+                transcript().info(
+                    f"Task stopped (total time: {format_progress_time(state.time)})"
+                )
+            return render_status(state)
+
+        return stop

inspect_ai/solver/_human_agent/commands/command.py

@@ -0,0 +1,59 @@
+import abc
+from argparse import Namespace
+from typing import Any, Awaitable, Callable, Literal, NamedTuple
+
+from pydantic import JsonValue
+
+from ..state import HumanAgentState
+
+
+class HumanAgentCommand:
+    @property
+    @abc.abstractmethod
+    def name(self) -> str:
+        """Command name (e.g. 'submit')"""
+        ...
+
+    @property
+    @abc.abstractmethod
+    def description(self) -> str:
+        """Command description."""
+        ...
+
+    @property
+    def group(self) -> Literal[1, 2, 3]:
+        return 1
+
+    @property
+    def contexts(self) -> list[Literal["cli", "service"]]:
+        """Contexts where this command runs (defaults to both cli and service)."""
+        return ["cli", "service"]
+
+    class CLIArg(NamedTuple):
+        name: str
+        description: str
+        required: bool = False
+
+    @property
+    def cli_args(self) -> list[CLIArg]:
+        """Positional command line arguments."""
+        return []
+
+    def cli(self, args: Namespace) -> None:
+        """CLI command (runs in container). Required for context "cli"."""
+        pass
+
+    def service(self, state: HumanAgentState) -> Callable[..., Awaitable[JsonValue]]:
+        """Service handler (runs in solver). Required for context "service"."""
+
+        async def no_handler() -> None:
+            pass
+
+        return no_handler
+
+
+# Dummy functions for implementation of call methods
+
+
+def call_human_agent(method: str, **params: Any) -> Any:
+    return None

inspect_ai/solver/_human_agent/commands/instructions.py

@@ -0,0 +1,74 @@
+from argparse import Namespace
+from typing import Awaitable, Callable, Literal
+
+from pydantic import JsonValue
+from rich.console import Group
+from rich.panel import Panel
+from rich.table import Table
+from rich.text import Text
+
+from inspect_ai._util.ansi import render_text
+from inspect_ai._util.transcript import DOUBLE_LINE
+
+from ..state import HumanAgentState
+from .command import HumanAgentCommand, call_human_agent
+
+
+class InstructionsCommand(HumanAgentCommand):
+    def __init__(self, commands: list[HumanAgentCommand]) -> None:
+        self._commands = commands.copy() + [self]
+
+    @property
+    def name(self) -> str:
+        return "instructions"
+
+    @property
+    def description(self) -> str:
+        return "Display task commands and instructions."
+
+    @property
+    def group(self) -> Literal[1, 2, 3]:
+        return 3
+
+    def cli(self, args: Namespace) -> None:
+        print(call_human_agent("instructions", **vars(args)))
+
+    def service(self, state: HumanAgentState) -> Callable[..., Awaitable[JsonValue]]:
+        async def instructions() -> str:
+            intro = "\nYou will be completing a task based on the instructions presented below. You can use the following commands as you work on the task:\n"
+            commands_table = Table(box=None, show_header=False)
+            commands_table.add_column("", justify="left")
+            commands_table.add_column("", justify="left")
+
+            def add_command_group(group: int) -> None:
+                for command in filter(
+                    lambda c: "cli" in c.contexts and c.group == group, self._commands
+                ):
+                    commands_table.add_row(f"task {command.name}", command.description)
+                if group != 3:
+                    commands_table.add_row("", "")
+
+            for i in range(1, 4):
+                add_command_group(i)
+
+            header_panel = Panel(
+                Group(intro, commands_table),
+                title=Text.from_markup("[bold]Human Agent Task[/bold]"),
+                box=DOUBLE_LINE,
+                padding=(0, 0),
+            )
+
+            instructions_panel = Panel(
+                f"{state.instructions.strip()}",
+                title="Task Instructions",
+                padding=(1, 1),
+            )
+
+            return render_text(
+                ["", header_panel, instructions_panel],
+                styles=False,
+                no_color=True,
+                width=90,
+            )
+
+        return instructions

inspect_ai/solver/_human_agent/commands/note.py

@@ -0,0 +1,42 @@
+from argparse import Namespace
+from typing import Awaitable, Callable, Literal
+
+from pydantic import JsonValue
+
+from ..state import HumanAgentState
+from .command import HumanAgentCommand, call_human_agent
+
+
+class NoteCommand(HumanAgentCommand):
+    @property
+    def name(self) -> str:
+        return "note"
+
+    @property
+    def description(self) -> str:
+        return "Record a note in the task transcript."
+
+    @property
+    def group(self) -> Literal[1, 2, 3]:
+        return 1
+
+    def cli(self, args: Namespace) -> None:
+        print(
+            "Enter a multiline markdown note (Press Ctrl+D on a new line to finish):\n"
+        )
+        lines = ["## Human Agent Note"]
+        try:
+            while True:
+                line = input()
+                lines.append(line)
+        except EOFError:
+            pass
+        call_human_agent("note", content="\n".join(lines))
+
+    def service(self, state: HumanAgentState) -> Callable[..., Awaitable[JsonValue]]:
+        from inspect_ai.log._transcript import transcript
+
+        async def note(content: str) -> None:
+            transcript().info(content)
+
+        return note

inspect_ai/solver/_human_agent/commands/score.py

@@ -0,0 +1,80 @@
+from argparse import Namespace
+from copy import deepcopy
+from textwrap import dedent
+from typing import Awaitable, Callable, Literal
+
+from pydantic import JsonValue
+
+from inspect_ai._util.ansi import render_text
+from inspect_ai.model._model_output import ModelOutput
+from inspect_ai.scorer._score import score
+
+from ..._task_state import TaskState
+from ..state import HumanAgentState, IntermediateScoring
+from .command import HumanAgentCommand, call_human_agent
+
+
+class ScoreCommand(HumanAgentCommand):
+    def __init__(self, state: TaskState):
+        self._state = state
+
+    @property
+    def name(self) -> str:
+        return "score"
+
+    @property
+    def description(self) -> str:
+        return "Score the task to check progress."
+
+    @property
+    def group(self) -> Literal[1, 2, 3]:
+        return 1
+
+    @property
+    def cli_args(self) -> list[HumanAgentCommand.CLIArg]:
+        return [
+            HumanAgentCommand.CLIArg(
+                name="answer",
+                description="Answer to submit for scoring (optional, not required for all tasks)",
+            )
+        ]
+
+    def cli(self, args: Namespace) -> None:
+        # first validate (print and exit if we get a str back)
+        call_args = vars(args)
+        error = call_human_agent("validate", **call_args)
+        if error:
+            print(error)
+            return
+
+        print(call_human_agent("score", **call_args))
+
+    def service(self, state: HumanAgentState) -> Callable[..., Awaitable[JsonValue]]:
+        async def score_task(answer: str | None) -> str:
+            from inspect_ai.log._transcript import transcript
+
+            # make a copy of TaskState, add the answer, then score
+            if answer:
+                task_state = deepcopy(self._state)
+                task_state.output = ModelOutput.from_content("human_agent", answer)
+                result = await score(task_state)
+            else:
+                result = await score(self._state)
+
+            # record the scoring action in our state
+            state.scorings.append(IntermediateScoring(time=state.time, scores=result))
+
+            # record to transcript
+            transcript().info(
+                dedent(f"""
+            ### Intermediate Score
+            **Answer:** {result[0].answer}, **Score:** {result[0].as_str()}
+            """)
+            )
+
+            # notify user
+            return render_text(
+                f"[bold]Answer:[/bold] {result[0].answer}, [bold]Score:[/bold] {result[0].as_str()}"
+            )
+
+        return score_task

inspect_ai/solver/_human_agent/commands/status.py

@@ -0,0 +1,62 @@
+from argparse import Namespace
+from typing import Awaitable, Callable, Literal
+
+from pydantic import JsonValue
+from rich.console import RenderableType
+from rich.table import Table
+from rich.text import Text
+
+from inspect_ai._util.ansi import render_text
+from inspect_ai._util.format import format_progress_time
+
+from ..state import HumanAgentState
+from .command import HumanAgentCommand, call_human_agent
+
+
+class StatusCommand(HumanAgentCommand):
+    @property
+    def name(self) -> str:
+        return "status"
+
+    @property
+    def description(self) -> str:
+        return "Print task status (clock, scoring, etc.)"
+
+    @property
+    def group(self) -> Literal[1, 2, 3]:
+        return 2
+
+    def cli(self, args: Namespace) -> None:
+        print(call_human_agent("status"))
+
+    def service(self, state: HumanAgentState) -> Callable[..., Awaitable[JsonValue]]:
+        async def status() -> str:
+            return render_status(state)
+
+        return status
+
+
+def render_status(state: HumanAgentState) -> str:
+    content: list[RenderableType] = [""]
+    content.append(
+        f"[bold]Status:[/bold] {'Running' if state.running else 'Stopped'}  "
+        + f"[bold]Time:[/bold] {format_progress_time(state.time, pad_hours=False)}"
+    )
+
+    if len(state.scorings) > 0:
+        content.append("")
+        content.append(Text.from_markup("[italic]Intermediate Scores[/italic]"))
+        scores_table = Table(box=None, min_width=35, padding=(0, 0))
+        scores_table.add_column("Answer", justify="left")
+        scores_table.add_column("Score", justify="center")
+        scores_table.add_column("Time", justify="right")
+
+        for score in state.scorings:
+            scores_table.add_row(
+                score.scores[0].answer,
+                score.scores[0].as_str(),
+                format_progress_time(score.time),
+            )
+        content.append(scores_table)
+
+    return render_text(content, highlight=False)

inspect_ai/solver/_human_agent/commands/submit.py

@@ -0,0 +1,151 @@
+from argparse import Namespace
+from logging import getLogger
+from pathlib import PurePosixPath
+from re import Pattern, compile, match
+from typing import Awaitable, Callable, Literal
+
+from pydantic import JsonValue
+
+from inspect_ai._util.ansi import render_text
+from inspect_ai.util._sandbox import sandbox
+
+from ..install import RECORD_SESSION_DIR
+from ..state import HumanAgentState
+from .command import HumanAgentCommand, call_human_agent
+
+logger = getLogger(__name__)
+
+
+class SubmitCommand(HumanAgentCommand):
+    def __init__(self, record_session: bool):
+        super().__init__()
+        self._record_session = record_session
+
+    @property
+    def name(self) -> str:
+        return "submit"
+
+    @property
+    def description(self) -> str:
+        return "Submit your final answer for the task."
+
+    @property
+    def group(self) -> Literal[1, 2, 3]:
+        return 1
+
+    @property
+    def cli_args(self) -> list[HumanAgentCommand.CLIArg]:
+        return [
+            HumanAgentCommand.CLIArg(
+                name="answer",
+                description="Answer to submit for scoring (optional, not required for all tasks)",
+            )
+        ]
+
+    def cli(self, args: Namespace) -> None:
+        # read cli args
+        call_args = vars(args)
+
+        # first validate (print and exit if we get a str back)
+        error = call_human_agent("validate", **call_args)
+        if error:
+            print(error)
+            return
+
+        # verify that the user wants to proceed
+        answer = call_args.get("answer", None)
+        answer_text = f" '{answer}'" if answer else ""
+        while True:
+            response = (
+                input(
+                    f"\nDo you definitely want to end the task and submit{answer_text}?\n\nThis will disconnect you from the task environment and you won't be able to reconnect.\n\nYes (y) or No (n): "
+                )
+                .lower()
+                .strip()
+            )
+            if response in ["yes", "y"]:
+                break
+            elif response in ["no", "n"]:
+                return
+            else:
+                print("Please enter yes or no.")
+
+        # thank the user!
+        print(
+            "\nThank you for working on this task!\n\n"
+            + "Your task will now be scored and you will be disconnected from this container.\n"
+        )
+
+        # submit the task
+        call_human_agent("submit", **call_args)
+
+    def service(self, state: HumanAgentState) -> Callable[..., Awaitable[JsonValue]]:
+        async def submit(
+            answer: str | None, session_logs: dict[str, str] | None = None
+        ) -> None:
+            if self._record_session:
+                state.logs = await self._read_session_logs()
+            state.running = False
+            state.answer = answer
+
+        return submit
+
+    async def _read_session_logs(self) -> dict[str, str]:
+        # retreive session logs (don't fail)
+        sessions_dir = PurePosixPath(RECORD_SESSION_DIR)
+        result = await sandbox().exec(["ls", "-1", sessions_dir.as_posix()])
+        if not result.success:
+            logger.warning(f"Error listing human agent session logs: {result.stderr}")
+            return {}
+
+        # read logs
+        session_logs: dict[str, str] = {}
+        for session_log in result.stdout.strip().splitlines():
+            try:
+                session_logs[session_log] = await sandbox().read_file(
+                    (sessions_dir / session_log).as_posix()
+                )
+            except Exception as ex:
+                logger.warning(f"Error reading human agent session log: {ex}")
+
+        return session_logs
+
+
+class ValidateCommand(HumanAgentCommand):
+    def __init__(self, answer: bool | str) -> None:
+        self._answer = compile(answer) if isinstance(answer, str) else answer
+
+    @property
+    def name(self) -> str:
+        return "validate"
+
+    @property
+    def description(self) -> str:
+        return "Validate a task submission."
+
+    @property
+    def contexts(self) -> list[Literal["cli", "service"]]:
+        return ["service"]
+
+    def service(self, state: HumanAgentState) -> Callable[..., Awaitable[JsonValue]]:
+        async def validate(answer: str | None) -> str | None:
+            def failed(reason: str) -> str:
+                return render_text(f"[bold]FAILED:[/bold] {reason}")
+
+            if not state.running:
+                return failed("Task is stopped (use 'task start' to start)")
+            if self._answer:
+                answer = answer.strip() if isinstance(answer, str) else answer
+                if not answer:
+                    return failed(
+                        "An explicit answer is required for scoring this task."
+                    )
+                elif isinstance(self._answer, Pattern) and not match(
+                    self._answer, answer
+                ):
+                    return failed(
+                        "Your answer was not in the required format (please review the task instructions)"
+                    )
+            return None  # made it through verification
+
+        return validate