inspect-ai 0.3.81__py3-none-any.whl → 0.3.83__py3-none-any.whl

inspect_ai/agent/_handoff.py

@@ -0,0 +1,93 @@
+from typing import Any
+
+from inspect_ai._util.registry import (
+    RegistryInfo,
+    is_registry_object,
+    registry_unqualified_name,
+    set_registry_info,
+)
+from inspect_ai.tool._tool import Tool, ToolResult
+from inspect_ai.tool._tool_description import ToolDescription, set_tool_description
+
+from ._agent import Agent
+from ._as_tool import agent_tool_info
+from ._filter import MessageFilter
+
+
+def handoff(
+    agent: Agent,
+    description: str | None = None,
+    input_filter: MessageFilter | None = None,
+    output_filter: MessageFilter | None = None,
+    tool_name: str | None = None,
+    **agent_kwargs: Any,
+) -> Tool:
+    """Create a tool that enables models to handoff to agents.
+
+    Args:
+        agent: Agent to hand off to.
+        description: Handoff tool description (defaults to agent description)
+        input_filter: Filter to modify the message history before calling the tool.
+            Use the built-in `remove_tools` filter to remove all tool calls
+            or alternatively specify a custom `MessageFilter` function.
+        output_filter: Filter to modify the message history after calling the tool.
+            Use the built-in `last_message` filter to return only the last message
+            or alternatively specify a custom `MessageFilter` function.
+        tool_name: Alternate tool name (defaults to `transfer_to_{agent_name}`)
+        **agent_kwargs: Arguments to curry to `Agent` function (arguments provided here
+            will not be presented to the model as part of the tool interface).
+
+    Returns:
+        Tool for handing off to the agent (must be called using `execute_tools()` to be
+        properly handled)
+    """
+    # agent must be registered (so we can get its name)
+    if not is_registry_object(agent):
+        raise RuntimeError(
+            "Agent passed to as_tool was not created by an @agent decorated function"
+        )
+
+    # get tool_info
+    tool_info = agent_tool_info(agent, description, **agent_kwargs)
+
+    # AgentTool calls will be intercepted by execute_tools
+    agent_tool = AgentTool(agent, input_filter, output_filter, **agent_kwargs)
+    tool_name = tool_name or f"transfer_to_{tool_info.name}"
+    set_registry_info(agent_tool, RegistryInfo(type="tool", name=tool_name))
+    set_tool_description(
+        agent_tool,
+        ToolDescription(
+            name=tool_name,
+            description=tool_info.description,
+            parameters=tool_info.parameters,
+        ),
+    )
+    return agent_tool
+
+
+class AgentTool(Tool):
+    def __init__(
+        self,
+        agent: Agent,
+        input_filter: MessageFilter | None = None,
+        output_filter: MessageFilter | None = None,
+        **kwargs: Any,
+    ):
+        self.agent = agent
+        self.input_filter = input_filter
+        self.output_filter = output_filter
+        self.kwargs = kwargs
+
+    @property
+    def __name__(self) -> str:
+        return registry_unqualified_name(self.agent)
+
+    async def __call__(self) -> ToolResult:
+        raise RuntimeError("AgentTool should not be called directly")
+
+
+def has_handoff(tools: list[Tool] | None) -> bool:
+    if tools:
+        return any([isinstance(tool, AgentTool) for tool in tools])
+    else:
+        return False

inspect_ai/{solver/_human_agent → agent/_human}/agent.py

@@ -5,8 +5,7 @@ import anyio
 from inspect_ai.util import display_type, input_panel, sandbox
 from inspect_ai.util._sandbox.events import SandboxEnvironmentProxy
 
-from .._solver import Generate, Solver, solver
-from .._task_state import TaskState
+from .._agent import Agent, AgentState, agent
 from .commands import human_agent_commands
 from .install import install_human_agent
 from .panel import HumanAgentPanel
@@ -14,15 +13,15 @@ from .service import run_human_agent_service
 from .view import ConsoleView, HumanAgentView
 
 
-@solver
-def human_agent(
+@agent
+def human_cli(
     answer: bool | str = True,
     intermediate_scoring: bool = False,
     record_session: bool = True,
-) -> Solver:
-    """Human solver for agentic tasks that run in a Linux environment.
+) -> Agent:
+    """Human CLI agent for tasks that run in a sandbox.
 
-    The Human agent solver installs agent task tools in the default
+    The Human CLI agent installs agent task tools in the default
     sandbox and presents the user with both task instructions and
     documentation for the various tools (e.g. `task submit`,
     `task start`, `task stop` `task instructions`, etc.). A human agent panel
@@ -40,12 +39,12 @@ def human_agent(
        record_session: Record all user commands and outputs in the sandbox bash session.
 
     Returns:
-       Solver: Human agent solver.
+       Agent: Human CLI agent.
     """
     # we can only run one human agent interaction at a time (use lock to enforce)
     agent_lock = anyio.Lock()
 
-    async def solve(state: TaskState, generate: Generate) -> TaskState:
+    async def execute(state: AgentState) -> AgentState:
         async with agent_lock:
             # ensure that we have a sandbox to work with
             try:
@@ -58,7 +57,7 @@ def human_agent(
                 )
 
             # helper function to run the agent (called for fullscreen vs. fallback below)
-            async def run_human_agent(view: HumanAgentView) -> TaskState:
+            async def run_human_agent(view: HumanAgentView) -> AgentState:
                 sandbox_proxy = cast(SandboxEnvironmentProxy, sandbox())
                 with sandbox_proxy.no_events():
                     # create agent commands
@@ -67,7 +66,7 @@ def human_agent(
                     )
 
                     # install agent tools
-                    await install_human_agent(state, commands, record_session)
+                    await install_human_agent(commands, record_session)
 
                     # hookup the view ui
                     view.connect(connection)
@@ -82,4 +81,4 @@ def human_agent(
             else:
                 return await run_human_agent(ConsoleView())
 
-    return solve
+    return execute

inspect_ai/{solver/_human_agent → agent/_human}/commands/__init__.py

@@ -1,5 +1,4 @@
-from inspect_ai.solver._task_state import TaskState
-
+from ..._agent import AgentState
 from .clock import StartCommand, StopCommand
 from .command import HumanAgentCommand
 from .instructions import InstructionsCommand
@@ -10,7 +9,7 @@ from .submit import QuitCommand, SubmitCommand, ValidateCommand
 
 
 def human_agent_commands(
-    state: TaskState,
+    state: AgentState,
     answer: bool | str,
     intermediate_scoring: bool,
     record_session: bool,

inspect_ai/{solver/_human_agent → agent/_human}/commands/clock.py

@@ -62,7 +62,9 @@ class StopCommand(HumanAgentCommand):
         return stop
 
 
-def clock_action_event(action: str, state: HumanAgentState) -> None:
+def clock_action_event(
+    action: Literal["start", "stop"], state: HumanAgentState
+) -> None:
     from inspect_ai.log._transcript import transcript
 
     transcript().info(

inspect_ai/{solver/_human_agent → agent/_human}/commands/score.py

@@ -8,13 +8,13 @@ from inspect_ai._util.ansi import render_text
 from inspect_ai.model._model_output import ModelOutput
 from inspect_ai.scorer._score import score
 
-from ..._task_state import TaskState
+from ..._agent import AgentState
 from ..state import HumanAgentState, IntermediateScoring
 from .command import HumanAgentCommand, call_human_agent
 
 
 class ScoreCommand(HumanAgentCommand):
-    def __init__(self, state: TaskState):
+    def __init__(self, state: AgentState):
         self._state = state
 
     @property
@@ -52,9 +52,9 @@ class ScoreCommand(HumanAgentCommand):
         async def score_task(answer: str | None) -> str:
             # make a copy of TaskState, add the answer, then score
             if answer:
-                task_state = deepcopy(self._state)
-                task_state.output = ModelOutput.from_content("human_agent", answer)
-                result = await score(task_state)
+                agent_state = deepcopy(self._state)
+                agent_state.output = ModelOutput.from_content("human_agent", answer)
+                result = await score(agent_state)
             else:
                 result = await score(self._state)
 

inspect_ai/{solver/_human_agent → agent/_human}/install.py

@@ -3,7 +3,6 @@ from textwrap import dedent
 
 from inspect_ai.util import sandbox
 
-from .._task_state import TaskState
 from .commands.command import HumanAgentCommand
 
 INSTALL_DIR = "human_agent_install"
@@ -18,7 +17,7 @@ RECORD_SESSION_DIR = "/var/tmp/user-sessions"
 
 
 async def install_human_agent(
-    state: TaskState, commands: list[HumanAgentCommand], record_session: bool
+    commands: list[HumanAgentCommand], record_session: bool
 ) -> None:
     # see if we have already installed
     if not (await sandbox().exec(["mkdir", HUMAN_AGENT_DIR])).success:
@@ -183,8 +182,12 @@ def human_agent_bashrc(commands: list[HumanAgentCommand], record_session: bool)
     fi
     """).lstrip()
 
+    CLOCK = dedent("""
+    task start
+    """).lstrip()
+
     # return .bashrc
-    return "\n".join([TERMINAL_CHECK, COMMANDS, RECORDING, INSTRUCTIONS])
+    return "\n".join([TERMINAL_CHECK, COMMANDS, RECORDING, INSTRUCTIONS, CLOCK])
 
 
 def human_agent_install_sh() -> str:

inspect_ai/{solver/_human_agent → agent/_human}/service.py

@@ -1,20 +1,24 @@
+from inspect_ai.agent._human.commands.clock import clock_action_event
 from inspect_ai.model import ModelOutput
 from inspect_ai.util._sandbox import sandbox
 from inspect_ai.util._sandbox.service import sandbox_service
 
-from .._task_state import TaskState
+from .._agent import AgentState
 from .commands.command import HumanAgentCommand
 from .state import HumanAgentState
 from .view import HumanAgentView
 
 
 async def run_human_agent_service(
-    state: TaskState, commands: list[HumanAgentCommand], view: HumanAgentView | None
-) -> TaskState:
+    state: AgentState, commands: list[HumanAgentCommand], view: HumanAgentView | None
+) -> AgentState:
     # initialise agent state
     instructions = "\n\n".join([message.text for message in state.messages]).strip()
     agent_state = HumanAgentState(instructions=instructions)
 
+    # record that clock is stopped
+    clock_action_event("stop", agent_state)
+
     # extract service methods from commands
     methods = {
         command.name: command.service(agent_state)

inspect_ai/{solver/_human_agent → agent/_human}/state.py

@@ -1,4 +1,4 @@
-from time import time as current_time
+import time as python_time
 
 from pydantic import BaseModel, Field
 
@@ -25,7 +25,7 @@ class HumanAgentState(StoreModel):
         """Set current running state."""
         # if we are flipping to running mode then update started running
         if not self.running_state and running:
-            self.started_running = current_time()
+            self.started_running = python_time.time()
 
         # if we are exiting running mode then update accumulated time
         if self.running_state and not running:
@@ -37,7 +37,7 @@ class HumanAgentState(StoreModel):
     @property
     def time(self) -> float:
         """Total time spend on task."""
-        running_time = current_time() - self.started_running if self.running else 0
+        running_time = python_time.time() - self.started_running if self.running else 0
         return self.accumulated_time + running_time
 
     scorings: list[IntermediateScoring] = Field(default_factory=list)
@@ -50,6 +50,6 @@ class HumanAgentState(StoreModel):
     """Session logs generated by `script` """
 
     # internal state variables used by running and time properties
-    running_state: bool = Field(default=True)
-    started_running: float = Field(default_factory=current_time)
+    running_state: bool = Field(default=False)
+    started_running: float = Field(default_factory=python_time.time)
     accumulated_time: float = Field(default=0.0)

inspect_ai/agent/_react.py

@@ -0,0 +1,241 @@
+from logging import getLogger
+
+from inspect_ai._util._async import is_callable_coroutine
+from inspect_ai.model._call_tools import execute_tools
+from inspect_ai.model._chat_message import (
+    ChatMessage,
+    ChatMessageSystem,
+    ChatMessageUser,
+)
+from inspect_ai.model._model import Model, get_model
+from inspect_ai.scorer._score import score
+from inspect_ai.tool._tool import Tool, ToolResult, tool
+from inspect_ai.tool._tool_call import ToolCall
+from inspect_ai.tool._tool_info import parse_tool_info
+from inspect_ai.tool._tool_with import tool_with
+
+from ._agent import Agent, AgentState, agent, agent_with
+from ._handoff import has_handoff
+from ._types import (
+    AgentAttempts,
+    AgentContinue,
+    AgentPrompt,
+    AgentSubmit,
+)
+
+logger = getLogger(__name__)
+
+
+@agent
+def react(
+    *,
+    name: str | None = None,
+    description: str | None = None,
+    prompt: str | AgentPrompt | None = AgentPrompt(),
+    tools: list[Tool] | None = None,
+    model: str | Model | Agent | None = None,
+    attempts: int | AgentAttempts = 1,
+    submit: AgentSubmit = AgentSubmit(),
+    on_continue: str | AgentContinue | None = None,
+) -> Agent:
+    """Extensible ReAct agent based on the paper [ReAct: Synergizing Reasoning and Acting in Language Models](https://arxiv.org/abs/2210.03629).
+
+    Provide a `name` and `description` for the agent if you plan on using it
+    in a multi-agent system (this is so other agents can clearly identify
+    its name and purpose). These fields are not required when using `react()`
+    as a top-level solver.
+
+    The agent runs a tool use loop until the model submits an answer using the
+    `submit()` tool. Use `instructions` to tailor the agent's system message
+    (the default `instructions` provides a basic ReAct prompt).
+
+    Use the `attempts` option to enable additional submissions if the initial
+    submission(s) are incorrect (by default, no additional attempts are permitted).
+
+    By default, the model will be urged to continue if it fails to call
+    a tool. Customise this behavior using the `on_continue` option.
+
+    Args:
+       name: Agent name (required when using with `handoff()` or `as_tool()`)
+       description: Agent description (required when using with `handoff()` or `as_tool()`)
+       prompt: Prompt for agent. Includes agent-specific contextual `instructions`
+          as well as an optional `assistant_prompt` and `handoff_prompt` (for agents
+          that use handoffs). both are provided by default but can be removed or
+          customized). Pass `str` to specify the instructions and use the defaults
+          for handoff and prompt messages.
+       tools: Tools available for the agent.
+       model: Model to use for agent (defaults to currently evaluated model).
+       attempts: Configure agent to make multiple attempts.
+       submit: Configure submit tool used by agent.
+       on_continue: Message to play back to the model to urge it to continue.
+          Optionally, can also be an async function to call to determine whether
+          the loop should continue (executed on every turn) and what message
+          to play back.
+
+    Returns:
+        ReAct agent.
+    """
+    # resolve prompt / system message
+    prompt = AgentPrompt(prompt) if isinstance(prompt, str) else prompt
+    if prompt:
+        prompt_lines: list[str] = []
+        if prompt.instructions:
+            prompt_lines.append(prompt.instructions)
+        if prompt.handoff_prompt and has_handoff(tools):
+            prompt_lines.append(prompt.handoff_prompt)
+        if prompt.assistant_prompt:
+            prompt_lines.append(prompt.assistant_prompt)
+        prompt_content = "\n\n".join(prompt_lines).format(submit=submit.name)
+        system_message: ChatMessage | None = ChatMessageSystem(content=prompt_content)
+    else:
+        system_message = None
+
+    # resolve on_continue
+    if on_continue is None:
+        on_continue = "If you believe you have completed the task, please call the `submit()` tool with your answer."
+    if isinstance(on_continue, str):
+        no_tools_continue_message = on_continue
+
+        async def no_tools_continue(state: AgentState) -> bool | str:
+            if state.output is None or not state.output.message.tool_calls:
+                return no_tools_continue_message
+            else:
+                return True
+
+        on_continue = no_tools_continue
+
+    # validate that on_continue is async
+    if not is_callable_coroutine(on_continue):
+        raise ValueError("The on_continue function must be async.")
+
+    # resolve attempts
+    attempts = AgentAttempts(attempts) if isinstance(attempts, int) else attempts
+
+    # submission tool
+    @tool
+    def submit_tool() -> Tool:
+        async def execute(answer: str) -> ToolResult:
+            """Submit an answer for evaluation.
+
+            Args:
+              answer (str): Submitted answer
+            """
+            return answer
+
+        return execute
+
+    # helper to see if there is a submit tool call
+    def submitted_answer(tool_calls: list[ToolCall] | None) -> str | None:
+        for tool_call in tool_calls or []:
+            if tool_call.function == submit.name and tool_call.parse_error is None:
+                return str(tool_call.arguments["answer"])
+        return None
+
+    # resolve tools
+    tools = tools or []
+    tools.append(tool_with(submit_tool(), submit.name, submit.description))
+
+    async def execute(state: AgentState) -> AgentState:
+        # prepend system message if we have one
+        if system_message:
+            state.messages.insert(0, system_message)
+
+        # track attempts
+        attempt_count = 0
+
+        # main loop = will terminate after submit (subject to max_attempts)
+        # or if a message or token limit is hit
+        while True:
+            # generate output and append assistant message
+            state = await _agent_generate(model, state, tools)
+
+            # check for context window overflow
+            if state.output.stop_reason == "model_length":
+                from inspect_ai.log._transcript import transcript
+
+                transcript().info("Agent terminated: model context window exceeded")
+                break
+
+            # check for a submission
+            answer = submitted_answer(state.output.message.tool_calls)
+            if answer is not None:
+                # remove the tool call and set the output to the answer for scoring
+                state.output.message.tool_calls = None
+                state.output.completion = (
+                    f"{state.output.completion}\n\n{answer}".strip()
+                )
+
+                # exit if we are at max_attempts
+                attempt_count += 1
+                if attempt_count >= attempts.attempts:
+                    break
+
+                # exit if the submission is successful
+                answer_scores = await score(state)
+                if attempts.score_value(answer_scores[0].value) == 1.0:
+                    break
+
+                # otherwise notify the model that it was incorrect and continue
+                else:
+                    if callable(attempts.incorrect_message):
+                        if not is_callable_coroutine(attempts.incorrect_message):
+                            raise ValueError(
+                                "The incorrect_message function must be async."
+                            )
+                        response_message: str = await attempts.incorrect_message(
+                            state, answer_scores
+                        )
+                    else:
+                        response_message = attempts.incorrect_message
+
+                    state.messages.append(ChatMessageUser(content=response_message))
+
+            # no submitted answer, call tools and evaluate whether we should continue
+            else:
+                if state.output.message.tool_calls:
+                    # call tool functions
+                    messages, output = await execute_tools(state.messages, tools)
+                    state.messages.extend(messages)
+                    if output:
+                        state.output = output
+
+                # check if we should continue....
+                do_continue = await on_continue(state)
+                if isinstance(do_continue, str):
+                    state.messages.append(ChatMessageUser(content=do_continue))
+                elif do_continue is False:
+                    break
+
+        return state
+
+    if name is not None or description is not None:
+        return agent_with(execute, name=name, description=description)
+    else:
+        return execute
+
+
+async def _agent_generate(
+    model: str | Model | Agent | None, state: AgentState, tools: list[Tool]
+) -> AgentState:
+    # convert model to agent
+    if isinstance(model, str | Model) or model is None:
+        model = _model_generate(model)
+
+    # confirm we have a tools param
+    agent_tool_info = parse_tool_info(model)
+    if "tools" not in agent_tool_info.parameters.properties:
+        raise ValueError(
+            "Agent passed as model for react agent must have a tools parameter."
+        )
+
+    # call the agent
+    return await model(state, tools)
+
+
+def _model_generate(model: str | Model | None) -> Agent:
+    async def generate(state: AgentState, tools: list[Tool]) -> AgentState:
+        state.output = await get_model(model).generate(state.messages, tools)
+        state.messages.append(state.output.message)
+        return state
+
+    return generate

inspect_ai/agent/_run.py

@@ -0,0 +1,36 @@
+from copy import copy
+from typing import Any
+
+from inspect_ai.model._chat_message import ChatMessage, ChatMessageUser
+
+from ._agent import Agent, AgentState
+
+
+async def run(
+    agent: Agent, input: str | list[ChatMessage] | AgentState, **agent_kwargs: Any
+) -> AgentState:
+    """Run an agent.
+
+    The input messages(s) will be copied prior to running so are
+    not modified in place.
+
+    Args:
+        agent: Agent to run.
+        input: Agent input (string, list of messages, or an `AgentState`).
+        **agent_kwargs: Additional arguments to pass to agent.
+
+    Returns:
+        AgentState: Messages and generated output.
+    """
+    # copy input so we don't mutate it in place
+    input = copy(input)
+
+    # resolve str
+    if isinstance(input, str):
+        input = [ChatMessageUser(content=input)]
+
+    # create state
+    state = AgentState(messages=input) if isinstance(input, list) else input
+
+    # run the agent
+    return await agent(state, **agent_kwargs)

inspect_ai/agent/_types.py

@@ -0,0 +1,81 @@
+from typing import Awaitable, Callable, NamedTuple, TypeAlias
+
+from inspect_ai.agent._agent import AgentState
+from inspect_ai.scorer._metric import Score, ValueToFloat, value_to_float
+
+DEFAULT_HANDOFF_PROMPT = """
+You are part of a multi-agent system designed to make agent coordination and
+execution easy. Agents uses two primary abstraction: **Agents** and **Handoffs**.
+An agent encompasses instructions and tools and can hand off a conversation to
+another agent when appropriate. Handoffs are achieved by calling a handoff function,
+generally named `transfer_to_<agent_name>`. Transfers between agents are handled
+seamlessly in the background; do not mention or draw attention to these transfers
+in your conversation with the user.
+"""
+
+
+DEFAULT_ASSISTANT_PROMPT = """
+You are a helpful assistant attempting to submit the best possible answer.
+You have several tools available to help with finding the answer. You will
+see the result of tool calls right after sending the message. If you need
+to perform multiple actions, you can always send more messages with additional
+tool calls. Do some reasoning before your actions, describing what tool calls
+you are going to use and how they fit into your plan.
+
+When you have completed the task and have an answer, call the {submit}()
+tool to report it.
+"""
+
+
+class AgentPrompt(NamedTuple):
+    """Prompt for agent."""
+
+    instructions: str | None = None
+    """Agent-specific contextual instructions."""
+
+    handoff_prompt: str | None = DEFAULT_HANDOFF_PROMPT
+    """Prompt used when there are additional handoff agents active."""
+
+    assistant_prompt: str | None = DEFAULT_ASSISTANT_PROMPT
+    """Prompt for assistant (covers tool use, submit tool, CoT, etc.)."""
+
+
+AgentContinue: TypeAlias = Callable[[AgentState], Awaitable[bool | str]]
+"""Function called to determine whether the agent should continue.
+
+Returns `True` to continue (with no additional messages inserted),
+return `False` to stop. Returns `str` to continue with an additional
+custom user message inserted.
+"""
+
+
+class AgentAttempts(NamedTuple):
+    """Configure a react agent to make multiple attempts.
+
+    Submissions are evaluated using the task's main scorer, with value of 1.0
+    indicating a correct answer. Scorer values are converted to float (e.g.
+    "C" becomes 1.0) using the standard value_to_float() function. Provide an
+    alternate conversion scheme as required via `score_value`.
+    """
+
+    attempts: int = 1
+    """Maximum number of attempts."""
+
+    incorrect_message: str | Callable[[AgentState, list[Score]], Awaitable[str]] = (
+        "Your submission was incorrect. Please proceed and attempt to find the correct answer."
+    )
+    """User message reply for an incorrect submission from the model. Alternatively,
+    an async function which returns a message."""
+
+    score_value: ValueToFloat = value_to_float()
+    """Function used to extract float from scores (defaults to standard value_to_float())"""
+
+
+class AgentSubmit(NamedTuple):
+    """Configure the submit tool of a react agent."""
+
+    name: str = "submit"
+    """Name for submit tool."""
+
+    description: str = "Submit an answer for evaluation."
+    """Description of submit tool."""