inspect-ai 0.3.82__py3-none-any.whl → 0.3.83__py3-none-any.whl

inspect_ai/agent/_as_solver.py

@@ -0,0 +1,72 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    from inspect_ai.solver._solver import Solver
+
+from inspect_ai._util.registry import (
+    is_registry_object,
+    registry_unqualified_name,
+)
+from inspect_ai.tool._tool_info import parse_tool_info
+
+from ._agent import Agent, AgentState
+
+
+def as_solver(agent: Agent, **agent_kwargs: Any) -> Solver:
+    """Convert an agent to a solver.
+
+    Note that agents used as solvers will only receive their first parameter
+    (`state`). Any other parameters must provide appropriate defaults
+    or be explicitly specified in `agent_kwargs`
+
+    Args:
+       agent: Agent to convert.
+       **agent_kwargs: Arguments to curry to Agent function (required
+          if the agent has parameters without default values).
+
+    Solver:
+       Solver from agent.
+    """
+    from inspect_ai.solver._solver import Generate, solver
+    from inspect_ai.solver._task_state import TaskState
+
+    # agent must be registered (so we can get its name)
+    if not is_registry_object(agent):
+        raise RuntimeError(
+            "Agent passed to as_solver was not created by an @agent decorated function"
+        )
+    agent_name = registry_unqualified_name(agent)
+
+    # check to make sure we have all the parameters we need to run the agent
+    agent_info = parse_tool_info(agent)
+    for name, param in list(agent_info.parameters.properties.items())[1:]:
+        if param.default is None and name not in agent_kwargs:
+            raise ValueError(
+                f"To use the {agent_name} agent as a solver "
+                + f"you must pass a value for the agent's required '{name}' "
+                + "parameter to the as_solver() function."
+            )
+
+    @solver(name=agent_name)
+    def agent_to_solver() -> Solver:
+        async def solve(state: TaskState, generate: Generate) -> TaskState:
+            # run agent
+            agent_state = await agent(
+                AgentState(messages=state.messages), **agent_kwargs
+            )
+
+            # update messages
+            state.messages = agent_state.messages
+
+            # update output if its not empty
+            if agent_state.output:
+                state.output = agent_state.output
+
+            return state
+
+        # return solver
+        return solve
+
+    return agent_to_solver()

inspect_ai/agent/_as_tool.py

@@ -0,0 +1,122 @@
+from typing import Any
+
+from inspect_ai._util.registry import (
+    is_registry_object,
+    registry_info,
+    registry_unqualified_name,
+)
+from inspect_ai.model._chat_message import ChatMessageAssistant, ChatMessageUser
+from inspect_ai.tool._tool import Tool, ToolResult, tool
+from inspect_ai.tool._tool_def import ToolDef, validate_tool_parameters
+from inspect_ai.tool._tool_info import ToolInfo, parse_tool_info
+from inspect_ai.tool._tool_params import ToolParam
+
+from ._agent import AGENT_DESCRIPTION, Agent, AgentState
+
+
+@tool
+def as_tool(agent: Agent, description: str | None = None, **agent_kwargs: Any) -> Tool:
+    """Convert an agent to a tool.
+
+    By default the model will see all of the agent's arguments as
+    tool arguments (save for `state` which is converted to an `input`
+    arguments of type `str`). Provide optional `agent_kwargs` to mask
+    out agent parameters with default values (these parameters will
+    not be presented to the model as part of the tool interface)
+
+    Args:
+       agent: Agent to convert.
+       description: Tool description (defaults to agent description)
+       **agent_kwargs: Arguments to curry to Agent function (arguments
+          provided here will not be presented to the model as part
+          of the tool interface).
+
+    Returns:
+        Tool from agent.
+    """
+    # agent must be registered (so we can get its name)
+    if not is_registry_object(agent):
+        raise RuntimeError(
+            "Agent passed to as_tool was not created by an @agent decorated function"
+        )
+
+    async def execute(input: str, *args: Any, **kwargs: Any) -> ToolResult:
+        # prepare state and call agent
+        state = AgentState(messages=[ChatMessageUser(content=input)])
+        state = await agent(state, *args, **(agent_kwargs | kwargs))
+
+        # find assistant message to read content from (prefer output)
+        if not state.output.empty:
+            return state.output.message.content
+        elif len(state.messages) > 0 and isinstance(
+            state.messages[-1], ChatMessageAssistant
+        ):
+            return state.messages[-1].content
+        else:
+            return ""
+
+    # get tool_info
+    tool_info = agent_tool_info(agent, description, **agent_kwargs)
+
+    # add "input" param
+    tool_info.parameters.properties = {
+        "input": ToolParam(type="string", description="Input message.")
+    } | tool_info.parameters.properties
+    tool_info.parameters.required.append("input")
+
+    # create tool
+    tool_def = ToolDef(
+        execute,
+        name=tool_info.name,
+        description=tool_info.description,
+        parameters=tool_info.parameters,
+    )
+    return tool_def.as_tool()
+
+
+def agent_tool_info(
+    agent: Agent, description: str | None, **agent_kwargs: Any
+) -> ToolInfo:
+    # get tool_info and name
+    tool_info = parse_tool_info(agent)
+    tool_info.name = registry_unqualified_name(agent)
+
+    # remove "state" param
+    def remove_param(param: str) -> None:
+        if param in tool_info.parameters.properties:
+            del tool_info.parameters.properties[param]
+        if param in tool_info.parameters.required:
+            tool_info.parameters.required.remove(param)
+
+    remove_param("state")
+
+    # validate and remove curried params
+    for agent_param in agent_kwargs.keys():
+        if agent_param in tool_info.parameters.properties:
+            remove_param(agent_param)
+        else:
+            raise ValueError(
+                f"Agent {tool_info.name} does not have a '{agent_param}' parameter."
+            )
+
+    # resolve and validate description. the description in the call takes
+    # precedence, then any @agent(description="<foo>"), and finally any
+    # doc comment on the agent's execute function
+    reg_info = registry_info(agent)
+    tool_info.description = (
+        description
+        or reg_info.metadata.get(AGENT_DESCRIPTION, None)
+        or tool_info.description
+    )
+    if len(tool_info.description) == 0:
+        raise ValueError(
+            f"Description not provided for agent function '{tool_info.name}'. Provide a "
+            + "description either via @agent(description='<description>'), the description "
+            + "argument to as_tool() or handoff(), or via a doc comment on the agent's "
+            + "execute function."
+        )
+
+    # validate parameter descriptions and types
+    validate_tool_parameters(tool_info.name, tool_info.parameters.properties)
+
+    return tool_info

inspect_ai/{solver → agent}/_bridge/bridge.py

@@ -5,17 +5,15 @@ from pydantic import BaseModel, Field, ValidationError
 from pydantic_core import to_json
 
 from inspect_ai._util._async import is_callable_coroutine
-from inspect_ai.model._chat_message import ChatMessage, ChatMessageUser
+from inspect_ai.agent._agent import Agent, AgentState, agent
+from inspect_ai.model._model import get_model
+from inspect_ai.model._model_output import ModelOutput
 from inspect_ai.model._providers.providers import validate_openai_client
-from inspect_ai.scorer._metric import Score
 
-from .._solver import Generate, Solver, solver
-from .._task_state import TaskState
 
-
-@solver
-def bridge(agent: Callable[[dict[str, Any]], Awaitable[dict[str, Any]]]) -> Solver:
-    """Bridge an external agent into an Inspect Solver.
+@agent
+def bridge(agent: Callable[[dict[str, Any]], Awaitable[dict[str, Any]]]) -> Agent:
+    """Bridge an external agent into an Inspect Agent.
 
     See documentation at <https://inspect.aisi.org.uk/agent-bridge.html>
 
@@ -25,7 +23,7 @@ def bridge(agent: Callable[[dict[str, Any]], Awaitable[dict[str, Any]]]) -> Solv
     Returns:
       Standard Inspect solver.
     """
-    validate_openai_client("Solver bridge()")
+    validate_openai_client("Agent bridge()")
 
     from openai.types.chat import ChatCompletionMessageParam
 
@@ -36,17 +34,15 @@ def bridge(agent: Callable[[dict[str, Any]], Awaitable[dict[str, Any]]]) -> Solv
 
     from .patch import openai_request_to_inspect_model
 
-    class BridgeSample(BaseModel):
-        sample_id: str
-        epoch: int
+    class BridgeInput(BaseModel):
+        messages: list[ChatCompletionMessageParam]
+
+        # temporarily here for backward compatibility w/ previous bridge
         input: list[ChatCompletionMessageParam]
-        metadata: dict[str, Any]
-        target: list[str]
 
     class BridgeResult(BaseModel):
         output: str
         messages: list[ChatCompletionMessageParam] | None = Field(default=None)
-        scores: dict[str, Score] | None = Field(default=None)
 
     result_schema = BridgeResult.model_json_schema()
     result_validator = Draft7Validator(result_schema)
@@ -55,27 +51,15 @@ def bridge(agent: Callable[[dict[str, Any]], Awaitable[dict[str, Any]]]) -> Solv
     if not is_callable_coroutine(agent):
         raise TypeError(f"'{agent.__name__}' is not declared as an async callable.")
 
-    async def solve(state: TaskState, generate: Generate) -> TaskState:
-        # resolve input to array
-        input: list[ChatMessage] = (
-            [ChatMessageUser(content=state.input)]
-            if isinstance(state.input, str)
-            else state.input
-        )
-
-        # create sample (use standard gpt-4 message encoding -- i.e. no 'developer' messages)
-        sample = BridgeSample(
-            sample_id=str(state.sample_id),
-            epoch=state.epoch,
-            input=await openai_chat_messages(input, model="gpt-4"),
-            metadata=state.metadata,
-            target=list(state.target),
-        )
+    async def execute(state: AgentState) -> AgentState:
+        # create input (use standard gpt-4 message encoding -- i.e. no 'developer' messages)
+        messages = await openai_chat_messages(state.messages, model="gpt-4")
+        input = BridgeInput(messages=messages, input=messages)
 
         # run target function
         async with openai_request_to_inspect_model():
             # call the function
-            result_dict = await agent(sample.model_dump())
+            result_dict = await agent(input.model_dump())
             try:
                 result = BridgeResult.model_validate(result_dict)
             except ValidationError:
@@ -89,12 +73,14 @@ def bridge(agent: Callable[[dict[str, Any]], Awaitable[dict[str, Any]]]) -> Solv
                 raise ValueError(message)
 
         # update and return state
-        state.output.completion = result.output
+        state.output = ModelOutput.from_content(
+            model=get_model().name, content=result.output
+        )
         if result.messages is not None:
-            state.messages = chat_messages_from_openai(result.messages)
-        if result.scores is not None:
-            state.scores = result.scores
+            state.messages = chat_messages_from_openai(
+                state.output.model, result.messages
+            )
 
         return state
 
-    return solve
+    return execute

inspect_ai/{solver → agent}/_bridge/patch.py

@@ -23,7 +23,6 @@ from inspect_ai.model._openai import (
     openai_chat_choices,
     openai_completion_usage,
 )
-from inspect_ai.solver._task_state import sample_state
 from inspect_ai.tool._tool_choice import ToolChoice, ToolFunction
 from inspect_ai.tool._tool_info import ToolInfo
 from inspect_ai.tool._tool_params import ToolParams
@@ -98,10 +97,18 @@ def init_openai_request_patch() -> None:
 async def inspect_model_request(
     model_name: str, options: FinalRequestOptions
 ) -> ChatCompletion:
+    from inspect_ai.solver._task_state import sample_state
+
+    # resolve model
+    if model_name == "inspect":
+        model = get_model()
+    else:
+        model = get_model(model_name.removeprefix("inspect/"))
+
     # convert openai messages to inspect messages
     json_data = cast(dict[str, Any], options.json_data)
     messages: list[ChatCompletionMessageParam] = json_data["messages"]
-    input = chat_messages_from_openai(messages)
+    input = chat_messages_from_openai(model.api.model_name, messages)
 
     # convert openai tools to inspect tools
     tools: list[ChatCompletionToolParam] = json_data.get("tools", [])
@@ -130,12 +137,6 @@ async def inspect_model_request(
             case _:
                 inspect_tool_choice = ToolFunction(name=tool_choice["function"]["name"])
 
-    # resolve model
-    if model_name == "inspect":
-        model = get_model()
-    else:
-        model = get_model(model_name.removeprefix("inspect/"))
-
     output = await model.generate(
         input=input,
         tools=inspect_tools,

inspect_ai/agent/_filter.py

@@ -0,0 +1,46 @@
+from typing import Awaitable, Callable
+
+from inspect_ai.model._chat_message import (
+    ChatMessage,
+    ChatMessageAssistant,
+    ChatMessageTool,
+)
+
+MessageFilter = Callable[[list[ChatMessage]], Awaitable[list[ChatMessage]]]
+"""Filter messages sent to or received from agent handoffs."""
+
+
+async def remove_tools(messages: list[ChatMessage]) -> list[ChatMessage]:
+    """Remove tool calls from messages.
+
+    Removes all instances of `ChatMessageTool` as well as the `tool_calls`
+    field from `ChatMessageAssistant`.
+
+    Args:
+       messages: Messages to remove tool calls from.
+
+    Returns:
+       Messages without tool calls.
+    """
+    filtered: list[ChatMessage] = []
+    for message in messages:
+        if isinstance(message, ChatMessageTool):
+            continue
+        if isinstance(message, ChatMessageAssistant):
+            message = message.model_copy(update=dict(tool_calls=None))
+        filtered.append(message)
+
+    return filtered
+
+
+async def last_message(messages: list[ChatMessage]) -> list[ChatMessage]:
+    """Remove all but the last message.
+
+    Args:
+       messages: Target messages.
+
+    Returns:
+       List containing only the last message from the input list.
+
+    """
+    return messages[-1:]

inspect_ai/agent/_handoff.py

@@ -0,0 +1,93 @@
+from typing import Any
+
+from inspect_ai._util.registry import (
+    RegistryInfo,
+    is_registry_object,
+    registry_unqualified_name,
+    set_registry_info,
+)
+from inspect_ai.tool._tool import Tool, ToolResult
+from inspect_ai.tool._tool_description import ToolDescription, set_tool_description
+
+from ._agent import Agent
+from ._as_tool import agent_tool_info
+from ._filter import MessageFilter
+
+
+def handoff(
+    agent: Agent,
+    description: str | None = None,
+    input_filter: MessageFilter | None = None,
+    output_filter: MessageFilter | None = None,
+    tool_name: str | None = None,
+    **agent_kwargs: Any,
+) -> Tool:
+    """Create a tool that enables models to handoff to agents.
+
+    Args:
+        agent: Agent to hand off to.
+        description: Handoff tool description (defaults to agent description)
+        input_filter: Filter to modify the message history before calling the tool.
+            Use the built-in `remove_tools` filter to remove all tool calls
+            or alternatively specify a custom `MessageFilter` function.
+        output_filter: Filter to modify the message history after calling the tool.
+            Use the built-in `last_message` filter to return only the last message
+            or alternatively specify a custom `MessageFilter` function.
+        tool_name: Alternate tool name (defaults to `transfer_to_{agent_name}`)
+        **agent_kwargs: Arguments to curry to `Agent` function (arguments provided here
+            will not be presented to the model as part of the tool interface).
+
+    Returns:
+        Tool for handing off to the agent (must be called using `execute_tools()` to be
+        properly handled)
+    """
+    # agent must be registered (so we can get its name)
+    if not is_registry_object(agent):
+        raise RuntimeError(
+            "Agent passed to as_tool was not created by an @agent decorated function"
+        )
+
+    # get tool_info
+    tool_info = agent_tool_info(agent, description, **agent_kwargs)
+
+    # AgentTool calls will be intercepted by execute_tools
+    agent_tool = AgentTool(agent, input_filter, output_filter, **agent_kwargs)
+    tool_name = tool_name or f"transfer_to_{tool_info.name}"
+    set_registry_info(agent_tool, RegistryInfo(type="tool", name=tool_name))
+    set_tool_description(
+        agent_tool,
+        ToolDescription(
+            name=tool_name,
+            description=tool_info.description,
+            parameters=tool_info.parameters,
+        ),
+    )
+    return agent_tool
+
+
+class AgentTool(Tool):
+    def __init__(
+        self,
+        agent: Agent,
+        input_filter: MessageFilter | None = None,
+        output_filter: MessageFilter | None = None,
+        **kwargs: Any,
+    ):
+        self.agent = agent
+        self.input_filter = input_filter
+        self.output_filter = output_filter
+        self.kwargs = kwargs
+
+    @property
+    def __name__(self) -> str:
+        return registry_unqualified_name(self.agent)
+
+    async def __call__(self) -> ToolResult:
+        raise RuntimeError("AgentTool should not be called directly")
+
+
+def has_handoff(tools: list[Tool] | None) -> bool:
+    if tools:
+        return any([isinstance(tool, AgentTool) for tool in tools])
+    else:
+        return False

inspect_ai/{solver/_human_agent → agent/_human}/agent.py

@@ -5,8 +5,7 @@ import anyio
 from inspect_ai.util import display_type, input_panel, sandbox
 from inspect_ai.util._sandbox.events import SandboxEnvironmentProxy
 
-from .._solver import Generate, Solver, solver
-from .._task_state import TaskState
+from .._agent import Agent, AgentState, agent
 from .commands import human_agent_commands
 from .install import install_human_agent
 from .panel import HumanAgentPanel
@@ -14,15 +13,15 @@ from .service import run_human_agent_service
 from .view import ConsoleView, HumanAgentView
 
 
-@solver
-def human_agent(
+@agent
+def human_cli(
     answer: bool | str = True,
     intermediate_scoring: bool = False,
     record_session: bool = True,
-) -> Solver:
-    """Human solver for agentic tasks that run in a Linux environment.
+) -> Agent:
+    """Human CLI agent for tasks that run in a sandbox.
 
-    The Human agent solver installs agent task tools in the default
+    The Human CLI agent installs agent task tools in the default
     sandbox and presents the user with both task instructions and
     documentation for the various tools (e.g. `task submit`,
     `task start`, `task stop` `task instructions`, etc.). A human agent panel
@@ -40,12 +39,12 @@ def human_agent(
        record_session: Record all user commands and outputs in the sandbox bash session.
 
     Returns:
-       Solver: Human agent solver.
+       Agent: Human CLI agent.
     """
     # we can only run one human agent interaction at a time (use lock to enforce)
     agent_lock = anyio.Lock()
 
-    async def solve(state: TaskState, generate: Generate) -> TaskState:
+    async def execute(state: AgentState) -> AgentState:
         async with agent_lock:
             # ensure that we have a sandbox to work with
             try:
@@ -58,7 +57,7 @@ def human_agent(
                 )
 
             # helper function to run the agent (called for fullscreen vs. fallback below)
-            async def run_human_agent(view: HumanAgentView) -> TaskState:
+            async def run_human_agent(view: HumanAgentView) -> AgentState:
                 sandbox_proxy = cast(SandboxEnvironmentProxy, sandbox())
                 with sandbox_proxy.no_events():
                     # create agent commands
@@ -67,7 +66,7 @@ def human_agent(
                     )
 
                     # install agent tools
-                    await install_human_agent(state, commands, record_session)
+                    await install_human_agent(commands, record_session)
 
                     # hookup the view ui
                     view.connect(connection)
@@ -82,4 +81,4 @@ def human_agent(
             else:
                 return await run_human_agent(ConsoleView())
 
-    return solve
+    return execute

inspect_ai/{solver/_human_agent → agent/_human}/commands/__init__.py

@@ -1,5 +1,4 @@
-from inspect_ai.solver._task_state import TaskState
-
+from ..._agent import AgentState
 from .clock import StartCommand, StopCommand
 from .command import HumanAgentCommand
 from .instructions import InstructionsCommand
@@ -10,7 +9,7 @@ from .submit import QuitCommand, SubmitCommand, ValidateCommand
 
 
 def human_agent_commands(
-    state: TaskState,
+    state: AgentState,
     answer: bool | str,
     intermediate_scoring: bool,
     record_session: bool,

inspect_ai/{solver/_human_agent → agent/_human}/commands/clock.py

@@ -62,7 +62,9 @@ class StopCommand(HumanAgentCommand):
         return stop
 
 
-def clock_action_event(action: str, state: HumanAgentState) -> None:
+def clock_action_event(
+    action: Literal["start", "stop"], state: HumanAgentState
+) -> None:
     from inspect_ai.log._transcript import transcript
 
     transcript().info(

inspect_ai/{solver/_human_agent → agent/_human}/commands/score.py

@@ -8,13 +8,13 @@ from inspect_ai._util.ansi import render_text
 from inspect_ai.model._model_output import ModelOutput
 from inspect_ai.scorer._score import score
 
-from ..._task_state import TaskState
+from ..._agent import AgentState
 from ..state import HumanAgentState, IntermediateScoring
 from .command import HumanAgentCommand, call_human_agent
 
 
 class ScoreCommand(HumanAgentCommand):
-    def __init__(self, state: TaskState):
+    def __init__(self, state: AgentState):
         self._state = state
 
     @property
@@ -52,9 +52,9 @@ class ScoreCommand(HumanAgentCommand):
         async def score_task(answer: str | None) -> str:
             # make a copy of TaskState, add the answer, then score
             if answer:
-                task_state = deepcopy(self._state)
-                task_state.output = ModelOutput.from_content("human_agent", answer)
-                result = await score(task_state)
+                agent_state = deepcopy(self._state)
+                agent_state.output = ModelOutput.from_content("human_agent", answer)
+                result = await score(agent_state)
             else:
                 result = await score(self._state)
 

inspect_ai/{solver/_human_agent → agent/_human}/install.py

@@ -3,7 +3,6 @@ from textwrap import dedent
 
 from inspect_ai.util import sandbox
 
-from .._task_state import TaskState
 from .commands.command import HumanAgentCommand
 
 INSTALL_DIR = "human_agent_install"
@@ -18,7 +17,7 @@ RECORD_SESSION_DIR = "/var/tmp/user-sessions"
 
 
 async def install_human_agent(
-    state: TaskState, commands: list[HumanAgentCommand], record_session: bool
+    commands: list[HumanAgentCommand], record_session: bool
 ) -> None:
     # see if we have already installed
     if not (await sandbox().exec(["mkdir", HUMAN_AGENT_DIR])).success:
@@ -183,8 +182,12 @@ def human_agent_bashrc(commands: list[HumanAgentCommand], record_session: bool)
     fi
     """).lstrip()
 
+    CLOCK = dedent("""
+    task start
+    """).lstrip()
+
     # return .bashrc
-    return "\n".join([TERMINAL_CHECK, COMMANDS, RECORDING, INSTRUCTIONS])
+    return "\n".join([TERMINAL_CHECK, COMMANDS, RECORDING, INSTRUCTIONS, CLOCK])
 
 
 def human_agent_install_sh() -> str:

inspect_ai/{solver/_human_agent → agent/_human}/service.py

@@ -1,20 +1,24 @@
+from inspect_ai.agent._human.commands.clock import clock_action_event
 from inspect_ai.model import ModelOutput
 from inspect_ai.util._sandbox import sandbox
 from inspect_ai.util._sandbox.service import sandbox_service
 
-from .._task_state import TaskState
+from .._agent import AgentState
 from .commands.command import HumanAgentCommand
 from .state import HumanAgentState
 from .view import HumanAgentView
 
 
 async def run_human_agent_service(
-    state: TaskState, commands: list[HumanAgentCommand], view: HumanAgentView | None
-) -> TaskState:
+    state: AgentState, commands: list[HumanAgentCommand], view: HumanAgentView | None
+) -> AgentState:
     # initialise agent state
     instructions = "\n\n".join([message.text for message in state.messages]).strip()
     agent_state = HumanAgentState(instructions=instructions)
 
+    # record that clock is stopped
+    clock_action_event("stop", agent_state)
+
     # extract service methods from commands
     methods = {
         command.name: command.service(agent_state)