inspect-ai 0.3.82__py3-none-any.whl → 0.3.84__py3-none-any.whl

inspect_ai/{solver/_human_agent → agent/_human}/state.py

@@ -1,4 +1,4 @@
-from time import time as current_time
+import time as python_time
 
 from pydantic import BaseModel, Field
 
@@ -25,7 +25,7 @@ class HumanAgentState(StoreModel):
         """Set current running state."""
         # if we are flipping to running mode then update started running
         if not self.running_state and running:
-            self.started_running = current_time()
+            self.started_running = python_time.time()
 
         # if we are exiting running mode then update accumulated time
         if self.running_state and not running:
@@ -37,7 +37,7 @@ class HumanAgentState(StoreModel):
     @property
     def time(self) -> float:
         """Total time spend on task."""
-        running_time = current_time() - self.started_running if self.running else 0
+        running_time = python_time.time() - self.started_running if self.running else 0
         return self.accumulated_time + running_time
 
     scorings: list[IntermediateScoring] = Field(default_factory=list)
@@ -50,6 +50,6 @@ class HumanAgentState(StoreModel):
     """Session logs generated by `script` """
 
     # internal state variables used by running and time properties
-    running_state: bool = Field(default=True)
-    started_running: float = Field(default_factory=current_time)
+    running_state: bool = Field(default=False)
+    started_running: float = Field(default_factory=python_time.time)
     accumulated_time: float = Field(default=0.0)

inspect_ai/agent/_react.py

@@ -0,0 +1,241 @@
+from logging import getLogger
+
+from inspect_ai._util._async import is_callable_coroutine
+from inspect_ai.model._call_tools import execute_tools
+from inspect_ai.model._chat_message import (
+    ChatMessage,
+    ChatMessageSystem,
+    ChatMessageUser,
+)
+from inspect_ai.model._model import Model, get_model
+from inspect_ai.scorer._score import score
+from inspect_ai.tool._tool import Tool, ToolResult, tool
+from inspect_ai.tool._tool_call import ToolCall
+from inspect_ai.tool._tool_info import parse_tool_info
+from inspect_ai.tool._tool_with import tool_with
+
+from ._agent import Agent, AgentState, agent, agent_with
+from ._handoff import has_handoff
+from ._types import (
+    AgentAttempts,
+    AgentContinue,
+    AgentPrompt,
+    AgentSubmit,
+)
+
+logger = getLogger(__name__)
+
+
+@agent
+def react(
+    *,
+    name: str | None = None,
+    description: str | None = None,
+    prompt: str | AgentPrompt | None = AgentPrompt(),
+    tools: list[Tool] | None = None,
+    model: str | Model | Agent | None = None,
+    attempts: int | AgentAttempts = 1,
+    submit: AgentSubmit = AgentSubmit(),
+    on_continue: str | AgentContinue | None = None,
+) -> Agent:
+    """Extensible ReAct agent based on the paper [ReAct: Synergizing Reasoning and Acting in Language Models](https://arxiv.org/abs/2210.03629).
+
+    Provide a `name` and `description` for the agent if you plan on using it
+    in a multi-agent system (this is so other agents can clearly identify
+    its name and purpose). These fields are not required when using `react()`
+    as a top-level solver.
+
+    The agent runs a tool use loop until the model submits an answer using the
+    `submit()` tool. Use `instructions` to tailor the agent's system message
+    (the default `instructions` provides a basic ReAct prompt).
+
+    Use the `attempts` option to enable additional submissions if the initial
+    submission(s) are incorrect (by default, no additional attempts are permitted).
+
+    By default, the model will be urged to continue if it fails to call
+    a tool. Customise this behavior using the `on_continue` option.
+
+    Args:
+       name: Agent name (required when using with `handoff()` or `as_tool()`)
+       description: Agent description (required when using with `handoff()` or `as_tool()`)
+       prompt: Prompt for agent. Includes agent-specific contextual `instructions`
+          as well as an optional `assistant_prompt` and `handoff_prompt` (for agents
+          that use handoffs). both are provided by default but can be removed or
+          customized). Pass `str` to specify the instructions and use the defaults
+          for handoff and prompt messages.
+       tools: Tools available for the agent.
+       model: Model to use for agent (defaults to currently evaluated model).
+       attempts: Configure agent to make multiple attempts.
+       submit: Configure submit tool used by agent.
+       on_continue: Message to play back to the model to urge it to continue.
+          Optionally, can also be an async function to call to determine whether
+          the loop should continue (executed on every turn) and what message
+          to play back.
+
+    Returns:
+        ReAct agent.
+    """
+    # resolve prompt / system message
+    prompt = AgentPrompt(prompt) if isinstance(prompt, str) else prompt
+    if prompt:
+        prompt_lines: list[str] = []
+        if prompt.instructions:
+            prompt_lines.append(prompt.instructions)
+        if prompt.handoff_prompt and has_handoff(tools):
+            prompt_lines.append(prompt.handoff_prompt)
+        if prompt.assistant_prompt:
+            prompt_lines.append(prompt.assistant_prompt)
+        prompt_content = "\n\n".join(prompt_lines).format(submit=submit.name)
+        system_message: ChatMessage | None = ChatMessageSystem(content=prompt_content)
+    else:
+        system_message = None
+
+    # resolve on_continue
+    if on_continue is None:
+        on_continue = "If you believe you have completed the task, please call the `submit()` tool with your answer."
+    if isinstance(on_continue, str):
+        no_tools_continue_message = on_continue
+
+        async def no_tools_continue(state: AgentState) -> bool | str:
+            if state.output is None or not state.output.message.tool_calls:
+                return no_tools_continue_message
+            else:
+                return True
+
+        on_continue = no_tools_continue
+
+    # validate that on_continue is async
+    if not is_callable_coroutine(on_continue):
+        raise ValueError("The on_continue function must be async.")
+
+    # resolve attempts
+    attempts = AgentAttempts(attempts) if isinstance(attempts, int) else attempts
+
+    # submission tool
+    @tool
+    def submit_tool() -> Tool:
+        async def execute(answer: str) -> ToolResult:
+            """Submit an answer for evaluation.
+
+            Args:
+              answer (str): Submitted answer
+            """
+            return answer
+
+        return execute
+
+    # helper to see if there is a submit tool call
+    def submitted_answer(tool_calls: list[ToolCall] | None) -> str | None:
+        for tool_call in tool_calls or []:
+            if tool_call.function == submit.name and tool_call.parse_error is None:
+                return str(tool_call.arguments["answer"])
+        return None
+
+    # resolve tools
+    tools = tools or []
+    tools.append(tool_with(submit_tool(), submit.name, submit.description))
+
+    async def execute(state: AgentState) -> AgentState:
+        # prepend system message if we have one
+        if system_message:
+            state.messages.insert(0, system_message)
+
+        # track attempts
+        attempt_count = 0
+
+        # main loop = will terminate after submit (subject to max_attempts)
+        # or if a message or token limit is hit
+        while True:
+            # generate output and append assistant message
+            state = await _agent_generate(model, state, tools)
+
+            # check for context window overflow
+            if state.output.stop_reason == "model_length":
+                from inspect_ai.log._transcript import transcript
+
+                transcript().info("Agent terminated: model context window exceeded")
+                break
+
+            # check for a submission
+            answer = submitted_answer(state.output.message.tool_calls)
+            if answer is not None:
+                # remove the tool call and set the output to the answer for scoring
+                state.output.message.tool_calls = None
+                state.output.completion = (
+                    f"{state.output.completion}\n\n{answer}".strip()
+                )
+
+                # exit if we are at max_attempts
+                attempt_count += 1
+                if attempt_count >= attempts.attempts:
+                    break
+
+                # exit if the submission is successful
+                answer_scores = await score(state)
+                if attempts.score_value(answer_scores[0].value) == 1.0:
+                    break
+
+                # otherwise notify the model that it was incorrect and continue
+                else:
+                    if callable(attempts.incorrect_message):
+                        if not is_callable_coroutine(attempts.incorrect_message):
+                            raise ValueError(
+                                "The incorrect_message function must be async."
+                            )
+                        response_message: str = await attempts.incorrect_message(
+                            state, answer_scores
+                        )
+                    else:
+                        response_message = attempts.incorrect_message
+
+                    state.messages.append(ChatMessageUser(content=response_message))
+
+            # no submitted answer, call tools and evaluate whether we should continue
+            else:
+                if state.output.message.tool_calls:
+                    # call tool functions
+                    messages, output = await execute_tools(state.messages, tools)
+                    state.messages.extend(messages)
+                    if output:
+                        state.output = output
+
+                # check if we should continue....
+                do_continue = await on_continue(state)
+                if isinstance(do_continue, str):
+                    state.messages.append(ChatMessageUser(content=do_continue))
+                elif do_continue is False:
+                    break
+
+        return state
+
+    if name is not None or description is not None:
+        return agent_with(execute, name=name, description=description)
+    else:
+        return execute
+
+
+async def _agent_generate(
+    model: str | Model | Agent | None, state: AgentState, tools: list[Tool]
+) -> AgentState:
+    # convert model to agent
+    if isinstance(model, str | Model) or model is None:
+        model = _model_generate(model)
+
+    # confirm we have a tools param
+    agent_tool_info = parse_tool_info(model)
+    if "tools" not in agent_tool_info.parameters.properties:
+        raise ValueError(
+            "Agent passed as model for react agent must have a tools parameter."
+        )
+
+    # call the agent
+    return await model(state, tools)
+
+
+def _model_generate(model: str | Model | None) -> Agent:
+    async def generate(state: AgentState, tools: list[Tool]) -> AgentState:
+        state.output = await get_model(model).generate(state.messages, tools)
+        state.messages.append(state.output.message)
+        return state
+
+    return generate

inspect_ai/agent/_run.py

@@ -0,0 +1,36 @@
+from copy import copy
+from typing import Any
+
+from inspect_ai.model._chat_message import ChatMessage, ChatMessageUser
+
+from ._agent import Agent, AgentState
+
+
+async def run(
+    agent: Agent, input: str | list[ChatMessage] | AgentState, **agent_kwargs: Any
+) -> AgentState:
+    """Run an agent.
+
+    The input messages(s) will be copied prior to running so are
+    not modified in place.
+
+    Args:
+        agent: Agent to run.
+        input: Agent input (string, list of messages, or an `AgentState`).
+        **agent_kwargs: Additional arguments to pass to agent.
+
+    Returns:
+        AgentState: Messages and generated output.
+    """
+    # copy input so we don't mutate it in place
+    input = copy(input)
+
+    # resolve str
+    if isinstance(input, str):
+        input = [ChatMessageUser(content=input)]
+
+    # create state
+    state = AgentState(messages=input) if isinstance(input, list) else input
+
+    # run the agent
+    return await agent(state, **agent_kwargs)

inspect_ai/agent/_types.py

@@ -0,0 +1,81 @@
+from typing import Awaitable, Callable, NamedTuple, TypeAlias
+
+from inspect_ai.agent._agent import AgentState
+from inspect_ai.scorer._metric import Score, ValueToFloat, value_to_float
+
+DEFAULT_HANDOFF_PROMPT = """
+You are part of a multi-agent system designed to make agent coordination and
+execution easy. Agents uses two primary abstraction: **Agents** and **Handoffs**.
+An agent encompasses instructions and tools and can hand off a conversation to
+another agent when appropriate. Handoffs are achieved by calling a handoff function,
+generally named `transfer_to_<agent_name>`. Transfers between agents are handled
+seamlessly in the background; do not mention or draw attention to these transfers
+in your conversation with the user.
+"""
+
+
+DEFAULT_ASSISTANT_PROMPT = """
+You are a helpful assistant attempting to submit the best possible answer.
+You have several tools available to help with finding the answer. You will
+see the result of tool calls right after sending the message. If you need
+to perform multiple actions, you can always send more messages with additional
+tool calls. Do some reasoning before your actions, describing what tool calls
+you are going to use and how they fit into your plan.
+
+When you have completed the task and have an answer, call the {submit}()
+tool to report it.
+"""
+
+
+class AgentPrompt(NamedTuple):
+    """Prompt for agent."""
+
+    instructions: str | None = None
+    """Agent-specific contextual instructions."""
+
+    handoff_prompt: str | None = DEFAULT_HANDOFF_PROMPT
+    """Prompt used when there are additional handoff agents active."""
+
+    assistant_prompt: str | None = DEFAULT_ASSISTANT_PROMPT
+    """Prompt for assistant (covers tool use, submit tool, CoT, etc.)."""
+
+
+AgentContinue: TypeAlias = Callable[[AgentState], Awaitable[bool | str]]
+"""Function called to determine whether the agent should continue.
+
+Returns `True` to continue (with no additional messages inserted),
+return `False` to stop. Returns `str` to continue with an additional
+custom user message inserted.
+"""
+
+
+class AgentAttempts(NamedTuple):
+    """Configure a react agent to make multiple attempts.
+
+    Submissions are evaluated using the task's main scorer, with value of 1.0
+    indicating a correct answer. Scorer values are converted to float (e.g.
+    "C" becomes 1.0) using the standard value_to_float() function. Provide an
+    alternate conversion scheme as required via `score_value`.
+    """
+
+    attempts: int = 1
+    """Maximum number of attempts."""
+
+    incorrect_message: str | Callable[[AgentState, list[Score]], Awaitable[str]] = (
+        "Your submission was incorrect. Please proceed and attempt to find the correct answer."
+    )
+    """User message reply for an incorrect submission from the model. Alternatively,
+    an async function which returns a message."""
+
+    score_value: ValueToFloat = value_to_float()
+    """Function used to extract float from scores (defaults to standard value_to_float())"""
+
+
+class AgentSubmit(NamedTuple):
+    """Configure the submit tool of a react agent."""
+
+    name: str = "submit"
+    """Name for submit tool."""
+
+    description: str = "Submit an answer for evaluation."
+    """Description of submit tool."""

inspect_ai/log/_log.py

@@ -209,14 +209,16 @@ class EvalSample(BaseModel):
     store: dict[str, Any] = Field(default_factory=dict)
     """State at end of sample execution."""
 
-    def store_as(self, model_cls: Type[SMT]) -> SMT:
+    def store_as(self, model_cls: Type[SMT], instance: str | None = None) -> SMT:
         """Pydantic model interface to the store.
 
         Args:
           model_cls: Pydantic model type (must derive from StoreModel)
+          instance: Optional instances name for store (enables multiple instances
+            of a given StoreModel type within a single sample)
 
         Returns:
-          StoreModel: Instance of model_cls bound to sample store data.
+          StoreModel: model_cls bound to sample store data.
         """
         # un-namespace names for creation
         data = {
@@ -226,6 +228,10 @@ class EvalSample(BaseModel):
         # since we are reading from the log provide a fully detached store
         data["store"] = Store()
 
+        # provide instance if specified
+        if instance is not None:
+            data["instance"] = instance
+
         # create the model
         return model_cls.model_validate(data)
 
@@ -566,6 +572,9 @@ class EvalSpec(BaseModel):
     task_file: str | None = Field(default=None)
     """Task source file."""
 
+    task_registry_name: str | None = Field(default=None)
+    """Task registry name."""
+
     task_attribs: dict[str, Any] = Field(default_factory=dict)
     """Attributes of the @task decorator."""
 

inspect_ai/log/_transcript.py

@@ -14,13 +14,7 @@ from typing import (
     Union,
 )
 
-from pydantic import (
-    BaseModel,
-    ConfigDict,
-    Field,
-    JsonValue,
-    field_serializer,
-)
+from pydantic import BaseModel, ConfigDict, Field, JsonValue, field_serializer
 from shortuuid import uuid
 
 from inspect_ai._util.constants import SAMPLE_SUBTASK
@@ -184,8 +178,8 @@ class ToolEvent(BaseEvent):
     arguments: dict[str, JsonValue]
     """Arguments to function."""
 
-    internal_name: str | None = Field(default=None)
-    """Internal name for tool (if any)."""
+    internal: JsonValue | None = Field(default=None)
+    """Model provider specific payload - typically used to aid transformation back to model types."""
 
     view: ToolCallContent | None = Field(default=None)
     """Custom view of tool call input."""
@@ -208,6 +202,12 @@ class ToolEvent(BaseEvent):
     working_time: float | None = Field(default=None)
     """Working time for tool call (i.e. time not spent waiting on semaphores)."""
 
+    agent: str | None = Field(default=None)
+    """Name of agent if the tool call was an agent handoff."""
+
+    failed: bool | None = Field(default=None)
+    """Did the tool call fail with a hard error?."""
+
     def _set_result(
         self,
         result: ToolResult,
@@ -215,6 +215,8 @@ class ToolEvent(BaseEvent):
         error: ToolCallError | None,
         events: list["Event"],
         waiting_time: float,
+        agent: str | None,
+        failed: bool | None,
     ) -> None:
         self.result = result
         self.truncated = truncated
@@ -224,6 +226,8 @@ class ToolEvent(BaseEvent):
         completed = datetime.now()
         self.completed = completed
         self.working_time = (completed - self.timestamp).total_seconds() - waiting_time
+        self.agent = agent
+        self.failed = failed
 
     # mechanism for operator to cancel the tool call
 

inspect_ai/model/__init__.py

@@ -18,7 +18,7 @@ from ._cache import (
     cache_prune,
     cache_size,
 )
-from ._call_tools import call_tools
+from ._call_tools import ExecuteToolsResult, call_tools, execute_tools
 from ._chat_message import (
     ChatMessage,
     ChatMessageAssistant,
@@ -27,6 +27,7 @@ from ._chat_message import (
     ChatMessageTool,
     ChatMessageUser,
 )
+from ._conversation import ModelConversation
 from ._generate_config import GenerateConfig, GenerateConfigArgs, ResponseSchema
 from ._model import (
     Model,
@@ -34,6 +35,7 @@ from ._model import (
     ModelName,
     get_model,
 )
+from ._model_call import ModelCall
 from ._model_output import (
     ChatCompletionChoice,
     Logprob,
@@ -64,7 +66,9 @@ __all__ = [
     "ChatMessageAssistant",
     "ChatMessageTool",
     "ChatCompletionChoice",
+    "ModelCall",
     "ModelOutput",
+    "ModelConversation",
     "Logprobs",
     "Logprob",
     "TopLogprob",
@@ -74,6 +78,8 @@ __all__ = [
     "ModelUsage",
     "StopReason",
     "call_tools",
+    "execute_tools",
+    "ExecuteToolsResult",
     "cache_clear",
     "cache_list_expired",
     "cache_path",