inspect-ai 0.3.88__py3-none-any.whl → 0.3.90__py3-none-any.whl

inspect_ai/agent/_bridge/bridge.py

@@ -6,6 +6,7 @@ from pydantic_core import to_json
 
 from inspect_ai._util._async import is_callable_coroutine
 from inspect_ai.agent._agent import Agent, AgentState, agent
+from inspect_ai.log._samples import sample_active
 from inspect_ai.model._model import get_model
 from inspect_ai.model._model_output import ModelOutput
 from inspect_ai.model._providers.providers import validate_openai_client
@@ -37,6 +38,10 @@ def bridge(agent: Callable[[dict[str, Any]], Awaitable[dict[str, Any]]]) -> Agen
     class BridgeInput(BaseModel):
         messages: list[ChatCompletionMessageParam]
 
+        # here for backward compatibilty w/ previous bridge
+        # (we may choose to add this to AgentState at some point)
+        metadata: dict[str, Any]
+
         # temporarily here for backward compatibility w/ previous bridge
         input: list[ChatCompletionMessageParam]
 
@@ -53,8 +58,10 @@ def bridge(agent: Callable[[dict[str, Any]], Awaitable[dict[str, Any]]]) -> Agen
 
     async def execute(state: AgentState) -> AgentState:
         # create input (use standard gpt-4 message encoding -- i.e. no 'developer' messages)
-        messages = await openai_chat_messages(state.messages, model="gpt-4")
-        input = BridgeInput(messages=messages, input=messages)
+        sample = sample_active()
+        metadata = (sample.sample.metadata if sample is not None else None) or {}
+        messages = await openai_chat_messages(state.messages)
+        input = BridgeInput(messages=messages, metadata=metadata, input=messages)
 
         # run target function
         async with openai_request_to_inspect_model():

inspect_ai/agent/_react.py

@@ -1,22 +1,27 @@
 from logging import getLogger
+from typing import Literal, cast
 
 from inspect_ai._util._async import is_callable_coroutine
 from inspect_ai.model._call_tools import execute_tools
 from inspect_ai.model._chat_message import (
     ChatMessage,
+    ChatMessageAssistant,
     ChatMessageSystem,
+    ChatMessageTool,
     ChatMessageUser,
 )
 from inspect_ai.model._model import Model, get_model
+from inspect_ai.model._trim import trim_messages
 from inspect_ai.scorer._score import score
 from inspect_ai.tool._tool import Tool, ToolResult, tool
-from inspect_ai.tool._tool_call import ToolCall
 from inspect_ai.tool._tool_info import parse_tool_info
 from inspect_ai.tool._tool_with import tool_with
 
 from ._agent import Agent, AgentState, agent, agent_with
+from ._filter import MessageFilter
 from ._handoff import has_handoff
 from ._types import (
+    DEFAULT_CONTINUE_PROMPT,
     AgentAttempts,
     AgentContinue,
     AgentPrompt,
@@ -37,6 +42,7 @@ def react(
     attempts: int | AgentAttempts = 1,
     submit: AgentSubmit = AgentSubmit(),
     on_continue: str | AgentContinue | None = None,
+    truncation: Literal["auto", "disabled"] | MessageFilter = "disabled",
 ) -> Agent:
     """Extensible ReAct agent based on the paper [ReAct: Synergizing Reasoning and Acting in Language Models](https://arxiv.org/abs/2210.03629).
 
@@ -68,9 +74,16 @@ def react(
        attempts: Configure agent to make multiple attempts.
        submit: Configure submit tool used by agent.
        on_continue: Message to play back to the model to urge it to continue.
-          Optionally, can also be an async function to call to determine whether
-          the loop should continue (executed on every turn) and what message
-          to play back.
+          Use the placeholder {submit} to refer to the submit tool within the message.
+          Alternatively, an async function to call to determine whether the loop
+          should continue and what message to play back. Note that this function
+          is called on _every_ iteration of the loop so if you only want to send
+          a message back when the model fails to call tools you need to code
+          that behavior explicitly.
+       truncation: Truncate the conversation history in the event of a context
+          window overflow. Defaults to "disabled" which does no truncation. Pass
+          "auto" to use `trim_messages()` to reduce the context size. Pass a
+          `MessageFilter` function to do custom truncation.
 
     Returns:
         ReAct agent.
@@ -90,24 +103,6 @@ def react(
     else:
         system_message = None
 
-    # resolve on_continue
-    if on_continue is None:
-        on_continue = "If you believe you have completed the task, please call the `submit()` tool with your answer."
-    if isinstance(on_continue, str):
-        no_tools_continue_message = on_continue
-
-        async def no_tools_continue(state: AgentState) -> bool | str:
-            if state.output is None or not state.output.message.tool_calls:
-                return no_tools_continue_message
-            else:
-                return True
-
-        on_continue = no_tools_continue
-
-    # validate that on_continue is async
-    if not is_callable_coroutine(on_continue):
-        raise ValueError("The on_continue function must be async.")
-
     # resolve attempts
     attempts = AgentAttempts(attempts) if isinstance(attempts, int) else attempts
 
@@ -124,12 +119,17 @@ def react(
 
         return execute
 
-    # helper to see if there is a submit tool call
-    def submitted_answer(tool_calls: list[ToolCall] | None) -> str | None:
-        for tool_call in tool_calls or []:
-            if tool_call.function == submit.name and tool_call.parse_error is None:
-                return str(tool_call.arguments["answer"])
-        return None
+    # helper to extract a submitted answer
+    def submission(tool_results: list[ChatMessage]) -> str | None:
+        return next(
+            (
+                result.text
+                for result in tool_results
+                if isinstance(result, ChatMessageTool)
+                and result.function == submit.name
+            ),
+            None,
+        )
 
     # resolve tools
     tools = tools or []
@@ -140,6 +140,14 @@ def react(
         if system_message:
             state.messages.insert(0, system_message)
 
+        # resolve overflow handling
+        if truncation == "auto":
+            overflow = cast(MessageFilter | None, trim_messages)
+        elif truncation == "disabled":
+            overflow = None
+        else:
+            overflow = truncation
+
         # track attempts
         attempt_count = 0
 
@@ -153,59 +161,95 @@ def react(
             if state.output.stop_reason == "model_length":
                 from inspect_ai.log._transcript import transcript
 
+                if overflow is not None:
+                    previous_messages = state.messages[:-1]
+                    state.messages = await overflow(previous_messages)
+                    if len(state.messages) < len(previous_messages):
+                        transcript().info(
+                            "Agent exceeded model context window, truncating messages and continuing."
+                        )
+                        continue
+
+                # no overflow policy or overflow didn't reduce conversation length
                 transcript().info("Agent terminated: model context window exceeded")
                 break
 
-            # check for a submission
-            answer = submitted_answer(state.output.message.tool_calls)
-            if answer is not None:
-                # remove the tool call and set the output to the answer for scoring
-                state.output.message.tool_calls = None
-                state.output.completion = (
-                    f"{state.output.completion}\n\n{answer}".strip()
-                )
-
-                # exit if we are at max_attempts
-                attempt_count += 1
-                if attempt_count >= attempts.attempts:
-                    break
-
-                # exit if the submission is successful
-                answer_scores = await score(state)
-                if attempts.score_value(answer_scores[0].value) == 1.0:
-                    break
-
-                # otherwise notify the model that it was incorrect and continue
-                else:
-                    if callable(attempts.incorrect_message):
-                        if not is_callable_coroutine(attempts.incorrect_message):
-                            raise ValueError(
-                                "The incorrect_message function must be async."
+            # resolve tool calls (if any)
+            if state.output.message.tool_calls:
+                # call tool functions
+                messages, output = await execute_tools(state.messages, tools)
+                state.messages.extend(messages)
+                if output:
+                    state.output = output
+
+                # check for a submission
+                answer = submission(messages)
+                if answer is not None:
+                    # set the output to the answer for scoring
+                    state.output.completion = (
+                        f"{state.output.completion}\n\n{answer}".strip()
+                    )
+
+                    # exit if we are at max_attempts
+                    attempt_count += 1
+                    if attempt_count >= attempts.attempts:
+                        break
+
+                    # exit if the submission is successful
+                    answer_scores = await score(state)
+                    if attempts.score_value(answer_scores[0].value) == 1.0:
+                        break
+
+                    # otherwise notify the model that it was incorrect and continue
+                    else:
+                        if callable(attempts.incorrect_message):
+                            if not is_callable_coroutine(attempts.incorrect_message):
+                                raise ValueError(
+                                    "The incorrect_message function must be async."
+                                )
+                            response_message: str = await attempts.incorrect_message(
+                                state, answer_scores
+                            )
+                        else:
+                            response_message = attempts.incorrect_message
+
+                        state.messages.append(ChatMessageUser(content=response_message))
+
+            # call the on_continue hook (if any)
+            if callable(on_continue):
+                if not is_callable_coroutine(on_continue):
+                    raise ValueError("The on_continue function must be async.")
+                do_continue = await cast(AgentContinue, on_continue)(state)
+                if do_continue is True:
+                    # if there were no tool calls we need to send back a user message
+                    if not state.output.message.tool_calls:
+                        state.messages.append(
+                            ChatMessageUser(
+                                content=DEFAULT_CONTINUE_PROMPT.format(
+                                    submit=submit.name
+                                )
                             )
-                        response_message: str = await attempts.incorrect_message(
-                            state, answer_scores
                         )
-                    else:
-                        response_message = attempts.incorrect_message
-
-                    state.messages.append(ChatMessageUser(content=response_message))
-
-            # no submitted answer, call tools and evaluate whether we should continue
-            else:
-                if state.output.message.tool_calls:
-                    # call tool functions
-                    messages, output = await execute_tools(state.messages, tools)
-                    state.messages.extend(messages)
-                    if output:
-                        state.output = output
-
-                # check if we should continue....
-                do_continue = await on_continue(state)
-                if isinstance(do_continue, str):
-                    state.messages.append(ChatMessageUser(content=do_continue))
-                elif do_continue is False:
+                elif isinstance(do_continue, str):
+                    state.messages.append(
+                        ChatMessageUser(content=do_continue.format(submit=submit.name))
+                    )
+                else:  # do_continue is False
                     break
 
+            # if there is no on_continue hook then add a user message if there were no tool calls
+            elif not state.output.message.tool_calls:
+                continue_msg = (
+                    DEFAULT_CONTINUE_PROMPT if on_continue is None else str(on_continue)
+                )
+                state.messages.append(
+                    ChatMessageUser(content=continue_msg.format(submit=submit.name))
+                )
+
+        # once we are complete, remove submit tool calls from the history
+        # (as they will potentially confuse parent agents who also have
+        # their own submit tools that they are 'watching' for)
+        state.messages = _remove_submit_tool(state.messages, submit.name)
         return state
 
     if name is not None or description is not None:
@@ -239,3 +283,27 @@ def _model_generate(model: str | Model | None) -> Agent:
         return state
 
     return generate
+
+
+def _remove_submit_tool(
+    messages: list[ChatMessage], submit_name: str
+) -> list[ChatMessage]:
+    filtered: list[ChatMessage] = []
+    for message in messages:
+        # skip submit tool messages
+        if isinstance(message, ChatMessageTool) and message.function == submit_name:
+            continue
+
+        # remove submit tool from assistant messages
+        if isinstance(message, ChatMessageAssistant) and message.tool_calls:
+            tools_calls = [
+                tool_call
+                for tool_call in message.tool_calls
+                if tool_call.function != submit_name
+            ]
+            message = message.model_copy(update=dict(tool_calls=tools_calls))
+
+        # always append message
+        filtered.append(message)
+
+    return filtered

inspect_ai/agent/_run.py

@@ -27,10 +27,21 @@ async def run(
 
     # resolve str
     if isinstance(input, str):
-        input = [ChatMessageUser(content=input)]
+        input_messages: list[ChatMessage] = [
+            ChatMessageUser(content=input, source="input")
+        ]
+    elif isinstance(input, list):
+        input_messages = [
+            message.model_copy(update=dict(source="input")) for message in input
+        ]
+    else:
+        input_messages = [
+            message.model_copy(update=dict(source="input"))
+            for message in input.messages
+        ]
 
     # create state
-    state = AgentState(messages=input) if isinstance(input, list) else input
+    state = AgentState(messages=input_messages)
 
     # run the agent
     return await agent(state, **agent_kwargs)

inspect_ai/agent/_types.py

@@ -40,6 +40,12 @@ class AgentPrompt(NamedTuple):
     """Prompt for assistant (covers tool use, submit tool, CoT, etc.)."""
 
 
+DEFAULT_CONTINUE_PROMPT = """
+Please proceed to the next step using your best judgement. If you believe you
+have completed the task, please call the `{submit}()` tool.
+"""
+
+
 AgentContinue: TypeAlias = Callable[[AgentState], Awaitable[bool | str]]
 """Function called to determine whether the agent should continue.
 

inspect_ai/approval/_apply.py

@@ -2,6 +2,7 @@ from contextvars import ContextVar
 
 from inspect_ai._util.format import format_function_call
 from inspect_ai.approval._approval import Approval
+from inspect_ai.model._chat_message import ChatMessage
 from inspect_ai.tool._tool_call import (
     ToolCall,
     ToolCallContent,
@@ -14,10 +15,11 @@ from ._policy import ApprovalPolicy, policy_approver
 
 
 async def apply_tool_approval(
-    message: str, call: ToolCall, viewer: ToolCallViewer | None
+    message: str,
+    call: ToolCall,
+    viewer: ToolCallViewer | None,
+    history: list[ChatMessage],
 ) -> tuple[bool, Approval | None]:
-    from inspect_ai.solver._task_state import sample_state
-
     approver = _tool_approver.get(None)
     if approver:
         # resolve view
@@ -28,15 +30,12 @@ async def apply_tool_approval(
         else:
             view = default_tool_call_viewer(call)
 
-        # current sample state
-        state = sample_state()
-
         # call approver
         approval = await approver(
             message=message,
             call=call,
             view=view,
-            state=state,
+            history=history,
         )
 
         # process decision

inspect_ai/approval/_approver.py

@@ -1,6 +1,6 @@
 from typing import Protocol
 
-from inspect_ai.solver._task_state import TaskState
+from inspect_ai.model._chat_message import ChatMessage
 from inspect_ai.tool._tool_call import ToolCall, ToolCallView
 
 from ._approval import Approval
@@ -14,7 +14,7 @@ class Approver(Protocol):
         message: str,
         call: ToolCall,
         view: ToolCallView,
-        state: TaskState | None = None,
+        history: list[ChatMessage],
     ) -> Approval:
         """
         Approve or reject a tool call.
@@ -23,7 +23,7 @@ class Approver(Protocol):
             message: Message genreated by the model along with the tool call.
             call: The tool call to be approved.
             view: Custom rendering of tool context and call.
-            state: The current task state, if available.
+            history: The current conversation history.
 
         Returns:
             Approval: An Approval object containing the decision and explanation.

inspect_ai/approval/_auto.py

@@ -1,4 +1,4 @@
-from inspect_ai.solver._task_state import TaskState
+from inspect_ai.model._chat_message import ChatMessage
 from inspect_ai.tool._tool_call import ToolCall, ToolCallView
 
 from ._approval import Approval, ApprovalDecision
@@ -21,7 +21,7 @@ def auto_approver(decision: ApprovalDecision = "approve") -> Approver:
         message: str,
         call: ToolCall,
         view: ToolCallView,
-        state: TaskState | None = None,
+        history: list[ChatMessage],
     ) -> Approval:
         return Approval(decision=decision, explanation="Automatic decision.")
 

inspect_ai/approval/_call.py

@@ -1,20 +1,36 @@
+import inspect
+from logging import getLogger
+
+from inspect_ai._util.logger import warn_once
 from inspect_ai._util.registry import registry_log_name
-from inspect_ai.solver._task_state import TaskState
+from inspect_ai.model._chat_message import ChatMessage
 from inspect_ai.tool._tool_call import ToolCall, ToolCallView
 
 from ._approval import Approval
 from ._approver import Approver
 
+logger = getLogger(__name__)
+
 
 async def call_approver(
     approver: Approver,
     message: str,
     call: ToolCall,
     view: ToolCallView,
-    state: TaskState | None = None,
+    history: list[ChatMessage],
 ) -> Approval:
-    # run approver
-    approval = await approver(message, call, view, state)
+    # run approver (if the approval is still using state then
+    # provide that but issue a warning)
+    signature = inspect.signature(approver)
+    if "state" in signature.parameters.keys():
+        from inspect_ai.solver._task_state import sample_state
+
+        warn_once(
+            logger, "Approver 'state' parameter is deprecated (use 'history' instead)"
+        )
+        approval = await approver(message, call, view, sample_state())  # type: ignore[arg-type]
+    else:
+        approval = await approver(message, call, view, history)
 
     # record
     record_approval(registry_log_name(approver), message, call, view, approval)

inspect_ai/approval/_human/approver.py

@@ -1,4 +1,4 @@
-from inspect_ai.solver._task_state import TaskState
+from inspect_ai.model._chat_message import ChatMessage
 from inspect_ai.tool._tool_call import ToolCall, ToolCallView
 
 from .._approval import Approval, ApprovalDecision
@@ -25,11 +25,11 @@ def human_approver(
         message: str,
         call: ToolCall,
         view: ToolCallView,
-        state: TaskState | None = None,
+        history: list[ChatMessage],
     ) -> Approval:
         # try to use the panel approval (available in fullscreen display)
         try:
-            return await panel_approval(message, call, view, state, choices)
+            return await panel_approval(message, call, view, history, choices)
 
         # fallback to plain console approval (available in all displays)
         except NotImplementedError:

inspect_ai/approval/_human/manager.py

@@ -3,7 +3,7 @@ from contextvars import ContextVar
 from typing import Callable, Literal, NamedTuple
 
 from inspect_ai._util.future import Future
-from inspect_ai.solver._task_state import TaskState
+from inspect_ai.model._chat_message import ChatMessage
 from inspect_ai.tool._tool_call import ToolCall, ToolCallView
 
 from .._approval import Approval, ApprovalDecision
@@ -13,7 +13,7 @@ class ApprovalRequest(NamedTuple):
     message: str
     call: ToolCall
     view: ToolCallView
-    state: TaskState | None
+    history: list[ChatMessage]
     choices: list[ApprovalDecision]
 
 

inspect_ai/approval/_human/panel.py

@@ -10,7 +10,7 @@ from textual.widgets import Button, Static
 from typing_extensions import override
 
 from inspect_ai._util.registry import registry_unqualified_name
-from inspect_ai.solver._task_state import TaskState
+from inspect_ai.model._chat_message import ChatMessage
 from inspect_ai.tool._tool_call import ToolCall, ToolCallView
 from inspect_ai.util._panel import InputPanel, input_panel
 
@@ -29,7 +29,7 @@ async def panel_approval(
     message: str,
     call: ToolCall,
     view: ToolCallView,
-    state: TaskState | None,
+    history: list[ChatMessage],
     choices: list[ApprovalDecision],
 ) -> Approval:
     # ensure the approvals panel is shown
@@ -39,7 +39,7 @@ async def panel_approval(
     approvals = human_approval_manager()
     id = approvals.request_approval(
         ApprovalRequest(
-            message=message, call=call, view=view, state=state, choices=choices
+            message=message, call=call, view=view, history=history, choices=choices
         )
     )
     try:

inspect_ai/approval/_policy.py

@@ -9,7 +9,7 @@ from pydantic import BaseModel, Field, model_validator
 from inspect_ai._util.config import read_config_object
 from inspect_ai._util.format import format_function_call
 from inspect_ai._util.registry import registry_create, registry_lookup
-from inspect_ai.solver._task_state import TaskState
+from inspect_ai.model._chat_message import ChatMessage
 from inspect_ai.tool._tool_call import ToolCall, ToolCallView
 from inspect_ai.util._resource import resource
 
@@ -59,13 +59,13 @@ def policy_approver(policies: str | list[ApprovalPolicy]) -> Approver:
         message: str,
         call: ToolCall,
         view: ToolCallView,
-        state: TaskState | None = None,
+        history: list[ChatMessage],
     ) -> Approval:
         # process approvers for this tool call (continue loop on "escalate")
         has_approver = False
         for approver in tool_approvers(call):
             has_approver = True
-            approval = await call_approver(approver, message, call, view, state)
+            approval = await call_approver(approver, message, call, view, history)
             if approval.decision != "escalate":
                 return approval
 

inspect_ai/log/__init__.py

@@ -19,6 +19,7 @@ from ._log import (
     EvalDataset,
     EvalLog,
     EvalMetric,
+    EvalModelConfig,
     EvalPlan,
     EvalPlanStep,
     EvalResults,
@@ -60,6 +61,7 @@ __all__ = [
     "EvalDataset",
     "EvalLog",
     "EvalMetric",
+    "EvalModelConfig",
     "EvalPlan",
     "EvalPlanStep",
     "EvalResults",

inspect_ai/log/_log.py

@@ -64,7 +64,9 @@ class EvalConfig(BaseModel):
     limit: int | tuple[int, int] | None = Field(default=None)
     """Sample limit (number of samples or range of samples)."""
 
-    sample_id: str | int | list[str | int] | None = Field(default=None)
+    sample_id: str | int | list[str] | list[int] | list[str | int] | None = Field(
+        default=None
+    )
     """Evaluate specific sample(s)."""
 
     epochs: int | None = Field(default=None)
@@ -507,7 +509,7 @@ class EvalDataset(BaseModel):
     samples: int | None = Field(default=None)
     """Number of samples in the dataset."""
 
-    sample_ids: list[int | str] | None = Field(default=None)
+    sample_ids: list[str] | list[int] | list[str | int] | None = Field(default=None)
     """IDs of samples in the dataset."""
 
     shuffled: bool | None = Field(default=None)
@@ -551,6 +553,22 @@ class EvalRevision(BaseModel):
     """Revision commit."""
 
 
+class EvalModelConfig(BaseModel):
+    """Model config."""
+
+    model: str
+    """Model name."""
+
+    config: GenerateConfig = Field(default_factory=GenerateConfig)
+    """Generate config"""
+
+    base_url: str | None = Field(default=None)
+    """Model base url."""
+
+    args: dict[str, Any] = Field(default_factory=dict)
+    """Model specific arguments."""
+
+
 class EvalSpec(BaseModel):
     """Eval target and configuration."""
 
@@ -608,6 +626,9 @@ class EvalSpec(BaseModel):
     model_args: dict[str, Any] = Field(default_factory=dict)
     """Model specific arguments."""
 
+    model_roles: dict[str, EvalModelConfig] | None = Field(default=None)
+    """Model roles."""
+
     config: EvalConfig
     """Configuration values for eval."""