inspect-ai 0.3.58__py3-none-any.whl → 0.3.60__py3-none-any.whl

inspect_ai/scorer/_reducer/reducer.py

@@ -111,7 +111,7 @@ def pass_at(
             if total - correct < k:
                 return 1.0
             else:
-                return 1.0 - cast(
+                return 1.0 - cast(  # type: ignore[redundant-cast]
                     float,
                     np.prod(1.0 - k / np.arange(total - correct + 1, total + 1)).item(),
                 )

inspect_ai/scorer/_scorer.py

@@ -1,3 +1,4 @@
+from functools import wraps
 from typing import (
     Any,
     Callable,
@@ -100,7 +101,6 @@ def scorer(
 
     Returns:
         Scorer with registry attributes.
-
     """
 
     def wrapper(scorer_type: Callable[P, Scorer]) -> Callable[P, Scorer]:
@@ -110,6 +110,7 @@ def scorer(
         )
 
         # wrap instantiations of scorer so they carry registry info and metrics
+        @wraps(scorer_type)
         def scorer_wrapper(*args: P.args, **kwargs: P.kwargs) -> Scorer:
             scorer = scorer_type(*args, **kwargs)
 
@@ -151,8 +152,8 @@ def scorer_metrics(
         return cast(list[Metric | dict[str, list[Metric]]], metrics_raw)
 
 
-def unique_scorer_name(scorer: Scorer, already_used_names: list[str]) -> str:
-    base_name = registry_unqualified_name(scorer)
+def unique_scorer_name(scorer: Scorer | str, already_used_names: list[str]) -> str:
+    base_name = scorer if isinstance(scorer, str) else registry_unqualified_name(scorer)
     scorer_name = base_name
     count = 1
     while scorer_name in already_used_names:

inspect_ai/solver/__init__.py

@@ -1,23 +1,21 @@
 from inspect_ai._util.deprecation import relocated_module_attribute
 
 from ._basic_agent import basic_agent
+from ._bridge import bridge
 from ._chain import chain
 from ._critique import self_critique
 from ._fork import fork
 from ._human_agent.agent import human_agent
 from ._multiple_choice import MultipleChoiceTemplate, multiple_choice
 from ._plan import Plan, plan
-from ._prompt import (
-    chain_of_thought,
-    prompt_template,
-    system_message,
-)
+from ._prompt import chain_of_thought, prompt_template, system_message, user_message
 from ._solver import Generate, Solver, SolverSpec, generate, solver
 from ._task_state import Choice, Choices, TaskState
 from ._use_tools import use_tools
 
 __all__ = [
     "basic_agent",
+    "bridge",
     "human_agent",
     "chain",
     "fork",
@@ -26,6 +24,7 @@ __all__ = [
     "chain_of_thought",
     "multiple_choice",
     "system_message",
+    "user_message",
     "self_critique",
     "use_tools",
     "plan",

inspect_ai/solver/_basic_agent.py

@@ -119,7 +119,7 @@ def basic_agent(
     # resolve tools
     if tools is None:
         tools = []
-    tools = tools if isinstance(tools, Solver) else use_tools(tools)
+    tools = tools if isinstance(tools, Solver) else use_tools(tools, append=True)
 
     # resolve score_value function
     score_value_fn = score_value or value_to_float()

inspect_ai/solver/_bridge/__init__.py

@@ -0,0 +1,3 @@
+from .bridge import bridge
+
+__all__ = ["bridge"]

inspect_ai/solver/_bridge/bridge.py

@@ -0,0 +1,100 @@
+from typing import Any, Awaitable, Callable
+
+from jsonschema import Draft7Validator
+from pydantic import BaseModel, Field, ValidationError
+from pydantic_core import to_json
+
+from inspect_ai._util._async import is_callable_coroutine
+from inspect_ai.model._chat_message import ChatMessage, ChatMessageUser
+from inspect_ai.model._providers.providers import validate_openai_client
+from inspect_ai.scorer._metric import Score
+
+from .._solver import Generate, Solver, solver
+from .._task_state import TaskState
+
+
+@solver
+def bridge(agent: Callable[[dict[str, Any]], Awaitable[dict[str, Any]]]) -> Solver:
+    """Bridge an external agent into an Inspect Solver.
+
+    See documentation at https://inspect.ai-safety-institute.org.uk/agent-bridge.html
+
+    Args:
+      agent: Callable which takes a sample `dict` and returns a result `dict`.
+
+    Returns:
+      Standard Inspect solver.
+    """
+    validate_openai_client("Solver bridge()")
+
+    from openai.types.chat import ChatCompletionMessageParam
+
+    from inspect_ai.model._openai import (
+        chat_messages_from_openai,
+        openai_chat_messages,
+    )
+
+    from .patch import openai_request_to_inspect_model
+
+    class BridgeSample(BaseModel):
+        sample_id: str
+        epoch: int
+        input: list[ChatCompletionMessageParam]
+        metadata: dict[str, Any]
+        target: list[str]
+
+    class BridgeResult(BaseModel):
+        output: str
+        messages: list[ChatCompletionMessageParam] | None = Field(default=None)
+        scores: dict[str, Score] | None = Field(default=None)
+
+    result_schema = BridgeResult.model_json_schema()
+    result_validator = Draft7Validator(result_schema)
+
+    # validate that the agent is an async function
+    if not is_callable_coroutine(agent):
+        raise TypeError(f"'{agent.__name__}' is not declared as an async callable.")
+
+    async def solve(state: TaskState, generate: Generate) -> TaskState:
+        # resolve input to array
+        input: list[ChatMessage] = (
+            [ChatMessageUser(content=state.input)]
+            if isinstance(state.input, str)
+            else state.input
+        )
+
+        # create sample
+        sample = BridgeSample(
+            sample_id=str(state.sample_id),
+            epoch=state.epoch,
+            input=await openai_chat_messages(input, state.model.name),
+            metadata=state.metadata,
+            target=list(state.target),
+        )
+
+        # run target function
+        async with openai_request_to_inspect_model():
+            # call the function
+            result_dict = await agent(sample.model_dump())
+            try:
+                result = BridgeResult.model_validate(result_dict)
+            except ValidationError:
+                # if we fail to validate provide a better human readable error
+                errors = list(result_validator.iter_errors(result_dict))
+                message = "\n".join(
+                    ["Result returned from bridged solver is not valid:"]
+                    + [f" - {error.message}" for error in errors]
+                    + ["", to_json(result_dict, indent=2).decode()]
+                )
+                raise ValueError(message)
+
+        # update and return state
+        state.output.completion = result.output
+        if result.messages is not None:
+            state.messages = chat_messages_from_openai(result.messages)
+        if result.scores is not None:
+            state.scores = result.scores
+
+        return state
+
+    return solve

inspect_ai/solver/_bridge/patch.py

@@ -0,0 +1,170 @@
+import contextlib
+import re
+from contextvars import ContextVar
+from functools import wraps
+from time import time
+from typing import Any, AsyncGenerator, Optional, Type, cast
+
+from openai._base_client import AsyncAPIClient, _AsyncStreamT
+from openai._models import FinalRequestOptions
+from openai._types import ResponseT
+from openai.types.chat import (
+    ChatCompletion,
+    ChatCompletionMessageParam,
+    ChatCompletionToolParam,
+)
+from shortuuid import uuid
+
+from inspect_ai.model._generate_config import GenerateConfig
+from inspect_ai.model._model import get_model
+from inspect_ai.model._openai import (
+    chat_messages_from_openai,
+    openai_chat_choices,
+    openai_completion_usage,
+)
+from inspect_ai.solver._task_state import sample_state
+from inspect_ai.tool._tool_info import ToolInfo
+from inspect_ai.tool._tool_params import ToolParams
+
+
+@contextlib.asynccontextmanager
+async def openai_request_to_inspect_model() -> AsyncGenerator[None, None]:
+    # ensure one time init
+    init_openai_request_patch()
+
+    # set the patch enabled for this context and child coroutines
+    token = _patch_enabled.set(True)
+    try:
+        yield
+    finally:
+        _patch_enabled.reset(token)
+
+
+_patch_initialised: bool = False
+
+_patch_enabled: ContextVar[bool] = ContextVar(
+    "openai_request_patch_enabled", default=False
+)
+
+
+def init_openai_request_patch() -> None:
+    global _patch_initialised
+    if not _patch_initialised:
+        # get reference to original method
+        original_request = getattr(AsyncAPIClient, "request")
+        if original_request is None:
+            raise RuntimeError("Couldn't find 'request' method on AsyncAPIClient")
+
+        @wraps(original_request)
+        async def patched_request(
+            self: AsyncAPIClient,
+            cast_to: Type[ResponseT],
+            options: FinalRequestOptions,
+            *,
+            stream: bool = False,
+            stream_cls: type[_AsyncStreamT] | None = None,
+            remaining_retries: Optional[int] = None,
+        ) -> Any:
+            # we have patched the underlying request method so now need to figure out when to
+            # patch and when to stand down
+            if (
+                # enabled for this coroutine
+                _patch_enabled.get()
+                # completions request
+                and options.url == "/chat/completions"
+                # call to openai not another service (e.g. TogetherAI)
+                and self.base_url == "https://api.openai.com/v1/"
+            ):
+                # must also be an explicit request for an inspect model
+                json_data = cast(dict[str, Any], options.json_data)
+                model_name = str(json_data["model"])
+                if re.match(r"^inspect/?", model_name):
+                    return await inspect_model_request(model_name, options)
+
+            # otherwise just delegate
+            return await original_request(
+                self,
+                cast_to,
+                options,
+                stream=stream,
+                stream_cls=stream_cls,
+                remaining_retries=remaining_retries,
+            )
+
+        setattr(AsyncAPIClient, "request", patched_request)
+
+
+async def inspect_model_request(
+    model_name: str, options: FinalRequestOptions
+) -> ChatCompletion:
+    # convert openai messages to inspect messages
+    json_data = cast(dict[str, Any], options.json_data)
+    messages: list[ChatCompletionMessageParam] = json_data["messages"]
+    input = chat_messages_from_openai(messages)
+
+    # convert openai tools to inspect tools
+    tools: list[ChatCompletionToolParam] = json_data.get("tools", [])
+    inspect_tools: list[ToolInfo] = []
+    for tool in tools:
+        function = tool["function"].copy()
+        inspect_tools.append(
+            ToolInfo(
+                name=function["name"],
+                description=function["description"],
+                parameters=ToolParams.model_validate(function["parameters"]),
+            )
+        )
+
+    # resolve model
+    if model_name == "inspect":
+        model = get_model()
+    else:
+        model = get_model(model_name.removeprefix("inspect/"))
+
+    output = await model.generate(
+        input=input,
+        tools=inspect_tools,
+        config=generate_config_from_openai(options),
+    )
+
+    # if we are using the "default" inspect model for the task, update state.messages
+    if model_name == "inspect":
+        state = sample_state()
+        if state:
+            state.messages = input + [output.choices[0].message]
+
+    # inspect completion to openai completion
+    return ChatCompletion(
+        id=uuid(),
+        created=int(time()),
+        object="chat.completion",
+        choices=openai_chat_choices(output.choices),
+        model=model_name,
+        usage=openai_completion_usage(output.usage) if output.usage else None,
+    )
+
+
+def generate_config_from_openai(options: FinalRequestOptions) -> GenerateConfig:
+    # get options dict
+    json_data = cast(dict[str, Any], options.json_data)
+
+    config = GenerateConfig()
+    config.max_tokens = json_data.get(
+        "max_completion_tokens", json_data.get("max_tokens", None)
+    )
+    config.top_p = json_data.get("top_p", None)
+    config.temperature = json_data.get("temperature", None)
+    stop = json_data.get("stop", None)
+    if stop:
+        config.stop_seqs = [stop] if isinstance(stop, str) else stop
+    config.frequency_penalty = json_data.get("frequency_penalty", None)
+    config.presence_penalty = json_data.get("presence_penalty", None)
+    config.seed = json_data.get("seed", None)
+    config.num_choices = json_data.get("n", None)
+    config.logprobs = json_data.get("logprobs", None)
+    config.top_logprobs = json_data.get("top_logprobs", None)
+    config.logit_bias = json_data.get("logit_bias", None)
+    config.parallel_tool_calls = json_data.get("parallel_tool_calls", None)
+    config.reasoning_effort = json_data.get("reasoning_effort", None)
+
+    return config

inspect_ai/solver/_prompt.py

@@ -2,6 +2,7 @@ from typing import Any
 
 from inspect_ai._util.dict import omit
 from inspect_ai.model import ChatMessageSystem
+from inspect_ai.model._chat_message import ChatMessageUser
 from inspect_ai.util import resource
 
 from ._solver import Generate, Solver, solver
@@ -15,7 +16,8 @@ def prompt_template(template: str, **params: Any) -> Solver:
 
     Prompt template containing a `{prompt}` placeholder and any
     number of additional `params`. All values contained in sample
-    `metadata` are also automatically included in the `params`.
+    `metadata` and `store` are also automatically included in the
+    `params`.
 
     Args:
       template: (str): Template for prompt.
@@ -29,7 +31,7 @@ def prompt_template(template: str, **params: Any) -> Solver:
 
     async def solve(state: TaskState, generate: Generate) -> TaskState:
         prompt = state.user_prompt
-        kwargs = omit(state.metadata, ["prompt"]) | params
+        kwargs = omit(state.metadata | state.store._data, ["prompt"]) | params
         prompt.text = prompt_template.format(prompt=prompt.text, **kwargs)
         return state
 
@@ -41,8 +43,9 @@ def system_message(template: str, **params: Any) -> Solver:
     """Solver which inserts a system message into the conversation.
 
     System message template containing any number of optional `params`.
-    for substitution. All values contained in sample `metadata` are also
-    automatically included in the `params`.
+    for substitution using the `str.format()` method. All values
+    contained in sample `metadata` and `store` are also automatically
+    included in the `params`.
 
     The new message will go after other system messages (if there
     are none it will be inserted at the beginning of the conversation).
@@ -58,7 +61,7 @@ def system_message(template: str, **params: Any) -> Solver:
     content = resource(template)
 
     async def solve(state: TaskState, generate: Generate) -> TaskState:
-        kwargs = state.metadata | params
+        kwargs = state.metadata | state.store._data | params
         append_system_message(
             state.messages, ChatMessageSystem(content=content.format(**kwargs))
         )
@@ -67,6 +70,33 @@ def system_message(template: str, **params: Any) -> Solver:
     return solve
 
 
+@solver
+def user_message(template: str, **params: Any) -> Solver:
+    """Solver which inserts a user message into the conversation.
+
+    User message template containing any number of optional `params`.
+    for substitution using the `str.format()` method. All values
+    contained in sample `metadata` and `store` are also automatically
+    included in the `params`.
+
+    Args:
+      template (str): Template for user message.
+      **params (dict[str,Any]): Parameters to fill into the template.
+
+    Returns:
+      A solver that inserts the parameterised user message.
+    """
+    # read template
+    content = resource(template)
+
+    async def solve(state: TaskState, generate: Generate) -> TaskState:
+        kwargs = state.metadata | state.store._data | params
+        state.messages.append(ChatMessageUser(content=content.format(**kwargs)))
+        return state
+
+    return solve
+
+
 DEFAULT_COT_TEMPLATE = r"""
 {prompt}
 

inspect_ai/solver/_solver.py

@@ -180,6 +180,7 @@ def solver(
             solver_type, name if name else getattr(solver_type, "__name__")
         )
 
+        @wraps(solver_type)
         def solver_wrapper(*args: P.args, **kwargs: P.kwargs) -> Solver:
             solver = solver_type(*args, **kwargs)
 
@@ -193,6 +194,7 @@ def solver(
             if inspect.isclass(type(solver)):
                 original_call = solver.__call__
 
+                @wraps(original_call)
                 async def call_with_state(
                     state: TaskState, generate: Generate
                 ) -> TaskState:
@@ -225,6 +227,10 @@ def solver(
 
             return registered_solver
 
+        # functools.wraps overrides the return type annotation of the inner function, so
+        # we explicitly set it again
+        solver_wrapper.__annotations__["return"] = Solver
+
         return solver_register(cast(Callable[P, Solver], solver_wrapper), solver_name)
 
     # for decorators with an explicit name, one more wrapper for the name

inspect_ai/solver/_task_state.py

@@ -2,8 +2,9 @@ from collections.abc import Sequence
 from contextvars import ContextVar
 from copy import deepcopy
 from dataclasses import dataclass
+from itertools import tee
 from random import Random
-from typing import Any, Type, Union, cast, overload
+from typing import Any, Iterable, SupportsIndex, Type, Union, cast, overload
 
 from pydantic_core import to_jsonable_python
 
@@ -15,9 +16,13 @@ from inspect_ai.model import (
     ModelOutput,
 )
 from inspect_ai.model._call_tools import tools_info
+from inspect_ai.model._chat_message import ChatMessageBase
 from inspect_ai.model._model import sample_total_tokens
+from inspect_ai.scorer._metric import Score
+from inspect_ai.scorer._target import Target
 from inspect_ai.tool import Tool, ToolChoice
 from inspect_ai.tool._tool_def import ToolDef
+from inspect_ai.util._limit import SampleLimitExceededError
 from inspect_ai.util._store import Store, store_jsonable
 from inspect_ai.util._store_model import SMT
 
@@ -136,6 +141,7 @@ class TaskState:
         epoch: int,
         input: str | list[ChatMessage],
         messages: list[ChatMessage],
+        target: Target = Target(""),
         choices: list[str] | None = [],
         output: ModelOutput | None = None,
         message_limit: int | None = None,
@@ -161,10 +167,13 @@ class TaskState:
         or `input_text` only
         """
 
+        self.target = target
+        """The scoring target for this `Sample`."""
+
         self.metadata = metadata
         """Metadata from the `Sample` for this `TaskState`"""
 
-        self.messages = messages
+        self._messages: list[ChatMessage] = ChatMessageList(messages)
         """
         Chat conversation history for sample.
 
@@ -189,9 +198,7 @@ class TaskState:
         """
 
         self._message_limit = message_limit
-        self._message_limit_exceeded = False
         self._token_limit = token_limit
-        self._token_limit_exceeded = False
         self._completed = completed
 
         """Store for shared data"""
@@ -202,6 +209,9 @@ class TaskState:
         else:
             self.choices = Choices([])
 
+        self.scores: dict[str, Score] | None = None
+        """Scores yielded by running task."""
+
     @property
     def model(self) -> ModelName:
         """Name of model being evaluated."""
@@ -254,6 +264,16 @@ class TaskState:
         else:
             raise ValueError("user_prompt requested from TaskState but none available")
 
+    @property
+    def messages(self) -> list[ChatMessage]:
+        """Messages in chat history"""
+        return self._messages
+
+    @messages.setter
+    def messages(self, messages: list[ChatMessage]) -> None:
+        """Set messages in chat history."""
+        self._messages = ChatMessageList(messages)
+
     @property
     def max_messages(self) -> int | None:
         """Deprecated (use message_limit)."""
@@ -300,40 +320,7 @@ class TaskState:
     @property
     def completed(self) -> bool:
         """Is the task completed."""
-        # update messages
-        from inspect_ai.log._samples import set_active_sample_total_messages
-        from inspect_ai.log._transcript import SampleLimitEvent, transcript
-
-        set_active_sample_total_messages(len(self.messages))
-
-        if self._completed:
-            return True
-        elif self.message_limit and len(self.messages) >= self.message_limit:
-            # log if this is the first time we hit this
-            if not self._message_limit_exceeded:
-                self._message_limit_exceeded = True
-                transcript()._event(
-                    SampleLimitEvent(
-                        type="message",
-                        message=f"Sample completed: exceeded message limit ({self.message_limit})",
-                        limit=self.message_limit,
-                    )
-                )
-            return True
-        elif self.token_limit and self.token_usage >= self.token_limit:
-            # log if this is the first time we hit this
-            if not self._token_limit_exceeded:
-                self._token_limit_exceeded = True
-                transcript()._event(
-                    SampleLimitEvent(
-                        type="token",
-                        message=f"Sample completed: exceeded token limit ({self.token_limit:,})",
-                        limit=self.token_limit,
-                    )
-                )
-            return True
-        else:
-            return False
+        return self._completed
 
     @completed.setter
     def completed(self, completed: bool) -> None:
@@ -413,3 +400,58 @@ def state_jsonable(state: TaskState | None = None) -> dict[str, Any]:
 def sample_jsonable(sample: Sample) -> dict[str, Any]:
     jsonable = to_jsonable_python(sample, exclude_none=True, fallback=lambda _x: None)
     return cast(dict[str, Any], deepcopy(jsonable))
+
+
+class ChatMessageList(list[ChatMessage]):
+    def __init__(self, iterable: Iterable[ChatMessage]):
+        items, length = self._iterable_length(iterable)
+        self._check_size(length)
+        super().__init__(items)
+
+    def _check_size(self, additional_items: int = 1) -> None:
+        from inspect_ai.log._samples import active_sample_message_limit
+
+        messages_limit = active_sample_message_limit()
+        if messages_limit is not None:
+            messages = len(self) + additional_items
+            if messages > messages_limit:
+                raise SampleLimitExceededError(
+                    "message", value=messages, limit=messages_limit
+                )
+
+    def append(self, item: ChatMessage) -> None:
+        self._check_size()
+        super().append(item)
+
+    def extend(self, items: Iterable[ChatMessage]) -> None:
+        items, length = self._iterable_length(items)
+        self._check_size(length)
+        super().extend(items)
+
+    def insert(self, index: SupportsIndex, item: ChatMessage) -> None:
+        self._check_size()
+        super().insert(index, item)
+
+    @overload
+    def __setitem__(self, index: SupportsIndex, item: ChatMessage) -> None: ...
+
+    @overload
+    def __setitem__(self, index: slice, item: Iterable[ChatMessage]) -> None: ...
+
+    def __setitem__(
+        self, index: SupportsIndex | slice, item: ChatMessage | Iterable[ChatMessage]
+    ) -> None:
+        if isinstance(index, slice) and not isinstance(item, ChatMessageBase):
+            item, length = self._iterable_length(item)
+            size_change = length - len(self[index])
+            if size_change > 0:
+                self._check_size(size_change)
+
+        super().__setitem__(index, item)  # type: ignore[assignment,index]
+
+    def _iterable_length(
+        self, items: Iterable[ChatMessage]
+    ) -> tuple[Iterable[ChatMessage], int]:
+        items, counter = tee(items)
+        length = sum(1 for _ in counter)
+        return items, length

inspect_ai/tool/__init__.py

@@ -12,6 +12,7 @@ from ._tool_call import (
     ToolCall,
     ToolCallContent,
     ToolCallError,
+    ToolCallModelInput,
     ToolCallView,
     ToolCallViewer,
 )
@@ -42,6 +43,7 @@ __all__ = [
     "ContentVideo",
     "ToolCall",
     "ToolCallContent",
+    "ToolCallModelInput",
     "ToolCallView",
     "ToolCallViewer",
     "ToolChoice",