inspect-ai 0.3.59__py3-none-any.whl → 0.3.60__py3-none-any.whl

inspect_ai/model/_providers/together.py

@@ -24,13 +24,13 @@ from .._model_output import (
     ModelOutput,
     ModelUsage,
     StopReason,
+    as_stop_reason,
 )
+from .._openai import chat_message_assistant_from_openai
 from .openai import (
     OpenAIAPI,
-    chat_message_assistant,
 )
 from .util import (
-    as_stop_reason,
     chat_api_input,
     chat_api_request,
     environment_prerequisite_error,
@@ -68,7 +68,7 @@ def chat_choices_from_response_together(
         logprobs_models.append(Logprobs(content=logprobs_sequence))
     return [
         ChatCompletionChoice(
-            message=chat_message_assistant(choice.message, tools),
+            message=chat_message_assistant_from_openai(choice.message, tools),
             stop_reason=as_stop_reason(choice.finish_reason),
             logprobs=logprobs,
         )
@@ -99,7 +99,7 @@ class TogetherAIAPI(OpenAIAPI):
 
     # Together uses a default of 512 so we bump it up
     @override
-    def max_tokens(self) -> int:
+    def max_tokens(self) -> int | None:
         return DEFAULT_MAX_TOKENS
 
     @override

inspect_ai/model/_providers/util/__init__.py

@@ -1,3 +1,5 @@
+from ..._call_tools import parse_tool_call, tool_parse_error_message
+from ..._model_output import as_stop_reason
 from .chatapi import (
     ChatAPIHandler,
     ChatAPIMessage,
@@ -8,11 +10,8 @@ from .chatapi import (
 from .hf_handler import HFHandler
 from .llama31 import Llama31Handler
 from .util import (
-    as_stop_reason,
     environment_prerequisite_error,
     model_base_url,
-    parse_tool_call,
-    tool_parse_error_message,
 )
 
 __all__ = [

inspect_ai/model/_providers/util/hf_handler.py

@@ -8,9 +8,9 @@ from typing_extensions import override
 from inspect_ai.tool._tool_call import ToolCall
 from inspect_ai.tool._tool_info import ToolInfo
 
+from ..._call_tools import parse_tool_call, tool_parse_error_message
 from ..._chat_message import ChatMessageAssistant
 from .chatapi import ChatAPIHandler
-from .util import parse_tool_call, tool_parse_error_message
 
 logger = getLogger(__name__)
 

inspect_ai/model/_providers/util/llama31.py

@@ -9,6 +9,7 @@ from typing_extensions import override
 from inspect_ai.tool._tool_call import ToolCall
 from inspect_ai.tool._tool_info import ToolInfo
 
+from ..._call_tools import parse_tool_call, tool_parse_error_message
 from ..._chat_message import (
     ChatMessage,
     ChatMessageAssistant,
@@ -16,7 +17,6 @@ from ..._chat_message import (
     ChatMessageTool,
 )
 from .chatapi import ChatAPIHandler, ChatAPIMessage
-from .util import parse_tool_call, tool_parse_error_message
 
 logger = getLogger(__name__)
 

inspect_ai/model/_providers/util/util.py

@@ -1,34 +1,11 @@
-import json
 import os
 from logging import getLogger
-from typing import Any
-
-import yaml
 
 from inspect_ai._util.error import PrerequisiteError
-from inspect_ai.tool._tool_call import ToolCall
-from inspect_ai.tool._tool_info import ToolInfo
-
-from ..._model_output import StopReason
 
 logger = getLogger(__name__)
 
 
-def as_stop_reason(reason: str | None) -> StopReason:
-    """Encode common reason strings into standard StopReason."""
-    match reason:
-        case "stop" | "eos":
-            return "stop"
-        case "length":
-            return "max_tokens"
-        case "tool_calls" | "function_call":
-            return "tool_calls"
-        case "content_filter" | "model_length" | "max_tokens":
-            return reason
-        case _:
-            return "unknown"
-
-
 def model_base_url(base_url: str | None, env_vars: str | list[str]) -> str | None:
     if base_url:
         return base_url
@@ -44,59 +21,6 @@ def model_base_url(base_url: str | None, env_vars: str | list[str]) -> str | Non
     return os.getenv("INSPECT_EVAL_MODEL_BASE_URL", None)
 
 
-def tool_parse_error_message(arguments: str, ex: Exception) -> str:
-    return f"Error parsing the following tool call arguments:\n\n{arguments}\n\nError details: {ex}"
-
-
-def parse_tool_call(
-    id: str, function: str, arguments: str, tools: list[ToolInfo]
-) -> ToolCall:
-    error: str | None = None
-    arguments_dict: dict[str, Any] = {}
-
-    def report_parse_error(ex: Exception) -> None:
-        nonlocal error
-        error = tool_parse_error_message(arguments, ex)
-        logger.info(error)
-
-    # if the arguments is a dict, then handle it with a plain json.loads
-    arguments = arguments.strip()
-    if arguments.startswith("{"):
-        try:
-            arguments_dict = json.loads(arguments)
-        except json.JSONDecodeError as ex:
-            report_parse_error(ex)
-
-    # otherwise parse it as yaml (which will pickup unquoted strings, numbers, and true/false)
-    # and then create a dict that maps it to the first function argument
-    else:
-        tool_info = next(
-            (
-                tool
-                for tool in tools
-                if tool.name == function and len(tool.parameters.properties) > 0
-            ),
-            None,
-        )
-        if tool_info:
-            param_names = list(tool_info.parameters.properties.keys())
-            try:
-                value = yaml.safe_load(arguments)
-                arguments_dict[param_names[0]] = value
-            except yaml.error.YAMLError:
-                # If the yaml parser fails, we treat it as a string argument.
-                arguments_dict[param_names[0]] = arguments
-
-    # return ToolCall with error payload
-    return ToolCall(
-        id=id,
-        function=function,
-        arguments=arguments_dict,
-        type="function",
-        parse_error=error,
-    )
-
-
 def environment_prerequisite_error(
     client: str, env_vars: str | list[str]
 ) -> PrerequisiteError:

inspect_ai/scorer/_metric.py

@@ -125,6 +125,9 @@ class SampleScore(BaseModel):
     sample_id: str | int | None = Field(default=None)
     """A sample id"""
 
+    scorer: str | None = Field(default=None)
+    """Registry name of scorer that created this score."""
+
 
 ValueToFloat = Callable[[Value], float]
 """Function used by metrics to translate from a Score value to a float value."""

inspect_ai/scorer/_scorer.py

@@ -1,3 +1,4 @@
+from functools import wraps
 from typing import (
     Any,
     Callable,
@@ -100,7 +101,6 @@ def scorer(
 
     Returns:
         Scorer with registry attributes.
-
     """
 
     def wrapper(scorer_type: Callable[P, Scorer]) -> Callable[P, Scorer]:
@@ -110,6 +110,7 @@ def scorer(
         )
 
         # wrap instantiations of scorer so they carry registry info and metrics
+        @wraps(scorer_type)
         def scorer_wrapper(*args: P.args, **kwargs: P.kwargs) -> Scorer:
             scorer = scorer_type(*args, **kwargs)
 

inspect_ai/solver/__init__.py

@@ -1,6 +1,7 @@
 from inspect_ai._util.deprecation import relocated_module_attribute
 
 from ._basic_agent import basic_agent
+from ._bridge import bridge
 from ._chain import chain
 from ._critique import self_critique
 from ._fork import fork
@@ -14,6 +15,7 @@ from ._use_tools import use_tools
 
 __all__ = [
     "basic_agent",
+    "bridge",
     "human_agent",
     "chain",
     "fork",

inspect_ai/solver/_basic_agent.py

@@ -119,7 +119,7 @@ def basic_agent(
     # resolve tools
     if tools is None:
         tools = []
-    tools = tools if isinstance(tools, Solver) else use_tools(tools)
+    tools = tools if isinstance(tools, Solver) else use_tools(tools, append=True)
 
     # resolve score_value function
     score_value_fn = score_value or value_to_float()

inspect_ai/solver/_bridge/__init__.py

@@ -0,0 +1,3 @@
+from .bridge import bridge
+
+__all__ = ["bridge"]

inspect_ai/solver/_bridge/bridge.py

@@ -0,0 +1,100 @@
+from typing import Any, Awaitable, Callable
+
+from jsonschema import Draft7Validator
+from pydantic import BaseModel, Field, ValidationError
+from pydantic_core import to_json
+
+from inspect_ai._util._async import is_callable_coroutine
+from inspect_ai.model._chat_message import ChatMessage, ChatMessageUser
+from inspect_ai.model._providers.providers import validate_openai_client
+from inspect_ai.scorer._metric import Score
+
+from .._solver import Generate, Solver, solver
+from .._task_state import TaskState
+
+
+@solver
+def bridge(agent: Callable[[dict[str, Any]], Awaitable[dict[str, Any]]]) -> Solver:
+    """Bridge an external agent into an Inspect Solver.
+
+    See documentation at https://inspect.ai-safety-institute.org.uk/agent-bridge.html
+
+    Args:
+      agent: Callable which takes a sample `dict` and returns a result `dict`.
+
+    Returns:
+      Standard Inspect solver.
+    """
+    validate_openai_client("Solver bridge()")
+
+    from openai.types.chat import ChatCompletionMessageParam
+
+    from inspect_ai.model._openai import (
+        chat_messages_from_openai,
+        openai_chat_messages,
+    )
+
+    from .patch import openai_request_to_inspect_model
+
+    class BridgeSample(BaseModel):
+        sample_id: str
+        epoch: int
+        input: list[ChatCompletionMessageParam]
+        metadata: dict[str, Any]
+        target: list[str]
+
+    class BridgeResult(BaseModel):
+        output: str
+        messages: list[ChatCompletionMessageParam] | None = Field(default=None)
+        scores: dict[str, Score] | None = Field(default=None)
+
+    result_schema = BridgeResult.model_json_schema()
+    result_validator = Draft7Validator(result_schema)
+
+    # validate that the agent is an async function
+    if not is_callable_coroutine(agent):
+        raise TypeError(f"'{agent.__name__}' is not declared as an async callable.")
+
+    async def solve(state: TaskState, generate: Generate) -> TaskState:
+        # resolve input to array
+        input: list[ChatMessage] = (
+            [ChatMessageUser(content=state.input)]
+            if isinstance(state.input, str)
+            else state.input
+        )
+
+        # create sample
+        sample = BridgeSample(
+            sample_id=str(state.sample_id),
+            epoch=state.epoch,
+            input=await openai_chat_messages(input, state.model.name),
+            metadata=state.metadata,
+            target=list(state.target),
+        )
+
+        # run target function
+        async with openai_request_to_inspect_model():
+            # call the function
+            result_dict = await agent(sample.model_dump())
+            try:
+                result = BridgeResult.model_validate(result_dict)
+            except ValidationError:
+                # if we fail to validate provide a better human readable error
+                errors = list(result_validator.iter_errors(result_dict))
+                message = "\n".join(
+                    ["Result returned from bridged solver is not valid:"]
+                    + [f" - {error.message}" for error in errors]
+                    + ["", to_json(result_dict, indent=2).decode()]
+                )
+                raise ValueError(message)
+
+        # update and return state
+        state.output.completion = result.output
+        if result.messages is not None:
+            state.messages = chat_messages_from_openai(result.messages)
+        if result.scores is not None:
+            state.scores = result.scores
+
+        return state
+
+    return solve

inspect_ai/solver/_bridge/patch.py

@@ -0,0 +1,170 @@
+import contextlib
+import re
+from contextvars import ContextVar
+from functools import wraps
+from time import time
+from typing import Any, AsyncGenerator, Optional, Type, cast
+
+from openai._base_client import AsyncAPIClient, _AsyncStreamT
+from openai._models import FinalRequestOptions
+from openai._types import ResponseT
+from openai.types.chat import (
+    ChatCompletion,
+    ChatCompletionMessageParam,
+    ChatCompletionToolParam,
+)
+from shortuuid import uuid
+
+from inspect_ai.model._generate_config import GenerateConfig
+from inspect_ai.model._model import get_model
+from inspect_ai.model._openai import (
+    chat_messages_from_openai,
+    openai_chat_choices,
+    openai_completion_usage,
+)
+from inspect_ai.solver._task_state import sample_state
+from inspect_ai.tool._tool_info import ToolInfo
+from inspect_ai.tool._tool_params import ToolParams
+
+
+@contextlib.asynccontextmanager
+async def openai_request_to_inspect_model() -> AsyncGenerator[None, None]:
+    # ensure one time init
+    init_openai_request_patch()
+
+    # set the patch enabled for this context and child coroutines
+    token = _patch_enabled.set(True)
+    try:
+        yield
+    finally:
+        _patch_enabled.reset(token)
+
+
+_patch_initialised: bool = False
+
+_patch_enabled: ContextVar[bool] = ContextVar(
+    "openai_request_patch_enabled", default=False
+)
+
+
+def init_openai_request_patch() -> None:
+    global _patch_initialised
+    if not _patch_initialised:
+        # get reference to original method
+        original_request = getattr(AsyncAPIClient, "request")
+        if original_request is None:
+            raise RuntimeError("Couldn't find 'request' method on AsyncAPIClient")
+
+        @wraps(original_request)
+        async def patched_request(
+            self: AsyncAPIClient,
+            cast_to: Type[ResponseT],
+            options: FinalRequestOptions,
+            *,
+            stream: bool = False,
+            stream_cls: type[_AsyncStreamT] | None = None,
+            remaining_retries: Optional[int] = None,
+        ) -> Any:
+            # we have patched the underlying request method so now need to figure out when to
+            # patch and when to stand down
+            if (
+                # enabled for this coroutine
+                _patch_enabled.get()
+                # completions request
+                and options.url == "/chat/completions"
+                # call to openai not another service (e.g. TogetherAI)
+                and self.base_url == "https://api.openai.com/v1/"
+            ):
+                # must also be an explicit request for an inspect model
+                json_data = cast(dict[str, Any], options.json_data)
+                model_name = str(json_data["model"])
+                if re.match(r"^inspect/?", model_name):
+                    return await inspect_model_request(model_name, options)
+
+            # otherwise just delegate
+            return await original_request(
+                self,
+                cast_to,
+                options,
+                stream=stream,
+                stream_cls=stream_cls,
+                remaining_retries=remaining_retries,
+            )
+
+        setattr(AsyncAPIClient, "request", patched_request)
+
+
+async def inspect_model_request(
+    model_name: str, options: FinalRequestOptions
+) -> ChatCompletion:
+    # convert openai messages to inspect messages
+    json_data = cast(dict[str, Any], options.json_data)
+    messages: list[ChatCompletionMessageParam] = json_data["messages"]
+    input = chat_messages_from_openai(messages)
+
+    # convert openai tools to inspect tools
+    tools: list[ChatCompletionToolParam] = json_data.get("tools", [])
+    inspect_tools: list[ToolInfo] = []
+    for tool in tools:
+        function = tool["function"].copy()
+        inspect_tools.append(
+            ToolInfo(
+                name=function["name"],
+                description=function["description"],
+                parameters=ToolParams.model_validate(function["parameters"]),
+            )
+        )
+
+    # resolve model
+    if model_name == "inspect":
+        model = get_model()
+    else:
+        model = get_model(model_name.removeprefix("inspect/"))
+
+    output = await model.generate(
+        input=input,
+        tools=inspect_tools,
+        config=generate_config_from_openai(options),
+    )
+
+    # if we are using the "default" inspect model for the task, update state.messages
+    if model_name == "inspect":
+        state = sample_state()
+        if state:
+            state.messages = input + [output.choices[0].message]
+
+    # inspect completion to openai completion
+    return ChatCompletion(
+        id=uuid(),
+        created=int(time()),
+        object="chat.completion",
+        choices=openai_chat_choices(output.choices),
+        model=model_name,
+        usage=openai_completion_usage(output.usage) if output.usage else None,
+    )
+
+
+def generate_config_from_openai(options: FinalRequestOptions) -> GenerateConfig:
+    # get options dict
+    json_data = cast(dict[str, Any], options.json_data)
+
+    config = GenerateConfig()
+    config.max_tokens = json_data.get(
+        "max_completion_tokens", json_data.get("max_tokens", None)
+    )
+    config.top_p = json_data.get("top_p", None)
+    config.temperature = json_data.get("temperature", None)
+    stop = json_data.get("stop", None)
+    if stop:
+        config.stop_seqs = [stop] if isinstance(stop, str) else stop
+    config.frequency_penalty = json_data.get("frequency_penalty", None)
+    config.presence_penalty = json_data.get("presence_penalty", None)
+    config.seed = json_data.get("seed", None)
+    config.num_choices = json_data.get("n", None)
+    config.logprobs = json_data.get("logprobs", None)
+    config.top_logprobs = json_data.get("top_logprobs", None)
+    config.logit_bias = json_data.get("logit_bias", None)
+    config.parallel_tool_calls = json_data.get("parallel_tool_calls", None)
+    config.reasoning_effort = json_data.get("reasoning_effort", None)
+
+    return config

inspect_ai/solver/_solver.py

@@ -180,6 +180,7 @@ def solver(
             solver_type, name if name else getattr(solver_type, "__name__")
         )
 
+        @wraps(solver_type)
         def solver_wrapper(*args: P.args, **kwargs: P.kwargs) -> Solver:
             solver = solver_type(*args, **kwargs)
 
@@ -193,6 +194,7 @@ def solver(
             if inspect.isclass(type(solver)):
                 original_call = solver.__call__
 
+                @wraps(original_call)
                 async def call_with_state(
                     state: TaskState, generate: Generate
                 ) -> TaskState:
@@ -225,6 +227,10 @@ def solver(
 
             return registered_solver
 
+        # functools.wraps overrides the return type annotation of the inner function, so
+        # we explicitly set it again
+        solver_wrapper.__annotations__["return"] = Solver
+
         return solver_register(cast(Callable[P, Solver], solver_wrapper), solver_name)
 
     # for decorators with an explicit name, one more wrapper for the name

inspect_ai/util/_display.py

@@ -49,3 +49,8 @@ def display_type() -> DisplayType:
         return _display_type
     else:
         return init_display_type()
+
+
+def display_type_initialized() -> bool:
+    global _display_type
+    return _display_type is not None

inspect_ai/util/_sandbox/docker/prereqs.py

@@ -57,7 +57,7 @@ async def validate_docker_compose(
     version: str = DOCKER_COMPOSE_REQUIRED_VERSION,
 ) -> None:
     def parse_version(stdout: str) -> semver.Version:
-        version = json.loads(stdout)["version"].removeprefix("v")
+        version = json.loads(stdout)["version"].removeprefix("v").split("+")[0]
         return semver.Version.parse(version)
 
     await validate_version(

{inspect_ai-0.3.59.dist-info → inspect_ai-0.3.60.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: inspect_ai
-Version: 0.3.59
+Version: 0.3.60
 Summary: Framework for large language model evaluations
 Author: UK AI Safety Institute
 License: MIT License
@@ -54,6 +54,7 @@ Requires-Dist: aioboto3; extra == "dev"
 Requires-Dist: azure-ai-inference; extra == "dev"
 Requires-Dist: google-cloud-aiplatform; extra == "dev"
 Requires-Dist: google-generativeai; extra == "dev"
+Requires-Dist: goodfire; extra == "dev"
 Requires-Dist: groq; extra == "dev"
 Requires-Dist: ipython; extra == "dev"
 Requires-Dist: mistralai; extra == "dev"
@@ -67,7 +68,7 @@ Requires-Dist: pytest-asyncio; extra == "dev"
 Requires-Dist: pytest-cov; extra == "dev"
 Requires-Dist: pytest-dotenv; extra == "dev"
 Requires-Dist: pytest-xdist; extra == "dev"
-Requires-Dist: ruff==0.9.2; extra == "dev"
+Requires-Dist: ruff==0.9.3; extra == "dev"
 Requires-Dist: textual-dev>=0.86.2; extra == "dev"
 Requires-Dist: types-PyYAML; extra == "dev"
 Requires-Dist: types-beautifulsoup4; extra == "dev"