inspect-ai 0.3.88__py3-none-any.whl → 0.3.89__py3-none-any.whl

inspect_ai/model/_providers/together.py

@@ -10,7 +10,6 @@ from openai.types.chat import (
 from typing_extensions import override
 
 from inspect_ai._util.constants import DEFAULT_MAX_TOKENS
-from inspect_ai.model._providers.util.chatapi import ChatAPIHandler
 from inspect_ai.tool._tool_choice import ToolChoice
 from inspect_ai.tool._tool_info import ToolInfo
 
@@ -27,16 +26,14 @@ from .._model_output import (
     as_stop_reason,
 )
 from .._openai import chat_message_assistant_from_openai
-from .openai import (
-    OpenAIAPI,
-)
+from .openai_compatible import OpenAICompatibleAPI
 from .util import (
     chat_api_input,
     chat_api_request,
-    environment_prerequisite_error,
     model_base_url,
     should_retry_chat_api_error,
 )
+from .util.chatapi import ChatAPIHandler
 
 
 def chat_choices_from_response_together(
@@ -78,10 +75,7 @@ def chat_choices_from_response_together(
     ]
 
 
-TOGETHER_API_KEY = "TOGETHER_API_KEY"
-
-
-class TogetherAIAPI(OpenAIAPI):
+class TogetherAIAPI(OpenAICompatibleAPI):
     def __init__(
         self,
         model_name: str,
@@ -89,14 +83,13 @@ class TogetherAIAPI(OpenAIAPI):
         api_key: str | None = None,
         config: GenerateConfig = GenerateConfig(),
     ) -> None:
-        if not api_key:
-            api_key = os.environ.get(TOGETHER_API_KEY, None)
-            if not api_key:
-                raise environment_prerequisite_error("TogetherAI", TOGETHER_API_KEY)
-        base_url = model_base_url(base_url, "TOGETHER_BASE_URL")
-        base_url = base_url if base_url else "https://api.together.xyz/v1"
         super().__init__(
-            model_name=model_name, base_url=base_url, api_key=api_key, config=config
+            model_name=model_name,
+            base_url=base_url,
+            api_key=api_key,
+            config=config,
+            service="Together",
+            service_base_url="https://api.together.xyz/v1",
         )
 
     # Together uses a default of 512 so we bump it up
@@ -119,22 +112,31 @@ class TogetherAIAPI(OpenAIAPI):
             return ex
 
     @override
-    def set_logprobs_params(
-        self, params: dict[str, Any], config: GenerateConfig
-    ) -> dict[str, Any]:
-        if config.logprobs is True:
+    def completion_params(self, config: GenerateConfig, tools: bool) -> dict[str, Any]:
+        params = super().completion_params(config, tools)
+        if "logprobs" in params:
             params["logprobs"] = 1
+        if "top_logprobs" in params:
+            del params["top_logprobs"]
+
+        # together requires temperature with num_choices
+        if config.num_choices is not None and config.temperature is None:
+            params["temperature"] = 1
+
         return params
 
     # Together has a slightly different logprobs structure to OpenAI, so we need to remap it.
-    def _chat_choices_from_response(
-        self, response: ChatCompletion, tools: list[ToolInfo]
+    @override
+    def chat_choices_from_completion(
+        self, completion: ChatCompletion, tools: list[ToolInfo]
     ) -> list[ChatCompletionChoice]:
-        return chat_choices_from_response_together(response, tools)
+        return chat_choices_from_response_together(completion, tools)
 
 
 # Implementation of REST client for Together (currently not used)
 
+TOGETHER_API_KEY = "TOGETHER_API_KEY"
+
 
 class TogetherRESTAPI(ModelAPI):
     def __init__(

inspect_ai/model/_trim.py

@@ -0,0 +1,83 @@
+from dataclasses import dataclass, field
+
+from ._chat_message import ChatMessage
+
+
+def trim_messages(
+    messages: list[ChatMessage], preserve: float = 0.7
+) -> list[ChatMessage]:
+    """Trim message list to fit within model context.
+
+    Trim the list of messages by:
+    - Retaining all system messages.
+    - Retaining the 'input' messages from the sample.
+    - Preserving a proportion of the remaining messages (`preserve=0.7` by default).
+    - Ensuring that all assistant tool calls have corresponding tool messages.
+
+    Args:
+        messages: List of messages to trim.
+        preserve: Ratio of converation messages to preserve
+            (defaults to 0.7)
+
+    Returns:
+        Trimmed messages.
+    """
+    # validate preserve
+    if not 0 <= preserve <= 1:
+        raise ValueError(f"preserve must be in range [0,1], got {preserve}")
+
+    # partition messages
+    partitioned = _partition_messages(messages)
+
+    # slice messages from the beginning of the conversation as-per preserve
+    start_idx = int(len(partitioned.conversation) * (1 - preserve))
+    preserved_messages = partitioned.conversation[start_idx:]
+
+    # one last step: many model apis require tool messages to have a parent assistant
+    # message with a corresponding tool_call_id. to ensure this, we build the
+    # final list of conversation messages by filtering out tool messages for which
+    # we haven't seen a corresponding assistant message with their id
+    conversation_messages: list[ChatMessage] = []
+    active_tool_ids = set()
+    for message in preserved_messages:
+        if message.role == "assistant":
+            active_tool_ids = {tc.id for tc in (message.tool_calls or [])}
+            conversation_messages.append(message)
+        elif message.role == "tool" and message.tool_call_id in active_tool_ids:
+            conversation_messages.append(message)
+        elif message.role == "user":
+            active_tool_ids = set()
+            conversation_messages.append(message)
+
+    # return trimmed messages
+    return partitioned.system + partitioned.input + conversation_messages
+
+
+@dataclass
+class PartitionedMessages:
+    system: list[ChatMessage] = field(default_factory=list)
+    input: list[ChatMessage] = field(default_factory=list)
+    conversation: list[ChatMessage] = field(default_factory=list)
+
+
+def _partition_messages(messages: list[ChatMessage]) -> PartitionedMessages:
+    # first pass at partitioning
+    partitioned = PartitionedMessages()
+    for message in messages:
+        if message.role == "system":
+            partitioned.system.append(message)
+        elif message.source == "input":
+            partitioned.input.append(message)
+        else:
+            partitioned.conversation.append(message)
+
+    # if there are no input messages then take up to the first user message
+    if len(partitioned.input) == 0:
+        while partitioned.conversation:
+            message = partitioned.conversation.pop(0)
+            partitioned.input.append(message)
+            if message.role == "user":
+                break
+
+    # all done!
+    return partitioned

inspect_ai/solver/_plan.py

@@ -164,7 +164,7 @@ def plan(*plan: PlanType | None, name: str | None = None, **attribs: Any) -> Any
                 plan_type,
                 plan,
                 RegistryInfo(
-                    type="plan",
+                    type="plan",  # type: ignore[arg-type]
                     name=plan_name,
                     metadata=dict(attribs=attribs, params=params),
                 ),
@@ -212,7 +212,9 @@ def plan_register(
     registry_add(
         plan,
         RegistryInfo(
-            type="plan", name=name, metadata=dict(attribs=attribs, params=params)
+            type="plan",  # type: ignore[arg-type]
+            name=name,
+            metadata=dict(attribs=attribs, params=params),
         ),
     )
     return plan
@@ -228,4 +230,4 @@ def plan_create(name: str, **kwargs: Any) -> Plan:
     Returns:
         Plan with registry info attribute
     """
-    return cast(Plan, registry_create("plan", name, **kwargs))
+    return cast(Plan, registry_create("plan", name, **kwargs))  # type: ignore[arg-type]

inspect_ai/tool/_tool_def.py

@@ -234,9 +234,15 @@ def validate_tool_parameters(tool_name: str, parameters: dict[str, ToolParam]) -
     # validate that we have types/descriptions for paramters
     for param_name, param in parameters.items():
 
-        def raise_not_provided_error(context: str) -> None:
+        def raise_not_provided_error(
+            context: str,
+            # Use the default value trick to avoid Python's late binding of
+            # closures issue.
+            # see: https://docs.python.org/3/faq/programming.html#why-do-lambdas-defined-in-a-loop-with-different-values-all-return-the-same-result
+            bound_name: str = param_name,
+        ) -> None:
             raise ValueError(
-                f"{context} provided for parameter '{param_name}' of function '{tool_name}'."
+                f"{context} provided for parameter '{bound_name}' of function '{tool_name}'."
             )
 
         if param.type is None and not param.anyOf and not param.enum:

inspect_ai/util/__init__.py

@@ -1,3 +1,4 @@
+from inspect_ai._util.registry import RegistryType, registry_create
 from inspect_ai._util.trace import trace_action, trace_message
 
 from ._concurrency import concurrency
@@ -64,4 +65,6 @@ __all__ = [
     "throttle",
     "trace_action",
     "trace_message",
+    "RegistryType",
+    "registry_create",
 ]

inspect_ai/util/_concurrency.py

@@ -56,10 +56,23 @@ async def concurrency(
         yield
 
 
-def concurrency_status() -> dict[str, tuple[int, int]]:
+def concurrency_status_display() -> dict[str, tuple[int, int]]:
     status: dict[str, tuple[int, int]] = {}
+    names = [c.name for c in _concurrency_semaphores.values()]
     for c in _concurrency_semaphores.values():
-        status[c.name] = (c.concurrency - c.semaphore.value, c.concurrency)
+        # compute name for status display. some resources (e.g. models) use
+        # a / prefix. if there are no duplicates of a given prefix then shorten
+        # it to be only the prefix (e.g. 'openai' rather than 'openai/gpt-4o')
+        prefix = c.name.split("/")[0]
+        prefix_count = sum([1 for name in names if name.startswith(prefix + "/")])
+        if prefix_count == 1:
+            name = prefix
+        else:
+            name = c.name
+
+        # status display entry
+        status[name] = (c.concurrency - c.semaphore.value, c.concurrency)
+
     return status
 
 

{inspect_ai-0.3.88.dist-info → inspect_ai-0.3.89.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: inspect_ai
-Version: 0.3.88
+Version: 0.3.89
 Summary: Framework for large language model evaluations
 Author: UK AI Security Institute
 License: MIT License