letta-nightly 0.11.6.dev20250902104140__py3-none-any.whl → 0.11.7.dev20250904045700__py3-none-any.whl

letta/interface.py

@@ -198,23 +198,23 @@ class CLIInterface(AgentInterface):
                         try:
                             msg_dict = eval(function_args)
                             if function_name == "archival_memory_search":
-                                output = f'\tquery: {msg_dict["query"]}, page: {msg_dict["page"]}'
+                                output = f"\tquery: {msg_dict['query']}, page: {msg_dict['page']}"
                                 if STRIP_UI:
                                     print(output)
                                 else:
                                     print(f"{Fore.RED}{output}{Style.RESET_ALL}")
                             elif function_name == "archival_memory_insert":
-                                output = f'\t→ {msg_dict["content"]}'
+                                output = f"\t→ {msg_dict['content']}"
                                 if STRIP_UI:
                                     print(output)
                                 else:
                                     print(f"{Style.BRIGHT}{Fore.RED}{output}{Style.RESET_ALL}")
                             else:
                                 if STRIP_UI:
-                                    print(f'\t {msg_dict["old_content"]}\n\t→ {msg_dict["new_content"]}')
+                                    print(f"\t {msg_dict['old_content']}\n\t→ {msg_dict['new_content']}")
                                 else:
                                     print(
-                                        f'{Style.BRIGHT}\t{Fore.RED} {msg_dict["old_content"]}\n\t{Fore.GREEN}→ {msg_dict["new_content"]}{Style.RESET_ALL}'
+                                        f"{Style.BRIGHT}\t{Fore.RED} {msg_dict['old_content']}\n\t{Fore.GREEN}→ {msg_dict['new_content']}{Style.RESET_ALL}"
                                     )
                         except Exception as e:
                             printd(str(e))
@@ -223,7 +223,7 @@ class CLIInterface(AgentInterface):
                         print_function_message("🧠", f"searching memory with {function_name}")
                         try:
                             msg_dict = eval(function_args)
-                            output = f'\tquery: {msg_dict["query"]}, page: {msg_dict["page"]}'
+                            output = f"\tquery: {msg_dict['query']}, page: {msg_dict['page']}"
                             if STRIP_UI:
                                 print(output)
                             else:
@@ -248,7 +248,7 @@ class CLIInterface(AgentInterface):
     @staticmethod
     def print_messages(message_sequence: List[Message], dump=False):
         # rewrite to dict format
-        message_sequence = [msg.to_openai_dict() for msg in message_sequence]
+        message_sequence = Message.to_openai_dicts_from_list(message_sequence)
 
         idx = len(message_sequence)
         for msg in message_sequence:
@@ -291,7 +291,7 @@ class CLIInterface(AgentInterface):
     @staticmethod
     def print_messages_simple(message_sequence: List[Message]):
         # rewrite to dict format
-        message_sequence = [msg.to_openai_dict() for msg in message_sequence]
+        message_sequence = Message.to_openai_dicts_from_list(message_sequence)
 
         for msg in message_sequence:
             role = msg["role"]
@@ -309,7 +309,7 @@ class CLIInterface(AgentInterface):
     @staticmethod
     def print_messages_raw(message_sequence: List[Message]):
         # rewrite to dict format
-        message_sequence = [msg.to_openai_dict() for msg in message_sequence]
+        message_sequence = Message.to_openai_dicts_from_list(message_sequence)
 
         for msg in message_sequence:
             print(msg)

letta/interfaces/anthropic_streaming_interface.py

@@ -289,6 +289,13 @@ class AnthropicStreamingInterface:
                 if not self.anthropic_mode == EventMode.TEXT:
                     raise RuntimeError(f"Streaming integrity failed - received BetaTextDelta object while not in TEXT EventMode: {delta}")
 
+                # Weird bug happens with native thinking where a single response can contain:
+                # [reasoning, text, tool_call]
+                # In these cases, we should pipe text out to null / ignore it
+                # TODO this will have to be redone to support non-tool calling message sending
+                if not self.put_inner_thoughts_in_kwarg:
+                    return
+
                 # Combine buffer with current text to handle tags split across chunks
                 combined_text = self.partial_tag_buffer + delta.text
 

letta/interfaces/openai_streaming_interface.py

@@ -10,7 +10,14 @@ from letta.constants import DEFAULT_MESSAGE_TOOL, DEFAULT_MESSAGE_TOOL_KWARG
 from letta.llm_api.openai_client import is_openai_reasoning_model
 from letta.local_llm.utils import num_tokens_from_functions, num_tokens_from_messages
 from letta.log import get_logger
-from letta.schemas.letta_message import AssistantMessage, LettaMessage, ReasoningMessage, ToolCallDelta, ToolCallMessage
+from letta.schemas.letta_message import (
+    AssistantMessage,
+    HiddenReasoningMessage,
+    LettaMessage,
+    ReasoningMessage,
+    ToolCallDelta,
+    ToolCallMessage,
+)
 from letta.schemas.letta_message_content import OmittedReasoningContent, TextContent
 from letta.schemas.letta_stop_reason import LettaStopReason, StopReasonType
 from letta.schemas.message import Message
@@ -35,13 +42,15 @@ class OpenAIStreamingInterface:
         is_openai_proxy: bool = False,
         messages: Optional[list] = None,
         tools: Optional[list] = None,
+        put_inner_thoughts_in_kwarg: bool = True,
     ):
         self.use_assistant_message = use_assistant_message
         self.assistant_message_tool_name = DEFAULT_MESSAGE_TOOL
         self.assistant_message_tool_kwarg = DEFAULT_MESSAGE_TOOL_KWARG
+        self.put_inner_thoughts_in_kwarg = put_inner_thoughts_in_kwarg
 
         self.optimistic_json_parser: OptimisticJSONParser = OptimisticJSONParser()
-        self.function_args_reader = JSONInnerThoughtsExtractor(wait_for_first_key=True)  # TODO: pass in kwarg
+        self.function_args_reader = JSONInnerThoughtsExtractor(wait_for_first_key=put_inner_thoughts_in_kwarg)
         self.function_name_buffer = None
         self.function_args_buffer = None
         self.function_id_buffer = None
@@ -75,6 +84,7 @@ class OpenAIStreamingInterface:
         self.tool_call_name: str | None = None
         self.tool_call_id: str | None = None
         self.reasoning_messages = []
+        self.emitted_hidden_reasoning = False  # Track if we've emitted hidden reasoning message
 
     def get_reasoning_content(self) -> list[TextContent | OmittedReasoningContent]:
         content = "".join(self.reasoning_messages).strip()
@@ -113,6 +123,7 @@ class OpenAIStreamingInterface:
             if self.messages:
                 # Convert messages to dict format for token counting
                 message_dicts = [msg.to_openai_dict() if hasattr(msg, "to_openai_dict") else msg for msg in self.messages]
+                message_dicts = [m for m in message_dicts if m is not None]
                 self.fallback_input_tokens = num_tokens_from_messages(message_dicts)  # fallback to gpt-4 cl100k-base
 
             if self.tools:
@@ -184,6 +195,22 @@ class OpenAIStreamingInterface:
             if message_delta.tool_calls is not None and len(message_delta.tool_calls) > 0:
                 tool_call = message_delta.tool_calls[0]
 
+                # For OpenAI reasoning models, emit a hidden reasoning message before the first tool call
+                if not self.emitted_hidden_reasoning and is_openai_reasoning_model(self.model) and not self.put_inner_thoughts_in_kwarg:
+                    self.emitted_hidden_reasoning = True
+                    if prev_message_type and prev_message_type != "hidden_reasoning_message":
+                        message_index += 1
+                    hidden_message = HiddenReasoningMessage(
+                        id=self.letta_message_id,
+                        date=datetime.now(timezone.utc),
+                        state="omitted",
+                        hidden_reasoning=None,
+                        otid=Message.generate_otid_from_id(self.letta_message_id, message_index),
+                    )
+                    yield hidden_message
+                    prev_message_type = hidden_message.message_type
+                    message_index += 1  # Increment for the next message
+
                 if tool_call.function.name:
                     # If we're waiting for the first key, then we should hold back the name
                     # ie add it to a buffer instead of returning it as a chunk
@@ -232,16 +259,13 @@ class OpenAIStreamingInterface:
 
                     # If we have main_json, we should output a ToolCallMessage
                     elif updates_main_json:
-
                         # If there's something in the function_name buffer, we should release it first
                         # NOTE: we could output it as part of a chunk that has both name and args,
                         #       however the frontend may expect name first, then args, so to be
                         #       safe we'll output name first in a separate chunk
                         if self.function_name_buffer:
-
                             # use_assisitant_message means that we should also not release main_json raw, and instead should only release the contents of "message": "..."
                             if self.use_assistant_message and self.function_name_buffer == self.assistant_message_tool_name:
-
                                 # Store the ID of the tool call so allow skipping the corresponding response
                                 if self.function_id_buffer:
                                     self.prev_assistant_message_id = self.function_id_buffer
@@ -373,7 +397,6 @@ class OpenAIStreamingInterface:
                                     # clear buffers
                                     self.function_id_buffer = None
                             else:
-
                                 # There may be a buffer from a previous chunk, for example
                                 # if the previous chunk had arguments but we needed to flush name
                                 if self.function_args_buffer:

letta/llm_api/anthropic_client.py

@@ -5,12 +5,12 @@ from typing import Dict, List, Optional, Union
 
 import anthropic
 from anthropic import AsyncStream
-from anthropic.types.beta import BetaMessage as AnthropicMessage
-from anthropic.types.beta import BetaRawMessageStreamEvent
+from anthropic.types.beta import BetaMessage as AnthropicMessage, BetaRawMessageStreamEvent
 from anthropic.types.beta.message_create_params import MessageCreateParamsNonStreaming
 from anthropic.types.beta.messages import BetaMessageBatch
 from anthropic.types.beta.messages.batch_create_params import Request
 
+from letta.constants import FUNC_FAILED_HEARTBEAT_MESSAGE, REQ_HEARTBEAT_MESSAGE
 from letta.errors import (
     ContextWindowExceededError,
     ErrorCode,
@@ -34,9 +34,14 @@ from letta.otel.tracing import trace_method
 from letta.schemas.llm_config import LLMConfig
 from letta.schemas.message import Message as PydanticMessage
 from letta.schemas.openai.chat_completion_request import Tool as OpenAITool
-from letta.schemas.openai.chat_completion_response import ChatCompletionResponse, Choice, FunctionCall
-from letta.schemas.openai.chat_completion_response import Message as ChoiceMessage
-from letta.schemas.openai.chat_completion_response import ToolCall, UsageStatistics
+from letta.schemas.openai.chat_completion_response import (
+    ChatCompletionResponse,
+    Choice,
+    FunctionCall,
+    Message as ChoiceMessage,
+    ToolCall,
+    UsageStatistics,
+)
 from letta.settings import model_settings
 
 DUMMY_FIRST_USER_MESSAGE = "User initializing bootup sequence."
@@ -45,7 +50,6 @@ logger = get_logger(__name__)
 
 
 class AnthropicClient(LLMClientBase):
-
     @trace_method
     @deprecated("Synchronous version of this is no longer valid. Will result in model_dump of coroutine")
     def request(self, request_data: dict, llm_config: LLMConfig) -> dict:
@@ -56,7 +60,12 @@ class AnthropicClient(LLMClientBase):
     @trace_method
     async def request_async(self, request_data: dict, llm_config: LLMConfig) -> dict:
         client = await self._get_anthropic_client_async(llm_config, async_client=True)
-        response = await client.beta.messages.create(**request_data)
+
+        if llm_config.enable_reasoner:
+            response = await client.beta.messages.create(**request_data, betas=["interleaved-thinking-2025-05-14"])
+        else:
+            response = await client.beta.messages.create(**request_data)
+
         return response.model_dump()
 
     @trace_method
@@ -69,6 +78,11 @@ class AnthropicClient(LLMClientBase):
         # See: https://docs.anthropic.com/en/docs/build-with-claude/tool-use/fine-grained-streaming
         betas = ["fine-grained-tool-streaming-2025-05-14"]
 
+        # If extended thinking, turn on interleaved header
+        # https://docs.anthropic.com/en/docs/build-with-claude/extended-thinking#interleaved-thinking
+        if llm_config.enable_reasoner:
+            betas.append("interleaved-thinking-2025-05-14")
+
         return await client.beta.messages.create(**request_data, betas=betas)
 
     @trace_method
@@ -173,11 +187,14 @@ class AnthropicClient(LLMClientBase):
             raise NotImplementedError("Only tool calling supported on Anthropic API requests")
 
         if not llm_config.max_tokens:
-            raise ValueError("Max  tokens must be set for anthropic")
+            # TODO strip this default once we add provider-specific defaults
+            max_output_tokens = 4096  # the minimum max tokens (for Haiku 3)
+        else:
+            max_output_tokens = llm_config.max_tokens
 
         data = {
             "model": llm_config.model,
-            "max_tokens": llm_config.max_tokens,
+            "max_tokens": max_output_tokens,
             "temperature": llm_config.temperature,
         }
 
@@ -249,13 +266,11 @@ class AnthropicClient(LLMClientBase):
             raise RuntimeError(f"First message is not a system message, instead has role {messages[0].role}")
         system_content = messages[0].content if isinstance(messages[0].content, str) else messages[0].content[0].text
         data["system"] = self._add_cache_control_to_system_message(system_content)
-        data["messages"] = [
-            m.to_anthropic_dict(
-                inner_thoughts_xml_tag=inner_thoughts_xml_tag,
-                put_inner_thoughts_in_kwargs=bool(llm_config.put_inner_thoughts_in_kwargs),
-            )
-            for m in messages[1:]
-        ]
+        data["messages"] = PydanticMessage.to_anthropic_dicts_from_list(
+            messages=messages[1:],
+            inner_thoughts_xml_tag=inner_thoughts_xml_tag,
+            put_inner_thoughts_in_kwargs=bool(llm_config.put_inner_thoughts_in_kwargs),
+        )
 
         # Ensure first message is user
         if data["messages"][0]["role"] != "user":
@@ -264,6 +279,10 @@ class AnthropicClient(LLMClientBase):
         # Handle alternating messages
         data["messages"] = merge_tool_results_into_user_messages(data["messages"])
 
+        # Strip heartbeat pings if extended thinking
+        if llm_config.enable_reasoner:
+            data["messages"] = merge_heartbeats_into_tool_responses(data["messages"])
+
         # Prefix fill
         # https://docs.anthropic.com/en/api/messages#body-messages
         # NOTE: cannot prefill with tools for opus:
@@ -599,16 +618,167 @@ def convert_tools_to_anthropic_format(tools: List[OpenAITool]) -> List[dict]:
     """
     formatted_tools = []
     for tool in tools:
+        # Get the input schema
+        input_schema = tool.function.parameters or {"type": "object", "properties": {}, "required": []}
+
+        # Clean up the properties in the schema
+        # The presence of union types / default fields seems Anthropic to produce invalid JSON for tool calls
+        if isinstance(input_schema, dict) and "properties" in input_schema:
+            cleaned_properties = {}
+            for prop_name, prop_schema in input_schema.get("properties", {}).items():
+                if isinstance(prop_schema, dict):
+                    cleaned_properties[prop_name] = _clean_property_schema(prop_schema)
+                else:
+                    cleaned_properties[prop_name] = prop_schema
+
+            # Create cleaned input schema
+            cleaned_input_schema = {
+                "type": input_schema.get("type", "object"),
+                "properties": cleaned_properties,
+            }
+
+            # Only add required field if it exists and is non-empty
+            if "required" in input_schema and input_schema["required"]:
+                cleaned_input_schema["required"] = input_schema["required"]
+        else:
+            cleaned_input_schema = input_schema
+
         formatted_tool = {
             "name": tool.function.name,
             "description": tool.function.description if tool.function.description else "",
-            "input_schema": tool.function.parameters or {"type": "object", "properties": {}, "required": []},
+            "input_schema": cleaned_input_schema,
         }
         formatted_tools.append(formatted_tool)
 
     return formatted_tools
 
 
+def _clean_property_schema(prop_schema: dict) -> dict:
+    """Clean up a property schema by removing defaults and simplifying union types."""
+    cleaned = {}
+
+    # Handle type field - simplify union types like ["null", "string"] to just "string"
+    if "type" in prop_schema:
+        prop_type = prop_schema["type"]
+        if isinstance(prop_type, list):
+            # Remove "null" from union types to simplify
+            # e.g., ["null", "string"] becomes "string"
+            non_null_types = [t for t in prop_type if t != "null"]
+            if len(non_null_types) == 1:
+                cleaned["type"] = non_null_types[0]
+            elif len(non_null_types) > 1:
+                # Keep as array if multiple non-null types
+                cleaned["type"] = non_null_types
+            else:
+                # If only "null" was in the list, default to string
+                cleaned["type"] = "string"
+        else:
+            cleaned["type"] = prop_type
+
+    # Copy over other fields except 'default'
+    for key, value in prop_schema.items():
+        if key not in ["type", "default"]:  # Skip 'default' field
+            if key == "properties" and isinstance(value, dict):
+                # Recursively clean nested properties
+                cleaned["properties"] = {k: _clean_property_schema(v) if isinstance(v, dict) else v for k, v in value.items()}
+            else:
+                cleaned[key] = value
+
+    return cleaned
+
+
+def is_heartbeat(message: dict, is_ping: bool = False) -> bool:
+    """Check if the message is an automated heartbeat ping"""
+
+    if "role" not in message or message["role"] != "user" or "content" not in message:
+        return False
+
+    try:
+        message_json = json.loads(message["content"])
+    except:
+        return False
+
+    if "reason" not in message_json:
+        return False
+
+    if message_json["type"] != "heartbeat":
+        return False
+
+    if not is_ping:
+        # Just checking if 'type': 'heartbeat'
+        return True
+    else:
+        # Also checking if it's specifically a 'ping' style message
+        # NOTE: this will not catch tool rule heartbeats
+        if REQ_HEARTBEAT_MESSAGE in message_json["reason"] or FUNC_FAILED_HEARTBEAT_MESSAGE in message_json["reason"]:
+            return True
+        else:
+            return False
+
+
+def merge_heartbeats_into_tool_responses(messages: List[dict]):
+    """For extended thinking mode, we don't want anything other than tool responses in-between assistant actions
+
+    Otherwise, the thinking will silently get dropped.
+
+    NOTE: assumes merge_tool_results_into_user_messages has already been called
+    """
+
+    merged_messages = []
+
+    # Loop through messages
+    # For messages with role 'user' and len(content) > 1,
+    #   Check if content[0].type == 'tool_result'
+    #   If so, iterate over content[1:] and while content.type == 'text' and is_heartbeat(content.text),
+    #     merge into content[0].content
+
+    for message in messages:
+        if "role" not in message or "content" not in message:
+            # Skip invalid messages
+            merged_messages.append(message)
+            continue
+
+        if message["role"] == "user" and len(message["content"]) > 1:
+            content_parts = message["content"]
+
+            # If the first content part is a tool result, merge the heartbeat content into index 0 of the content
+            # Two end cases:
+            # 1. It was [tool_result, heartbeat], in which case merged result is [tool_result+heartbeat] (len 1)
+            # 2. It was [tool_result, user_text], in which case it should be unchanged (len 2)
+            if "type" in content_parts[0] and "content" in content_parts[0] and content_parts[0]["type"] == "tool_result":
+                new_content_parts = [content_parts[0]]
+
+                # If the first content part is a tool result, merge the heartbeat content into index 0 of the content
+                for i, content_part in enumerate(content_parts[1:]):
+                    # If it's a heartbeat, add it to the merge
+                    if (
+                        content_part["type"] == "text"
+                        and "text" in content_part
+                        and is_heartbeat({"role": "user", "content": content_part["text"]})
+                    ):
+                        # NOTE: joining with a ','
+                        new_content_parts[0]["content"] += ", " + content_part["text"]
+
+                    # If it's not, break, and concat to finish
+                    else:
+                        # Append the rest directly, no merging of content strings
+                        new_content_parts.extend(content_parts[i + 1 :])
+                        break
+
+                # Set the content_parts
+                message["content"] = new_content_parts
+                merged_messages.append(message)
+
+            else:
+                # Skip invalid messages parts
+                merged_messages.append(message)
+                continue
+        else:
+            merged_messages.append(message)
+
+    return merged_messages
+
+
 def merge_tool_results_into_user_messages(messages: List[dict]):
     """Anthropic API doesn't allow role 'tool'->'user' sequences
 
@@ -647,7 +817,7 @@ def merge_tool_results_into_user_messages(messages: List[dict]):
                 if isinstance(next_message["content"], list)
                 else [{"type": "text", "text": next_message["content"]}]
             )
-            merged_content = current_content + next_content
+            merged_content: list = current_content + next_content
             current_message["content"] = merged_content
         else:
             # Append the current message to result as it's complete

letta/llm_api/azure_client.py

@@ -13,7 +13,6 @@ from letta.settings import model_settings
 
 
 class AzureClient(OpenAIClient):
-
     def get_byok_overrides(self, llm_config: LLMConfig) -> Tuple[Optional[str], Optional[str], Optional[str]]:
         if llm_config.provider_category == ProviderCategory.byok:
             from letta.services.provider_manager import ProviderManager

letta/llm_api/bedrock_client.py

@@ -16,7 +16,6 @@ logger = get_logger(__name__)
 
 
 class BedrockClient(AnthropicClient):
-
     async def get_byok_overrides_async(self, llm_config: LLMConfig) -> tuple[str, str, str]:
         override_access_key_id, override_secret_access_key, override_default_region = None, None, None
         if llm_config.provider_category == ProviderCategory.byok:
@@ -28,7 +27,7 @@ class BedrockClient(AnthropicClient):
                 llm_config.provider_name,
                 actor=self.actor,
             )
-        return override_access_key_id, override_secret_access_key, override_default_regions
+        return override_access_key_id, override_secret_access_key, override_default_region
 
     @trace_method
     async def _get_anthropic_client_async(