letta-nightly 0.8.0.dev20250606104326__py3-none-any.whl → 0.8.2.dev20250606215616__py3-none-any.whl

letta/interfaces/anthropic_streaming_interface.py

@@ -23,7 +23,7 @@ from anthropic.types.beta import (
 )
 
 from letta.constants import DEFAULT_MESSAGE_TOOL, DEFAULT_MESSAGE_TOOL_KWARG
-from letta.helpers.datetime_helpers import get_utc_timestamp_ns
+from letta.helpers.datetime_helpers import get_utc_timestamp_ns, ns_to_ms
 from letta.local_llm.constants import INNER_THOUGHTS_KWARG
 from letta.log import get_logger
 from letta.schemas.letta_message import (
@@ -62,8 +62,7 @@ class AnthropicStreamingInterface:
         self.use_assistant_message = use_assistant_message
 
         # Premake IDs for database writes
-        self.letta_assistant_message_id = Message.generate_id()
-        self.letta_tool_message_id = Message.generate_id()
+        self.letta_message_id = Message.generate_id()
 
         self.anthropic_mode = None
         self.message_id = None
@@ -132,7 +131,7 @@ class AnthropicStreamingInterface:
                         now = get_utc_timestamp_ns()
                         ttft_ns = now - provider_request_start_timestamp_ns
                         ttft_span.add_event(
-                            name="anthropic_time_to_first_token_ms", attributes={"anthropic_time_to_first_token_ms": ttft_ns // 1_000_000}
+                            name="anthropic_time_to_first_token_ms", attributes={"anthropic_time_to_first_token_ms": ns_to_ms(ttft_ns)}
                         )
                         first_chunk = False
 
@@ -152,7 +151,7 @@ class AnthropicStreamingInterface:
                             if not self.use_assistant_message:
                                 # Buffer the initial tool call message instead of yielding immediately
                                 tool_call_msg = ToolCallMessage(
-                                    id=self.letta_tool_message_id,
+                                    id=self.letta_message_id,
                                     tool_call=ToolCallDelta(name=self.tool_call_name, tool_call_id=self.tool_call_id),
                                     date=datetime.now(timezone.utc).isoformat(),
                                 )
@@ -165,11 +164,11 @@ class AnthropicStreamingInterface:
                             if prev_message_type and prev_message_type != "hidden_reasoning_message":
                                 message_index += 1
                             hidden_reasoning_message = HiddenReasoningMessage(
-                                id=self.letta_assistant_message_id,
+                                id=self.letta_message_id,
                                 state="redacted",
                                 hidden_reasoning=content.data,
                                 date=datetime.now(timezone.utc).isoformat(),
-                                otid=Message.generate_otid_from_id(self.letta_assistant_message_id, message_index),
+                                otid=Message.generate_otid_from_id(self.letta_message_id, message_index),
                             )
                             self.reasoning_messages.append(hidden_reasoning_message)
                             prev_message_type = hidden_reasoning_message.message_type
@@ -206,10 +205,10 @@ class AnthropicStreamingInterface:
                             if prev_message_type and prev_message_type != "reasoning_message":
                                 message_index += 1
                             reasoning_message = ReasoningMessage(
-                                id=self.letta_assistant_message_id,
+                                id=self.letta_message_id,
                                 reasoning=self.accumulated_inner_thoughts[-1],
                                 date=datetime.now(timezone.utc).isoformat(),
-                                otid=Message.generate_otid_from_id(self.letta_assistant_message_id, message_index),
+                                otid=Message.generate_otid_from_id(self.letta_message_id, message_index),
                             )
                             self.reasoning_messages.append(reasoning_message)
                             prev_message_type = reasoning_message.message_type
@@ -233,10 +232,10 @@ class AnthropicStreamingInterface:
                                 if prev_message_type and prev_message_type != "reasoning_message":
                                     message_index += 1
                                 reasoning_message = ReasoningMessage(
-                                    id=self.letta_assistant_message_id,
+                                    id=self.letta_message_id,
                                     reasoning=inner_thoughts_diff,
                                     date=datetime.now(timezone.utc).isoformat(),
-                                    otid=Message.generate_otid_from_id(self.letta_assistant_message_id, message_index),
+                                    otid=Message.generate_otid_from_id(self.letta_message_id, message_index),
                                 )
                                 self.reasoning_messages.append(reasoning_message)
                                 prev_message_type = reasoning_message.message_type
@@ -249,10 +248,28 @@ class AnthropicStreamingInterface:
                                 if len(self.tool_call_buffer) > 0:
                                     if prev_message_type and prev_message_type != "tool_call_message":
                                         message_index += 1
+
+                                    # Strip out the inner thoughts from the buffered tool call arguments before streaming
+                                    tool_call_args = ""
                                     for buffered_msg in self.tool_call_buffer:
-                                        buffered_msg.otid = Message.generate_otid_from_id(self.letta_tool_message_id, message_index)
-                                        prev_message_type = buffered_msg.message_type
-                                        yield buffered_msg
+                                        tool_call_args += buffered_msg.tool_call.arguments if buffered_msg.tool_call.arguments else ""
+                                    tool_call_args = tool_call_args.replace(f'"{INNER_THOUGHTS_KWARG}": "{current_inner_thoughts}"', "")
+
+                                    tool_call_msg = ToolCallMessage(
+                                        id=self.tool_call_buffer[0].id,
+                                        otid=Message.generate_otid_from_id(self.tool_call_buffer[0].id, message_index),
+                                        date=self.tool_call_buffer[0].date,
+                                        name=self.tool_call_buffer[0].name,
+                                        sender_id=self.tool_call_buffer[0].sender_id,
+                                        step_id=self.tool_call_buffer[0].step_id,
+                                        tool_call=ToolCallDelta(
+                                            name=self.tool_call_name,
+                                            tool_call_id=self.tool_call_id,
+                                            arguments=tool_call_args,
+                                        ),
+                                    )
+                                    prev_message_type = tool_call_msg.message_type
+                                    yield tool_call_msg
                                     self.tool_call_buffer = []
 
                             # Start detecting special case of "send_message"
@@ -266,24 +283,26 @@ class AnthropicStreamingInterface:
                                     if prev_message_type and prev_message_type != "assistant_message":
                                         message_index += 1
                                     assistant_msg = AssistantMessage(
-                                        id=self.letta_assistant_message_id,
+                                        id=self.letta_message_id,
                                         content=[TextContent(text=send_message_diff)],
                                         date=datetime.now(timezone.utc).isoformat(),
-                                        otid=Message.generate_otid_from_id(self.letta_assistant_message_id, message_index),
+                                        otid=Message.generate_otid_from_id(self.letta_message_id, message_index),
                                     )
                                     prev_message_type = assistant_msg.message_type
                                     yield assistant_msg
                             else:
                                 # Otherwise, it is a normal tool call - buffer or yield based on inner thoughts status
                                 tool_call_msg = ToolCallMessage(
-                                    id=self.letta_tool_message_id,
-                                    tool_call=ToolCallDelta(arguments=delta.partial_json),
+                                    id=self.letta_message_id,
+                                    tool_call=ToolCallDelta(
+                                        name=self.tool_call_name, tool_call_id=self.tool_call_id, arguments=delta.partial_json
+                                    ),
                                     date=datetime.now(timezone.utc).isoformat(),
                                 )
                                 if self.inner_thoughts_complete:
                                     if prev_message_type and prev_message_type != "tool_call_message":
                                         message_index += 1
-                                    tool_call_msg.otid = Message.generate_otid_from_id(self.letta_tool_message_id, message_index)
+                                    tool_call_msg.otid = Message.generate_otid_from_id(self.letta_message_id, message_index)
                                     prev_message_type = tool_call_msg.message_type
                                     yield tool_call_msg
                                 else:
@@ -301,11 +320,11 @@ class AnthropicStreamingInterface:
                             if prev_message_type and prev_message_type != "reasoning_message":
                                 message_index += 1
                             reasoning_message = ReasoningMessage(
-                                id=self.letta_assistant_message_id,
+                                id=self.letta_message_id,
                                 source="reasoner_model",
                                 reasoning=delta.thinking,
                                 date=datetime.now(timezone.utc).isoformat(),
-                                otid=Message.generate_otid_from_id(self.letta_assistant_message_id, message_index),
+                                otid=Message.generate_otid_from_id(self.letta_message_id, message_index),
                             )
                             self.reasoning_messages.append(reasoning_message)
                             prev_message_type = reasoning_message.message_type
@@ -320,12 +339,12 @@ class AnthropicStreamingInterface:
                             if prev_message_type and prev_message_type != "reasoning_message":
                                 message_index += 1
                             reasoning_message = ReasoningMessage(
-                                id=self.letta_assistant_message_id,
+                                id=self.letta_message_id,
                                 source="reasoner_model",
                                 reasoning="",
                                 date=datetime.now(timezone.utc).isoformat(),
                                 signature=delta.signature,
-                                otid=Message.generate_otid_from_id(self.letta_assistant_message_id, message_index),
+                                otid=Message.generate_otid_from_id(self.letta_message_id, message_index),
                             )
                             self.reasoning_messages.append(reasoning_message)
                             prev_message_type = reasoning_message.message_type
@@ -360,7 +379,7 @@ class AnthropicStreamingInterface:
             group: List[Union[ReasoningMessage, HiddenReasoningMessage]], group_type: str
         ) -> Union[TextContent, ReasoningContent, RedactedReasoningContent]:
             if group_type == "reasoning":
-                reasoning_text = "".join(chunk.reasoning for chunk in group)
+                reasoning_text = "".join(chunk.reasoning for chunk in group).strip()
                 is_native = any(chunk.source == "reasoner_model" for chunk in group)
                 signature = next((chunk.signature for chunk in group if chunk.signature is not None), None)
                 if is_native:

letta/interfaces/openai_streaming_interface.py

@@ -5,7 +5,7 @@ from openai import AsyncStream
 from openai.types.chat.chat_completion_chunk import ChatCompletionChunk
 
 from letta.constants import DEFAULT_MESSAGE_TOOL, DEFAULT_MESSAGE_TOOL_KWARG
-from letta.helpers.datetime_helpers import get_utc_timestamp_ns
+from letta.helpers.datetime_helpers import get_utc_timestamp_ns, ns_to_ms
 from letta.schemas.letta_message import AssistantMessage, LettaMessage, ReasoningMessage, ToolCallDelta, ToolCallMessage
 from letta.schemas.letta_message_content import TextContent
 from letta.schemas.message import Message
@@ -32,14 +32,14 @@ class OpenAIStreamingInterface:
         self.function_args_buffer = None
         self.function_id_buffer = None
         self.last_flushed_function_name = None
+        self.last_flushed_function_id = None
 
         # Buffer to hold function arguments until inner thoughts are complete
         self.current_function_arguments = ""
         self.current_json_parse_result = {}
 
         # Premake IDs for database writes
-        self.letta_assistant_message_id = Message.generate_id()
-        self.letta_tool_message_id = Message.generate_id()
+        self.letta_message_id = Message.generate_id()
 
         self.message_id = None
         self.model = None
@@ -54,14 +54,14 @@ class OpenAIStreamingInterface:
         self.reasoning_messages = []
 
     def get_reasoning_content(self) -> List[TextContent]:
-        content = "".join(self.reasoning_messages)
+        content = "".join(self.reasoning_messages).strip()
         return [TextContent(text=content)]
 
     def get_tool_call_object(self) -> ToolCall:
         """Useful for agent loop"""
         function_name = self.last_flushed_function_name if self.last_flushed_function_name else self.function_name_buffer
         return ToolCall(
-            id=self.letta_tool_message_id,
+            id=self.last_flushed_function_id,
             function=FunctionCall(arguments=self.current_function_arguments, name=function_name),
         )
 
@@ -85,7 +85,7 @@ class OpenAIStreamingInterface:
                     now = get_utc_timestamp_ns()
                     ttft_ns = now - provider_request_start_timestamp_ns
                     ttft_span.add_event(
-                        name="openai_time_to_first_token_ms", attributes={"openai_time_to_first_token_ms": ttft_ns // 1_000_000}
+                        name="openai_time_to_first_token_ms", attributes={"openai_time_to_first_token_ms": ns_to_ms(ttft_ns)}
                     )
                     first_chunk = False
 
@@ -133,11 +133,11 @@ class OpenAIStreamingInterface:
                                     message_index += 1
                                 self.reasoning_messages.append(updates_inner_thoughts)
                                 reasoning_message = ReasoningMessage(
-                                    id=self.letta_tool_message_id,
+                                    id=self.letta_message_id,
                                     date=datetime.now(timezone.utc),
                                     reasoning=updates_inner_thoughts,
                                     # name=name,
-                                    otid=Message.generate_otid_from_id(self.letta_tool_message_id, message_index),
+                                    otid=Message.generate_otid_from_id(self.letta_message_id, message_index),
                                 )
                                 prev_message_type = reasoning_message.message_type
                                 yield reasoning_message
@@ -171,20 +171,22 @@ class OpenAIStreamingInterface:
                                             message_index += 1
                                         self.tool_call_name = str(self.function_name_buffer)
                                         tool_call_msg = ToolCallMessage(
-                                            id=self.letta_tool_message_id,
+                                            id=self.letta_message_id,
                                             date=datetime.now(timezone.utc),
                                             tool_call=ToolCallDelta(
                                                 name=self.function_name_buffer,
                                                 arguments=None,
                                                 tool_call_id=self.function_id_buffer,
                                             ),
-                                            otid=Message.generate_otid_from_id(self.letta_tool_message_id, message_index),
+                                            otid=Message.generate_otid_from_id(self.letta_message_id, message_index),
                                         )
                                         prev_message_type = tool_call_msg.message_type
                                         yield tool_call_msg
 
                                     # Record what the last function name we flushed was
                                     self.last_flushed_function_name = self.function_name_buffer
+                                    if self.last_flushed_function_id is None:
+                                        self.last_flushed_function_id = self.function_id_buffer
                                     # Clear the buffer
                                     self.function_name_buffer = None
                                     self.function_id_buffer = None
@@ -236,10 +238,10 @@ class OpenAIStreamingInterface:
                                             if prev_message_type and prev_message_type != "assistant_message":
                                                 message_index += 1
                                             assistant_message = AssistantMessage(
-                                                id=self.letta_assistant_message_id,
+                                                id=self.letta_message_id,
                                                 date=datetime.now(timezone.utc),
                                                 content=combined_chunk,
-                                                otid=Message.generate_otid_from_id(self.letta_assistant_message_id, message_index),
+                                                otid=Message.generate_otid_from_id(self.letta_message_id, message_index),
                                             )
                                             prev_message_type = assistant_message.message_type
                                             yield assistant_message
@@ -268,11 +270,11 @@ class OpenAIStreamingInterface:
                                                 if prev_message_type and prev_message_type != "assistant_message":
                                                     message_index += 1
                                                 assistant_message = AssistantMessage(
-                                                    id=self.letta_assistant_message_id,
+                                                    id=self.letta_message_id,
                                                     date=datetime.now(timezone.utc),
                                                     content=diff,
                                                     # name=name,
-                                                    otid=Message.generate_otid_from_id(self.letta_assistant_message_id, message_index),
+                                                    otid=Message.generate_otid_from_id(self.letta_message_id, message_index),
                                                 )
                                                 prev_message_type = assistant_message.message_type
                                                 yield assistant_message
@@ -292,15 +294,15 @@ class OpenAIStreamingInterface:
                                             if prev_message_type and prev_message_type != "tool_call_message":
                                                 message_index += 1
                                             tool_call_msg = ToolCallMessage(
-                                                id=self.letta_tool_message_id,
+                                                id=self.letta_message_id,
                                                 date=datetime.now(timezone.utc),
                                                 tool_call=ToolCallDelta(
-                                                    name=None,
+                                                    name=self.function_name_buffer,
                                                     arguments=combined_chunk,
                                                     tool_call_id=self.function_id_buffer,
                                                 ),
                                                 # name=name,
-                                                otid=Message.generate_otid_from_id(self.letta_tool_message_id, message_index),
+                                                otid=Message.generate_otid_from_id(self.letta_message_id, message_index),
                                             )
                                             prev_message_type = tool_call_msg.message_type
                                             yield tool_call_msg
@@ -312,7 +314,7 @@ class OpenAIStreamingInterface:
                                             if prev_message_type and prev_message_type != "tool_call_message":
                                                 message_index += 1
                                             tool_call_msg = ToolCallMessage(
-                                                id=self.letta_tool_message_id,
+                                                id=self.letta_message_id,
                                                 date=datetime.now(timezone.utc),
                                                 tool_call=ToolCallDelta(
                                                     name=None,
@@ -320,7 +322,7 @@ class OpenAIStreamingInterface:
                                                     tool_call_id=self.function_id_buffer,
                                                 ),
                                                 # name=name,
-                                                otid=Message.generate_otid_from_id(self.letta_tool_message_id, message_index),
+                                                otid=Message.generate_otid_from_id(self.letta_message_id, message_index),
                                             )
                                             prev_message_type = tool_call_msg.message_type
                                             yield tool_call_msg

letta/llm_api/anthropic.py

@@ -26,6 +26,7 @@ from letta.llm_api.helpers import add_inner_thoughts_to_functions
 from letta.local_llm.constants import INNER_THOUGHTS_KWARG, INNER_THOUGHTS_KWARG_DESCRIPTION
 from letta.local_llm.utils import num_tokens_from_functions, num_tokens_from_messages
 from letta.log import get_logger
+from letta.otel.tracing import log_event
 from letta.schemas.enums import ProviderCategory
 from letta.schemas.message import Message as _Message
 from letta.schemas.message import MessageRole as _MessageRole
@@ -45,7 +46,6 @@ from letta.services.provider_manager import ProviderManager
 from letta.services.user_manager import UserManager
 from letta.settings import model_settings
 from letta.streaming_interface import AgentChunkStreamingInterface, AgentRefreshStreamingInterface
-from letta.tracing import log_event
 
 logger = get_logger(__name__)
 

letta/llm_api/anthropic_client.py

@@ -27,16 +27,16 @@ from letta.llm_api.helpers import add_inner_thoughts_to_functions, unpack_all_in
 from letta.llm_api.llm_client_base import LLMClientBase
 from letta.local_llm.constants import INNER_THOUGHTS_KWARG, INNER_THOUGHTS_KWARG_DESCRIPTION
 from letta.log import get_logger
+from letta.otel.tracing import trace_method
 from letta.schemas.enums import ProviderCategory
 from letta.schemas.llm_config import LLMConfig
 from letta.schemas.message import Message as PydanticMessage
-from letta.schemas.openai.chat_completion_request import Tool
+from letta.schemas.openai.chat_completion_request import Tool as OpenAITool
 from letta.schemas.openai.chat_completion_response import ChatCompletionResponse, Choice, FunctionCall
 from letta.schemas.openai.chat_completion_response import Message as ChoiceMessage
 from letta.schemas.openai.chat_completion_response import ToolCall, UsageStatistics
 from letta.services.provider_manager import ProviderManager
 from letta.settings import model_settings
-from letta.tracing import trace_method
 
 DUMMY_FIRST_USER_MESSAGE = "User initializing bootup sequence."
 
@@ -199,10 +199,10 @@ class AnthropicClient(LLMClientBase):
         elif llm_config.enable_reasoner:
             # NOTE: reasoning models currently do not allow for `any`
             tool_choice = {"type": "auto", "disable_parallel_tool_use": True}
-            tools_for_request = [Tool(function=f) for f in tools]
+            tools_for_request = [OpenAITool(function=f) for f in tools]
         elif force_tool_call is not None:
             tool_choice = {"type": "tool", "name": force_tool_call}
-            tools_for_request = [Tool(function=f) for f in tools if f["name"] == force_tool_call]
+            tools_for_request = [OpenAITool(function=f) for f in tools if f["name"] == force_tool_call]
 
             # need to have this setting to be able to put inner thoughts in kwargs
             if not llm_config.put_inner_thoughts_in_kwargs:
@@ -216,7 +216,7 @@ class AnthropicClient(LLMClientBase):
                 tool_choice = {"type": "any", "disable_parallel_tool_use": True}
             else:
                 tool_choice = {"type": "auto", "disable_parallel_tool_use": True}
-            tools_for_request = [Tool(function=f) for f in tools] if tools is not None else None
+            tools_for_request = [OpenAITool(function=f) for f in tools] if tools is not None else None
 
         # Add tool choice
         if tool_choice:
@@ -230,7 +230,7 @@ class AnthropicClient(LLMClientBase):
                 inner_thoughts_key=INNER_THOUGHTS_KWARG,
                 inner_thoughts_description=INNER_THOUGHTS_KWARG_DESCRIPTION,
             )
-            tools_for_request = [Tool(function=f) for f in tools_with_inner_thoughts]
+            tools_for_request = [OpenAITool(function=f) for f in tools_with_inner_thoughts]
 
         if tools_for_request and len(tools_for_request) > 0:
             # TODO eventually enable parallel tool use
@@ -270,7 +270,7 @@ class AnthropicClient(LLMClientBase):
 
         return data
 
-    async def count_tokens(self, messages: List[dict] = None, model: str = None, tools: List[Tool] = None) -> int:
+    async def count_tokens(self, messages: List[dict] = None, model: str = None, tools: List[OpenAITool] = None) -> int:
         client = anthropic.AsyncAnthropic()
         if messages and len(messages) == 0:
             messages = None
@@ -278,11 +278,19 @@ class AnthropicClient(LLMClientBase):
             anthropic_tools = convert_tools_to_anthropic_format(tools)
         else:
             anthropic_tools = None
-        result = await client.beta.messages.count_tokens(
-            model=model or "claude-3-7-sonnet-20250219",
-            messages=messages or [{"role": "user", "content": "hi"}],
-            tools=anthropic_tools or [],
-        )
+
+        try:
+            result = await client.beta.messages.count_tokens(
+                model=model or "claude-3-7-sonnet-20250219",
+                messages=messages or [{"role": "user", "content": "hi"}],
+                tools=anthropic_tools or [],
+            )
+        except:
+            import ipdb
+
+            ipdb.set_trace()
+            raise
+
         token_count = result.input_tokens
         if messages is None:
             token_count -= 8
@@ -477,7 +485,7 @@ class AnthropicClient(LLMClientBase):
         return chat_completion_response
 
 
-def convert_tools_to_anthropic_format(tools: List[Tool]) -> List[dict]:
+def convert_tools_to_anthropic_format(tools: List[OpenAITool]) -> List[dict]:
     """See: https://docs.anthropic.com/claude/docs/tool-use
 
     OpenAI style:
@@ -527,7 +535,7 @@ def convert_tools_to_anthropic_format(tools: List[Tool]) -> List[dict]:
     for tool in tools:
         formatted_tool = {
             "name": tool.function.name,
-            "description": tool.function.description,
+            "description": tool.function.description if tool.function.description else "",
             "input_schema": tool.function.parameters or {"type": "object", "properties": {}, "required": []},
         }
         formatted_tools.append(formatted_tool)

letta/llm_api/google_vertex_client.py

@@ -12,12 +12,12 @@ from letta.llm_api.llm_client_base import LLMClientBase
 from letta.local_llm.json_parser import clean_json_string_extra_backslash
 from letta.local_llm.utils import count_tokens
 from letta.log import get_logger
+from letta.otel.tracing import trace_method
 from letta.schemas.llm_config import LLMConfig
 from letta.schemas.message import Message as PydanticMessage
 from letta.schemas.openai.chat_completion_request import Tool
 from letta.schemas.openai.chat_completion_response import ChatCompletionResponse, Choice, FunctionCall, Message, ToolCall, UsageStatistics
 from letta.settings import model_settings, settings
-from letta.tracing import trace_method
 from letta.utils import get_tool_call_id
 
 logger = get_logger(__name__)

letta/llm_api/helpers.py

@@ -63,11 +63,11 @@ def _convert_to_structured_output_helper(property: dict) -> dict:
 
 
 def convert_to_structured_output(openai_function: dict, allow_optional: bool = False) -> dict:
-    """Convert function call objects to structured output objects
+    """Convert function call objects to structured output objects.
 
     See: https://platform.openai.com/docs/guides/structured-outputs/supported-schemas
     """
-    description = openai_function["description"] if "description" in openai_function else ""
+    description = openai_function.get("description", "")
 
     structured_output = {
         "name": openai_function["name"],
@@ -81,54 +81,58 @@ def convert_to_structured_output(openai_function: dict, allow_optional: bool = F
         },
     }
 
-    # This code needs to be able to handle nested properties
-    # For example, the param details may have "type" + "description",
-    # but if "type" is "object" we expected "properties", where each property has details
-    # and if "type" is "array" we expect "items": <type>
     for param, details in openai_function["parameters"]["properties"].items():
         param_type = details["type"]
-        description = details.get("description", "")
+        param_description = details.get("description", "")
 
         if param_type == "object":
             if "properties" not in details:
-                # Structured outputs requires the properties on dicts be specified ahead of time
-                raise ValueError(f"Property {param} of type object is missing properties")
+                raise ValueError(f"Property {param} of type object is missing 'properties'")
             structured_output["parameters"]["properties"][param] = {
                 "type": "object",
-                "description": description,
+                "description": param_description,
                 "properties": {k: _convert_to_structured_output_helper(v) for k, v in details["properties"].items()},
                 "additionalProperties": False,
                 "required": list(details["properties"].keys()),
             }
 
         elif param_type == "array":
-            structured_output["parameters"]["properties"][param] = {
-                "type": "array",
-                "description": description,
-                "items": _convert_to_structured_output_helper(details["items"]),
-            }
+            items_schema = details.get("items")
+            prefix_items_schema = details.get("prefixItems")
+
+            if prefix_items_schema:
+                # assume fixed-length tuple — safe fallback to use first type for items
+                fallback_item = prefix_items_schema[0] if isinstance(prefix_items_schema, list) else prefix_items_schema
+                structured_output["parameters"]["properties"][param] = {
+                    "type": "array",
+                    "description": param_description,
+                    "prefixItems": [_convert_to_structured_output_helper(item) for item in prefix_items_schema],
+                    "items": _convert_to_structured_output_helper(fallback_item),
+                    "minItems": details.get("minItems", len(prefix_items_schema)),
+                    "maxItems": details.get("maxItems", len(prefix_items_schema)),
+                }
+            elif items_schema:
+                structured_output["parameters"]["properties"][param] = {
+                    "type": "array",
+                    "description": param_description,
+                    "items": _convert_to_structured_output_helper(items_schema),
+                }
+            else:
+                raise ValueError(f"Array param '{param}' is missing both 'items' and 'prefixItems'")
 
         else:
-            structured_output["parameters"]["properties"][param] = {
-                "type": param_type,  # simple type
-                "description": description,
+            prop = {
+                "type": param_type,
+                "description": param_description,
             }
-
-        if "enum" in details:
-            structured_output["parameters"]["properties"][param]["enum"] = details["enum"]
+            if "enum" in details:
+                prop["enum"] = details["enum"]
+            structured_output["parameters"]["properties"][param] = prop
 
     if not allow_optional:
-        # Add all properties to required list
         structured_output["parameters"]["required"] = list(structured_output["parameters"]["properties"].keys())
-
     else:
-        # See what parameters exist that aren't required
-        # Those are implied "optional" types
-        # For those types, turn each of them into a union type with "null"
-        # e.g.
-        # "type": "string" -> "type": ["string", "null"]
-        # TODO
-        raise NotImplementedError
+        raise NotImplementedError("Optional parameter handling is not implemented.")
 
     return structured_output
 
@@ -292,6 +296,8 @@ def unpack_inner_thoughts_from_kwargs(choice: Choice, inner_thoughts_key: str) -
 
         except json.JSONDecodeError as e:
             warnings.warn(f"Failed to strip inner thoughts from kwargs: {e}")
+            print(f"\nFailed to strip inner thoughts from kwargs: {e}")
+            print(f"\nTool call arguments: {tool_call.function.arguments}")
             raise e
     else:
         warnings.warn(f"Did not find tool call in message: {str(message)}")

letta/llm_api/llm_api_tools.py

@@ -26,6 +26,7 @@ from letta.local_llm.chat_completion_proxy import get_chat_completion
 from letta.local_llm.constants import INNER_THOUGHTS_KWARG, INNER_THOUGHTS_KWARG_DESCRIPTION
 from letta.local_llm.utils import num_tokens_from_functions, num_tokens_from_messages
 from letta.orm.user import User
+from letta.otel.tracing import log_event, trace_method
 from letta.schemas.enums import ProviderCategory
 from letta.schemas.llm_config import LLMConfig
 from letta.schemas.message import Message
@@ -35,7 +36,6 @@ from letta.schemas.provider_trace import ProviderTraceCreate
 from letta.services.telemetry_manager import TelemetryManager
 from letta.settings import ModelSettings
 from letta.streaming_interface import AgentChunkStreamingInterface, AgentRefreshStreamingInterface
-from letta.tracing import log_event, trace_method
 
 LLM_API_PROVIDER_OPTIONS = ["openai", "azure", "anthropic", "google_ai", "cohere", "local", "groq", "deepseek"]
 

letta/llm_api/llm_client_base.py

@@ -1,3 +1,4 @@
+import json
 from abc import abstractmethod
 from typing import TYPE_CHECKING, Dict, List, Optional, Union
 
@@ -6,13 +7,13 @@ from openai import AsyncStream, Stream
 from openai.types.chat.chat_completion_chunk import ChatCompletionChunk
 
 from letta.errors import LLMError
+from letta.otel.tracing import log_event, trace_method
 from letta.schemas.embedding_config import EmbeddingConfig
 from letta.schemas.llm_config import LLMConfig
 from letta.schemas.message import Message
 from letta.schemas.openai.chat_completion_response import ChatCompletionResponse
 from letta.schemas.provider_trace import ProviderTraceCreate
 from letta.services.telemetry_manager import TelemetryManager
-from letta.tracing import log_event, trace_method
 
 if TYPE_CHECKING:
     from letta.orm import User
@@ -186,3 +187,30 @@ class LLMClientBase:
             An LLMError subclass that represents the error in a provider-agnostic way
         """
         return LLMError(f"Unhandled LLM error: {str(e)}")
+
+    def _fix_truncated_json_response(self, response: ChatCompletionResponse) -> ChatCompletionResponse:
+        """
+        Fixes truncated JSON responses by ensuring the content is properly formatted.
+        This is a workaround for some providers that may return incomplete JSON.
+        """
+        if response.choices and response.choices[0].message and response.choices[0].message.tool_calls:
+            tool_call_args_str = response.choices[0].message.tool_calls[0].function.arguments
+            try:
+                json.loads(tool_call_args_str)
+            except json.JSONDecodeError:
+                try:
+                    json_str_end = ""
+                    quote_count = tool_call_args_str.count('"')
+                    if quote_count % 2 != 0:
+                        json_str_end = json_str_end + '"'
+
+                    open_braces = tool_call_args_str.count("{")
+                    close_braces = tool_call_args_str.count("}")
+                    missing_braces = open_braces - close_braces
+                    json_str_end += "}" * missing_braces
+                    fixed_tool_call_args_str = tool_call_args_str[: -len(json_str_end)] + json_str_end
+                    json.loads(fixed_tool_call_args_str)
+                    response.choices[0].message.tool_calls[0].function.arguments = fixed_tool_call_args_str
+                except json.JSONDecodeError:
+                    pass
+        return response