letta-nightly 0.11.7.dev20250909104137__py3-none-any.whl → 0.11.7.dev20250910104051__py3-none-any.whl

letta/adapters/letta_llm_adapter.py

@@ -0,0 +1,81 @@
+from abc import ABC, abstractmethod
+from typing import AsyncGenerator
+
+from letta.llm_api.llm_client_base import LLMClientBase
+from letta.schemas.letta_message import LettaMessage
+from letta.schemas.letta_message_content import ReasoningContent, RedactedReasoningContent, TextContent
+from letta.schemas.llm_config import LLMConfig
+from letta.schemas.openai.chat_completion_response import ChatCompletionResponse, ToolCall
+from letta.schemas.usage import LettaUsageStatistics
+from letta.schemas.user import User
+from letta.services.telemetry_manager import TelemetryManager
+
+
+class LettaLLMAdapter(ABC):
+    """
+    Base adapter for handling LLM calls in a unified way.
+
+    This abstract class defines the interface for both blocking and streaming
+    LLM interactions, allowing the agent to use different execution modes
+    through a consistent API.
+    """
+
+    def __init__(self, llm_client: LLMClientBase, llm_config: LLMConfig) -> None:
+        self.llm_client: LLMClientBase = llm_client
+        self.llm_config: LLMConfig = llm_config
+        self.message_id: str | None = None
+        self.request_data: dict | None = None
+        self.response_data: dict | None = None
+        self.chat_completions_response: ChatCompletionResponse | None = None
+        self.reasoning_content: list[TextContent | ReasoningContent | RedactedReasoningContent] | None = None
+        self.tool_call: ToolCall | None = None
+        self.usage: LettaUsageStatistics = LettaUsageStatistics()
+        self.telemetry_manager: TelemetryManager = TelemetryManager()
+        self.llm_request_finish_timestamp_ns: int | None = None
+
+    @abstractmethod
+    async def invoke_llm(
+        self,
+        request_data: dict,
+        messages: list,
+        tools: list,
+        use_assistant_message: bool,
+        requires_approval_tools: list[str] = [],
+        step_id: str | None = None,
+        actor: User | None = None,
+    ) -> AsyncGenerator[LettaMessage | None, None]:
+        """
+        Execute the LLM call and yield results as they become available.
+
+        Args:
+            request_data: The prepared request data for the LLM API
+            messages: The messages in context for the request
+            tools: The tools available for the LLM to use
+            use_assistant_message: If true, use assistant messages when streaming response
+            requires_approval_tools: The subset of tools that require approval before use
+            step_id: The step ID associated with this request. If provided, logs request and response data.
+            actor: The optional actor associated with this request for logging purposes.
+
+        Yields:
+            LettaMessage: Chunks of data for streaming adapters, or None for blocking adapters
+        """
+        raise NotImplementedError
+
+    def supports_token_streaming(self) -> bool:
+        """
+        Check if the adapter supports token-level streaming.
+
+        Returns:
+            bool: True if the adapter can stream back tokens as they are generated, False otherwise
+        """
+        return False
+
+    def log_provider_trace(self, step_id: str | None, actor: User | None) -> None:
+        """
+        Log provider trace data for telemetry purposes.
+
+        Args:
+            step_id: The step ID associated with this request for logging purposes
+            actor: The user associated with this request for logging purposes
+        """
+        raise NotImplementedError

letta/adapters/letta_llm_request_adapter.py

@@ -0,0 +1,111 @@
+import asyncio
+from typing import AsyncGenerator
+
+from letta.adapters.letta_llm_adapter import LettaLLMAdapter
+from letta.helpers.datetime_helpers import get_utc_timestamp_ns
+from letta.schemas.letta_message import LettaMessage
+from letta.schemas.letta_message_content import OmittedReasoningContent, ReasoningContent, TextContent
+from letta.schemas.provider_trace import ProviderTraceCreate
+from letta.schemas.user import User
+from letta.settings import settings
+
+
+class LettaLLMRequestAdapter(LettaLLMAdapter):
+    """
+    Adapter for handling blocking (non-streaming) LLM requests.
+
+    This adapter makes synchronous requests to the LLM and returns complete
+    responses. It extracts reasoning content, tool calls, and usage statistics
+    from the response and updates instance variables for access by the agent.
+    """
+
+    async def invoke_llm(
+        self,
+        request_data: dict,
+        messages: list,
+        tools: list,
+        use_assistant_message: bool,
+        requires_approval_tools: list[str] = [],
+        step_id: str | None = None,
+        actor: str | None = None,
+    ) -> AsyncGenerator[LettaMessage | None, None]:
+        """
+        Execute a blocking LLM request and yield the response.
+
+        This adapter:
+        1. Makes a blocking request to the LLM
+        2. Converts the response to chat completion format
+        3. Extracts reasoning and tool call information
+        4. Updates all instance variables
+        5. Yields nothing (blocking mode doesn't stream)
+        """
+        # Store request data
+        self.request_data = request_data
+
+        # Make the blocking LLM request
+        self.response_data = await self.llm_client.request_async(request_data, self.llm_config)
+        self.llm_request_finish_timestamp_ns = get_utc_timestamp_ns()
+
+        # Convert response to chat completion format
+        self.chat_completions_response = self.llm_client.convert_response_to_chat_completion(self.response_data, messages, self.llm_config)
+
+        # Extract reasoning content from the response
+        if self.chat_completions_response.choices[0].message.reasoning_content:
+            self.reasoning_content = [
+                ReasoningContent(
+                    reasoning=self.chat_completions_response.choices[0].message.reasoning_content,
+                    is_native=True,
+                    signature=self.chat_completions_response.choices[0].message.reasoning_content_signature,
+                )
+            ]
+        elif self.chat_completions_response.choices[0].message.omitted_reasoning_content:
+            self.reasoning_content = [OmittedReasoningContent()]
+        elif self.chat_completions_response.choices[0].message.content:
+            # Reasoning placed into content for legacy reasons
+            self.reasoning_content = [TextContent(text=self.chat_completions_response.choices[0].message.content)]
+        else:
+            # logger.info("No reasoning content found.")
+            self.reasoning_content = None
+
+        # Extract tool call
+        if self.chat_completions_response.choices[0].message.tool_calls:
+            self.tool_call = self.chat_completions_response.choices[0].message.tool_calls[0]
+        else:
+            self.tool_call = None
+
+        # Extract usage statistics
+        self.usage.step_count = 1
+        self.usage.completion_tokens = self.chat_completions_response.usage.completion_tokens
+        self.usage.prompt_tokens = self.chat_completions_response.usage.prompt_tokens
+        self.usage.total_tokens = self.chat_completions_response.usage.total_tokens
+
+        self.log_provider_trace(step_id=step_id, actor=actor)
+
+        yield None
+        return
+
+    def log_provider_trace(self, step_id: str | None, actor: User | None) -> None:
+        """
+        Log provider trace data for telemetry purposes in a fire-and-forget manner.
+
+        Creates an async task to log the request/response data without blocking
+        the main execution flow. The task runs in the background.
+
+        Args:
+            step_id: The step ID associated with this request for logging purposes
+            actor: The user associated with this request for logging purposes
+        """
+        if step_id is None or actor is None or not settings.track_provider_trace:
+            return
+
+        asyncio.create_task(
+            self.telemetry_manager.create_provider_trace_async(
+                actor=actor,
+                provider_trace_create=ProviderTraceCreate(
+                    request_json=self.request_data,
+                    response_json=self.response_data,
+                    step_id=step_id,  # Use original step_id for telemetry
+                    organization_id=actor.organization_id,
+                ),
+            )
+        )

letta/adapters/letta_llm_stream_adapter.py

@@ -0,0 +1,169 @@
+import asyncio
+from typing import AsyncGenerator
+
+from letta.adapters.letta_llm_adapter import LettaLLMAdapter
+from letta.helpers.datetime_helpers import get_utc_timestamp_ns
+from letta.interfaces.anthropic_streaming_interface import AnthropicStreamingInterface
+from letta.interfaces.openai_streaming_interface import OpenAIStreamingInterface
+from letta.llm_api.llm_client_base import LLMClientBase
+from letta.schemas.enums import ProviderType
+from letta.schemas.letta_message import LettaMessage
+from letta.schemas.llm_config import LLMConfig
+from letta.schemas.provider_trace import ProviderTraceCreate
+from letta.schemas.usage import LettaUsageStatistics
+from letta.schemas.user import User
+from letta.settings import settings
+
+
+class LettaLLMStreamAdapter(LettaLLMAdapter):
+    """
+    Adapter for handling streaming LLM requests with immediate token yielding.
+
+    This adapter supports real-time streaming of tokens from the LLM, providing
+    minimal time-to-first-token (TTFT) latency. It uses specialized streaming
+    interfaces for different providers (OpenAI, Anthropic) to handle their
+    specific streaming formats.
+    """
+
+    def __init__(self, llm_client: LLMClientBase, llm_config: LLMConfig) -> None:
+        super().__init__(llm_client, llm_config)
+        self.interface: OpenAIStreamingInterface | AnthropicStreamingInterface | None = None
+
+    async def invoke_llm(
+        self,
+        request_data: dict,
+        messages: list,
+        tools: list,
+        use_assistant_message: bool,
+        requires_approval_tools: list[str] = [],
+        step_id: str | None = None,
+        actor: User | None = None,
+    ) -> AsyncGenerator[LettaMessage, None]:
+        """
+        Execute a streaming LLM request and yield tokens/chunks as they arrive.
+
+        This adapter:
+        1. Makes a streaming request to the LLM
+        2. Yields chunks immediately for minimal TTFT
+        3. Accumulates response data through the streaming interface
+        4. Updates all instance variables after streaming completes
+        """
+        # Store request data
+        self.request_data = request_data
+
+        # Instantiate streaming interface
+        if self.llm_config.model_endpoint_type in [ProviderType.anthropic, ProviderType.bedrock]:
+            self.interface = AnthropicStreamingInterface(
+                use_assistant_message=use_assistant_message,
+                put_inner_thoughts_in_kwarg=self.llm_config.put_inner_thoughts_in_kwargs,
+                requires_approval_tools=requires_approval_tools,
+            )
+        elif self.llm_config.model_endpoint_type == ProviderType.openai:
+            self.interface = OpenAIStreamingInterface(
+                use_assistant_message=use_assistant_message,
+                is_openai_proxy=self.llm_config.provider_name == "lmstudio_openai",
+                put_inner_thoughts_in_kwarg=self.llm_config.put_inner_thoughts_in_kwargs,
+                messages=messages,
+                tools=tools,
+                requires_approval_tools=requires_approval_tools,
+            )
+        else:
+            raise ValueError(f"Streaming not supported for provider {self.llm_config.model_endpoint_type}")
+
+        # Extract optional parameters
+        # ttft_span = kwargs.get('ttft_span', None)
+
+        # Start the streaming request
+        stream = await self.llm_client.stream_async(request_data, self.llm_config)
+
+        # Process the stream and yield chunks immediately for TTFT
+        async for chunk in self.interface.process(stream):  # TODO: add ttft span
+            # Yield each chunk immediately as it arrives
+            yield chunk
+
+        # After streaming completes, extract the accumulated data
+        self.llm_request_finish_timestamp_ns = get_utc_timestamp_ns()
+
+        # Extract tool call from the interface
+        try:
+            self.tool_call = self.interface.get_tool_call_object()
+        except ValueError as e:
+            # No tool call, handle upstream
+            self.tool_call = None
+
+        # Extract reasoning content from the interface
+        self.reasoning_content = self.interface.get_reasoning_content()
+
+        # Extract usage statistics
+        # Some providers don't provide usage in streaming, use fallback if needed
+        if hasattr(self.interface, "input_tokens") and hasattr(self.interface, "output_tokens"):
+            # Handle cases where tokens might not be set (e.g., LMStudio)
+            input_tokens = self.interface.input_tokens
+            output_tokens = self.interface.output_tokens
+
+            # Fallback to estimated values if not provided
+            if not input_tokens and hasattr(self.interface, "fallback_input_tokens"):
+                input_tokens = self.interface.fallback_input_tokens
+            if not output_tokens and hasattr(self.interface, "fallback_output_tokens"):
+                output_tokens = self.interface.fallback_output_tokens
+
+            self.usage = LettaUsageStatistics(
+                step_count=1,
+                completion_tokens=output_tokens or 0,
+                prompt_tokens=input_tokens or 0,
+                total_tokens=(input_tokens or 0) + (output_tokens or 0),
+            )
+        else:
+            # Default usage statistics if not available
+            self.usage = LettaUsageStatistics(step_count=1, completion_tokens=0, prompt_tokens=0, total_tokens=0)
+
+        # Store any additional data from the interface
+        self.message_id = self.interface.letta_message_id
+
+        # Log request and response data
+        self.log_provider_trace(step_id=step_id, actor=actor)
+
+    def supports_token_streaming(self) -> bool:
+        return True
+
+    def log_provider_trace(self, step_id: str | None, actor: User | None) -> None:
+        """
+        Log provider trace data for telemetry purposes in a fire-and-forget manner.
+
+        Creates an async task to log the request/response data without blocking
+        the main execution flow. For streaming adapters, this includes the final
+        tool call and reasoning content collected during streaming.
+
+        Args:
+            step_id: The step ID associated with this request for logging purposes
+            actor: The user associated with this request for logging purposes
+        """
+        if step_id is None or actor is None or not settings.track_provider_trace:
+            return
+
+        asyncio.create_task(
+            self.telemetry_manager.create_provider_trace_async(
+                actor=actor,
+                provider_trace_create=ProviderTraceCreate(
+                    request_json=self.request_data,
+                    response_json={
+                        "content": {
+                            "tool_call": self.tool_call.model_dump_json(),
+                            "reasoning": [content.model_dump_json() for content in self.reasoning_content],
+                        },
+                        "id": self.interface.message_id,
+                        "model": self.interface.model,
+                        "role": "assistant",
+                        # "stop_reason": "",
+                        # "stop_sequence": None,
+                        "type": "message",
+                        "usage": {
+                            "input_tokens": self.usage.prompt_tokens,
+                            "output_tokens": self.usage.completion_tokens,
+                        },
+                    },
+                    step_id=step_id,  # Use original step_id for telemetry
+                    organization_id=actor.organization_id,
+                ),
+            )
+        )

letta/agents/base_agent.py

@@ -175,7 +175,10 @@ class BaseAgent(ABC):
 
                 # [DB Call] Update Messages
                 new_system_message = await self.message_manager.update_message_by_id_async(
-                    curr_system_message.id, message_update=MessageUpdate(content=new_system_message_str), actor=self.actor
+                    curr_system_message.id,
+                    message_update=MessageUpdate(content=new_system_message_str),
+                    actor=self.actor,
+                    project_id=agent_state.project_id,
                 )
                 return [new_system_message] + in_context_messages[1:]
 

letta/agents/base_agent_v2.py

@@ -0,0 +1,68 @@
+from abc import ABC, abstractmethod
+from typing import AsyncGenerator
+
+from letta.constants import DEFAULT_MAX_STEPS
+from letta.log import get_logger
+from letta.schemas.agent import AgentState
+from letta.schemas.enums import MessageStreamStatus
+from letta.schemas.letta_message import LegacyLettaMessage, LettaMessage, MessageType
+from letta.schemas.letta_response import LettaResponse
+from letta.schemas.message import MessageCreate
+from letta.schemas.user import User
+
+
+class BaseAgentV2(ABC):
+    """
+    Abstract base class for the main agent execution loop for letta agents, handling
+    message management, llm api request, tool execution, and context tracking.
+    """
+
+    def __init__(self, agent_state: AgentState, actor: User):
+        self.agent_state = agent_state
+        self.actor = actor
+        self.logger = get_logger(agent_state.id)
+
+    @abstractmethod
+    async def build_request(
+        self,
+        input_messages: list[MessageCreate],
+    ) -> dict:
+        """
+        Execute the agent loop in dry_run mode, returning just the generated request
+        payload sent to the underlying llm provider.
+        """
+        raise NotImplementedError
+
+    @abstractmethod
+    async def step(
+        self,
+        input_messages: list[MessageCreate],
+        max_steps: int = DEFAULT_MAX_STEPS,
+        run_id: str | None = None,
+        use_assistant_message: bool = True,
+        include_return_message_types: list[MessageType] | None = None,
+        request_start_timestamp_ns: int | None = None,
+    ) -> LettaResponse:
+        """
+        Execute the agent loop in blocking mode, returning all messages at once.
+        """
+        raise NotImplementedError
+
+    @abstractmethod
+    async def stream(
+        self,
+        input_messages: list[MessageCreate],
+        max_steps: int = DEFAULT_MAX_STEPS,
+        stream_tokens: bool = False,
+        run_id: str | None = None,
+        use_assistant_message: bool = True,
+        include_return_message_types: list[MessageType] | None = None,
+        request_start_timestamp_ns: int | None = None,
+    ) -> AsyncGenerator[LettaMessage | LegacyLettaMessage | MessageStreamStatus, None]:
+        """
+        Execute the agent loop in streaming mode, yielding chunks as they become available.
+        If stream_tokens is True, individual tokens are streamed as they arrive from the LLM,
+        providing the lowest latency experience, otherwise each complete step (reasoning +
+        tool call + tool return) is yielded as it completes.
+        """
+        raise NotImplementedError

letta/agents/helpers.py

@@ -3,6 +3,7 @@ import uuid
 import xml.etree.ElementTree as ET
 from typing import List, Optional, Tuple
 
+from letta.errors import PendingApprovalError
 from letta.helpers import ToolRulesSolver
 from letta.log import get_logger
 from letta.schemas.agent import AgentState
@@ -117,7 +118,7 @@ async def _prepare_in_context_messages_async(
     new_in_context_messages = await message_manager.create_many_messages_async(
         create_input_messages(input_messages=input_messages, agent_id=agent_state.id, timezone=agent_state.timezone, actor=actor),
         actor=actor,
-        embedding_config=agent_state.embedding_config,
+        project_id=agent_state.project_id,
     )
 
     return current_in_context_messages, new_in_context_messages
@@ -168,10 +169,7 @@ async def _prepare_in_context_messages_no_persist_async(
     else:
         # User is trying to send a regular message
         if current_in_context_messages[-1].role == "approval":
-            raise ValueError(
-                "Cannot send a new message: The agent is waiting for approval on a tool call. "
-                "Please approve or deny the pending request before continuing."
-            )
+            raise PendingApprovalError(pending_request_id=current_in_context_messages[-1].id)
 
         # Create a new user message from the input but dont store it yet
         new_in_context_messages = create_input_messages(

letta/agents/letta_agent.py

@@ -495,7 +495,10 @@ class LettaAgent(BaseAgent):
                                     message.is_err = True
                                     message.step_id = effective_step_id
                                 await self.message_manager.create_many_messages_async(
-                                    initial_messages, actor=self.actor, embedding_config=agent_state.embedding_config
+                                    initial_messages,
+                                    actor=self.actor,
+                                    project_id=agent_state.project_id,
+                                    template_id=agent_state.template_id,
                                 )
                         elif step_progression <= StepProgression.LOGGED_TRACE:
                             if stop_reason is None:
@@ -823,7 +826,10 @@ class LettaAgent(BaseAgent):
                                     message.is_err = True
                                     message.step_id = effective_step_id
                                 await self.message_manager.create_many_messages_async(
-                                    initial_messages, actor=self.actor, embedding_config=agent_state.embedding_config
+                                    initial_messages,
+                                    actor=self.actor,
+                                    project_id=agent_state.project_id,
+                                    template_id=agent_state.template_id,
                                 )
                         elif step_progression <= StepProgression.LOGGED_TRACE:
                             if stop_reason is None:
@@ -1018,6 +1024,7 @@ class LettaAgent(BaseAgent):
                         interface = AnthropicStreamingInterface(
                             use_assistant_message=use_assistant_message,
                             put_inner_thoughts_in_kwarg=agent_state.llm_config.put_inner_thoughts_in_kwargs,
+                            requires_approval_tools=tool_rules_solver.get_requires_approval_tools(valid_tool_names),
                         )
                     elif agent_state.llm_config.model_endpoint_type == ProviderType.openai:
                         interface = OpenAIStreamingInterface(
@@ -1026,6 +1033,7 @@ class LettaAgent(BaseAgent):
                             messages=current_in_context_messages + new_in_context_messages,
                             tools=request_data.get("tools", []),
                             put_inner_thoughts_in_kwarg=agent_state.llm_config.put_inner_thoughts_in_kwargs,
+                            requires_approval_tools=tool_rules_solver.get_requires_approval_tools(valid_tool_names),
                         )
                     else:
                         raise ValueError(f"Streaming not supported for {agent_state.llm_config}")
@@ -1170,12 +1178,13 @@ class LettaAgent(BaseAgent):
                         )
                         step_progression = StepProgression.LOGGED_TRACE
 
-                    # yields tool response as this is handled from Letta and not the response from the LLM provider
-                    tool_return = [msg for msg in persisted_messages if msg.role == "tool"][-1].to_letta_messages()[0]
-                    if not (use_assistant_message and tool_return.name == "send_message"):
-                        # Apply message type filtering if specified
-                        if include_return_message_types is None or tool_return.message_type in include_return_message_types:
-                            yield f"data: {tool_return.model_dump_json()}\n\n"
+                    if persisted_messages[-1].role != "approval":
+                        # yields tool response as this is handled from Letta and not the response from the LLM provider
+                        tool_return = [msg for msg in persisted_messages if msg.role == "tool"][-1].to_letta_messages()[0]
+                        if not (use_assistant_message and tool_return.name == "send_message"):
+                            # Apply message type filtering if specified
+                            if include_return_message_types is None or tool_return.message_type in include_return_message_types:
+                                yield f"data: {tool_return.model_dump_json()}\n\n"
 
                     # TODO (cliandy): consolidate and expand with trace
                     MetricRegistry().step_execution_time_ms_histogram.record(get_utc_timestamp_ns() - step_start, get_ctx_attributes())
@@ -1259,7 +1268,10 @@ class LettaAgent(BaseAgent):
                                     message.is_err = True
                                     message.step_id = effective_step_id
                                 await self.message_manager.create_many_messages_async(
-                                    initial_messages, actor=self.actor, embedding_config=agent_state.embedding_config
+                                    initial_messages,
+                                    actor=self.actor,
+                                    project_id=agent_state.project_id,
+                                    template_id=agent_state.template_id,
                                 )
                         elif step_progression <= StepProgression.LOGGED_TRACE:
                             if stop_reason is None:
@@ -1667,7 +1679,7 @@ class LettaAgent(BaseAgent):
             )
             messages_to_persist = (initial_messages or []) + tool_call_messages
             persisted_messages = await self.message_manager.create_many_messages_async(
-                messages_to_persist, actor=self.actor, embedding_config=agent_state.embedding_config
+                messages_to_persist, actor=self.actor, project_id=agent_state.project_id, template_id=agent_state.template_id
             )
             return persisted_messages, continue_stepping, stop_reason
 
@@ -1686,7 +1698,6 @@ class LettaAgent(BaseAgent):
             tool_call_id=tool_call_id,
             request_heartbeat=request_heartbeat,
         )
-
         if not is_approval and tool_rules_solver.is_requires_approval_tool(tool_call_name):
             approval_message = create_approval_request_message_from_llm_response(
                 agent_id=agent_state.id,
@@ -1779,7 +1790,7 @@ class LettaAgent(BaseAgent):
             messages_to_persist = (initial_messages or []) + tool_call_messages
 
         persisted_messages = await self.message_manager.create_many_messages_async(
-            messages_to_persist, actor=self.actor, embedding_config=agent_state.embedding_config
+            messages_to_persist, actor=self.actor, project_id=agent_state.project_id, template_id=agent_state.template_id
         )
 
         if run_id: