letta-nightly 0.7.5.dev20250428110034__py3-none-any.whl → 0.7.6.dev20250429062643__py3-none-any.whl

letta/server/rest_api/routers/openai/chat_completions/chat_completions.py

@@ -6,14 +6,14 @@ from fastapi.responses import StreamingResponse
 from openai.types.chat.completion_create_params import CompletionCreateParams
 
 from letta.agent import Agent
-from letta.constants import DEFAULT_MESSAGE_TOOL, DEFAULT_MESSAGE_TOOL_KWARG
+from letta.constants import DEFAULT_MESSAGE_TOOL, DEFAULT_MESSAGE_TOOL_KWARG, LETTA_MODEL_ENDPOINT
 from letta.log import get_logger
 from letta.schemas.message import Message, MessageCreate
 from letta.schemas.user import User
 from letta.server.rest_api.chat_completions_interface import ChatCompletionsStreamingInterface
 
 # TODO this belongs in a controller!
-from letta.server.rest_api.utils import get_letta_server, get_messages_from_completion_request, sse_async_generator
+from letta.server.rest_api.utils import get_letta_server, get_user_message_from_chat_completions_request, sse_async_generator
 
 if TYPE_CHECKING:
     from letta.server.server import SyncServer
@@ -43,10 +43,6 @@ async def create_chat_completions(
     user_id: Optional[str] = Header(None, alias="user_id"),
 ):
     # Validate and process fields
-    messages = get_messages_from_completion_request(completion_request)
-    input_message = messages[-1]
-
-    # Process remaining fields
     if not completion_request["stream"]:
         raise HTTPException(status_code=400, detail="Must be streaming request: `stream` was set to `False` in the request.")
 
@@ -54,7 +50,7 @@ async def create_chat_completions(
 
     letta_agent = server.load_agent(agent_id=agent_id, actor=actor)
     llm_config = letta_agent.agent_state.llm_config
-    if llm_config.model_endpoint_type != "openai" or "inference.memgpt.ai" in llm_config.model_endpoint:
+    if llm_config.model_endpoint_type != "openai" or llm_config.model_endpoint == LETTA_MODEL_ENDPOINT:
         error_msg = f"You can only use models with type 'openai' for chat completions. This agent {agent_id} has llm_config: \n{llm_config.model_dump_json(indent=4)}"
         logger.error(error_msg)
         raise HTTPException(status_code=400, detail=error_msg)
@@ -65,13 +61,11 @@ async def create_chat_completions(
         logger.warning(f"Defaulting to {llm_config.model}...")
         logger.warning(warning_msg)
 
-    logger.info(f"Received input message: {input_message}")
-
     return await send_message_to_agent_chat_completions(
         server=server,
         letta_agent=letta_agent,
         actor=actor,
-        messages=[MessageCreate(role=input_message["role"], content=input_message["content"])],
+        messages=get_user_message_from_chat_completions_request(completion_request),
     )
 
 

letta/server/rest_api/routers/v1/voice.py

@@ -1,6 +1,5 @@
 from typing import TYPE_CHECKING, Optional
 
-import httpx
 import openai
 from fastapi import APIRouter, Body, Depends, Header
 from fastapi.responses import StreamingResponse
@@ -8,8 +7,7 @@ from openai.types.chat.completion_create_params import CompletionCreateParams
 
 from letta.agents.voice_agent import VoiceAgent
 from letta.log import get_logger
-from letta.schemas.openai.chat_completions import UserMessage
-from letta.server.rest_api.utils import get_letta_server, get_messages_from_completion_request
+from letta.server.rest_api.utils import get_letta_server, get_user_message_from_chat_completions_request
 from letta.settings import model_settings
 
 if TYPE_CHECKING:
@@ -42,22 +40,11 @@ async def create_voice_chat_completions(
 ):
     actor = server.user_manager.get_user_or_default(user_id=user_id)
 
-    # Also parse the user's new input
-    input_message = UserMessage(**get_messages_from_completion_request(completion_request)[-1])
-
     # Create OpenAI async client
     client = openai.AsyncClient(
         api_key=model_settings.openai_api_key,
         max_retries=0,
-        http_client=httpx.AsyncClient(
-            timeout=httpx.Timeout(connect=15.0, read=30.0, write=15.0, pool=15.0),
-            follow_redirects=True,
-            limits=httpx.Limits(
-                max_connections=50,
-                max_keepalive_connections=50,
-                keepalive_expiry=120,
-            ),
-        ),
+        http_client=server.httpx_client,
     )
 
     # Instantiate our LowLatencyAgent
@@ -67,10 +54,13 @@ async def create_voice_chat_completions(
         message_manager=server.message_manager,
         agent_manager=server.agent_manager,
         block_manager=server.block_manager,
+        passage_manager=server.passage_manager,
         actor=actor,
-        message_buffer_limit=50,
-        message_buffer_min=10,
+        message_buffer_limit=40,
+        message_buffer_min=15,
     )
 
     # Return the streaming generator
-    return StreamingResponse(agent.step_stream(input_message=input_message), media_type="text/event-stream")
+    return StreamingResponse(
+        agent.step_stream(input_messages=get_user_message_from_chat_completions_request(completion_request)), media_type="text/event-stream"
+    )

letta/server/rest_api/utils.py

@@ -210,19 +210,20 @@ def create_letta_messages_from_llm_response(
 
     # TODO: Use ToolReturnContent instead of TextContent
     # TODO: This helps preserve ordering
-    tool_message = Message(
-        role=MessageRole.tool,
-        content=[TextContent(text=package_function_response(function_call_success, function_response))],
-        organization_id=actor.organization_id,
-        agent_id=agent_id,
-        model=model,
-        tool_calls=[],
-        tool_call_id=tool_call_id,
-        created_at=get_utc_time(),
-    )
-    if pre_computed_tool_message_id:
-        tool_message.id = pre_computed_tool_message_id
-    messages.append(tool_message)
+    if function_response:
+        tool_message = Message(
+            role=MessageRole.tool,
+            content=[TextContent(text=package_function_response(function_call_success, function_response))],
+            organization_id=actor.organization_id,
+            agent_id=agent_id,
+            model=model,
+            tool_calls=[],
+            tool_call_id=tool_call_id,
+            created_at=get_utc_time(),
+        )
+        if pre_computed_tool_message_id:
+            tool_message.id = pre_computed_tool_message_id
+        messages.append(tool_message)
 
     if add_heartbeat_request_system_message:
         heartbeat_system_message = create_heartbeat_system_message(
@@ -278,7 +279,7 @@ def create_assistant_messages_from_openai_response(
     )
 
 
-def convert_letta_messages_to_openai(messages: List[Message]) -> List[dict]:
+def convert_in_context_letta_messages_to_openai(in_context_messages: List[Message], exclude_system_messages: bool = False) -> List[dict]:
     """
     Flattens Letta's messages (with system, user, assistant, tool roles, etc.)
     into standard OpenAI chat messages (system, user, assistant).
@@ -289,10 +290,15 @@ def convert_letta_messages_to_openai(messages: List[Message]) -> List[dict]:
       3. User messages might store actual text inside JSON => parse that into content
       4. System => pass through as normal
     """
+    # Always include the system prompt
+    # TODO: This is brittle
+    openai_messages = [in_context_messages[0].to_openai_dict()]
 
-    openai_messages = []
+    for msg in in_context_messages[1:]:
+        if msg.role == MessageRole.system and exclude_system_messages:
+            # Skip if exclude_system_messages is set to True
+            continue
 
-    for msg in messages:
         # 1. Assistant + 'send_message' tool_calls => flatten
         if msg.role == MessageRole.assistant and msg.tool_calls:
             # Find any 'send_message' tool_calls
@@ -350,15 +356,13 @@ def convert_letta_messages_to_openai(messages: List[Message]) -> List[dict]:
                 except json.JSONDecodeError:
                     pass  # It's not JSON, leave as-is
 
-        # 4. System is left as-is (or any other role that doesn't need special handling)
-        #
         # Finally, convert to dict using your existing method
         openai_messages.append(msg.to_openai_dict())
 
     return openai_messages
 
 
-def get_messages_from_completion_request(completion_request: CompletionCreateParams) -> List[Dict]:
+def get_user_message_from_chat_completions_request(completion_request: CompletionCreateParams) -> List[MessageCreate]:
     try:
         messages = list(cast(Iterable[ChatCompletionMessageParam], completion_request["messages"]))
     except KeyError:
@@ -380,4 +384,6 @@ def get_messages_from_completion_request(completion_request: CompletionCreatePar
         logger.error(f"The input message does not have valid content: {input_message}")
         raise HTTPException(status_code=400, detail="'messages[-1].content' must be a 'string'")
 
-    return messages
+    for message in reversed(messages):
+        if message["role"] == "user":
+            return [MessageCreate(role=MessageRole.user, content=[TextContent(text=message["content"])])]

letta/server/server.py

@@ -1,4 +1,3 @@
-# inspecting tools
 import asyncio
 import json
 import os
@@ -6,8 +5,10 @@ import traceback
 import warnings
 from abc import abstractmethod
 from datetime import datetime
+from pathlib import Path
 from typing import Any, Callable, Dict, List, Optional, Tuple, Union
 
+import httpx
 from anthropic import AsyncAnthropic
 from composio.client import Composio
 from composio.client.collections import ActionModel, AppModel
@@ -19,6 +20,7 @@ import letta.server.utils as server_utils
 import letta.system as system
 from letta.agent import Agent, save_agent
 from letta.config import LettaConfig
+from letta.constants import LETTA_TOOL_EXECUTION_DIR
 from letta.data_sources.connectors import DataConnector, load_data
 from letta.errors import HandleNotFoundError
 from letta.functions.mcp_client.base_client import BaseMCPClient
@@ -70,7 +72,7 @@ from letta.schemas.providers import (
     VLLMCompletionsProvider,
     XAIProvider,
 )
-from letta.schemas.sandbox_config import SandboxType
+from letta.schemas.sandbox_config import LocalSandboxConfig, SandboxConfigCreate, SandboxType
 from letta.schemas.source import Source
 from letta.schemas.tool import Tool
 from letta.schemas.usage import LettaUsageStatistics
@@ -81,6 +83,7 @@ from letta.server.rest_api.utils import sse_async_generator
 from letta.services.agent_manager import AgentManager
 from letta.services.block_manager import BlockManager
 from letta.services.group_manager import GroupManager
+from letta.services.helpers.tool_execution_helper import prepare_local_sandbox
 from letta.services.identity_manager import IdentityManager
 from letta.services.job_manager import JobManager
 from letta.services.llm_batch_manager import LLMBatchManager
@@ -211,6 +214,11 @@ class SyncServer(Server):
         self.group_manager = GroupManager()
         self.batch_manager = LLMBatchManager()
 
+        # A resusable httpx client
+        timeout = httpx.Timeout(connect=10.0, read=20.0, write=10.0, pool=10.0)
+        limits = httpx.Limits(max_connections=100, max_keepalive_connections=80, keepalive_expiry=300)
+        self.httpx_client = httpx.AsyncClient(timeout=timeout, follow_redirects=True, limits=limits)
+
         # Make default user and org
         if init_with_default_org_and_user:
             self.default_org = self.organization_manager.create_default_organization()
@@ -229,6 +237,36 @@ class SyncServer(Server):
                     actor=self.default_user,
                 )
 
+            # For OSS users, create a local sandbox config
+            oss_default_user = self.user_manager.get_default_user()
+            use_venv = False if not tool_settings.tool_exec_venv_name else True
+            venv_name = tool_settings.tool_exec_venv_name or "venv"
+            tool_dir = tool_settings.tool_exec_dir or LETTA_TOOL_EXECUTION_DIR
+
+            venv_dir = Path(tool_dir) / venv_name
+            if not Path(tool_dir).is_dir():
+                logger.error(f"Provided LETTA_TOOL_SANDBOX_DIR is not a valid directory: {tool_dir}")
+            else:
+                if tool_settings.tool_exec_venv_name and not venv_dir.is_dir():
+                    logger.warning(
+                        f"Provided LETTA_TOOL_SANDBOX_VENV_NAME is not a valid venv ({venv_dir}), one will be created for you during tool execution."
+                    )
+
+                sandbox_config_create = SandboxConfigCreate(
+                    config=LocalSandboxConfig(sandbox_dir=tool_settings.tool_exec_dir, use_venv=use_venv, venv_name=venv_name)
+                )
+                sandbox_config = self.sandbox_config_manager.create_or_update_sandbox_config(
+                    sandbox_config_create=sandbox_config_create, actor=oss_default_user
+                )
+                logger.info(f"Successfully created default local sandbox config:\n{sandbox_config.get_local_config().model_dump()}")
+
+                if use_venv and tool_settings.tool_exec_autoreload_venv:
+                    prepare_local_sandbox(
+                        sandbox_config.get_local_config(),
+                        env=os.environ.copy(),
+                        force_recreate=True,
+                    )
+
         # collect providers (always has Letta as a default)
         self._enabled_providers: List[Provider] = [LettaProvider()]
         if model_settings.openai_api_key:
@@ -325,29 +363,29 @@ class SyncServer(Server):
 
         # For MCP
         """Initialize the MCP clients (there may be multiple)"""
-        mcp_server_configs = self.get_mcp_servers()
+        # mcp_server_configs = self.get_mcp_servers()
         self.mcp_clients: Dict[str, BaseMCPClient] = {}
-
-        for server_name, server_config in mcp_server_configs.items():
-            if server_config.type == MCPServerType.SSE:
-                self.mcp_clients[server_name] = SSEMCPClient(server_config)
-            elif server_config.type == MCPServerType.STDIO:
-                self.mcp_clients[server_name] = StdioMCPClient(server_config)
-            else:
-                raise ValueError(f"Invalid MCP server config: {server_config}")
-
-            try:
-                self.mcp_clients[server_name].connect_to_server()
-            except Exception as e:
-                logger.error(e)
-                self.mcp_clients.pop(server_name)
-
-        # Print out the tools that are connected
-        for server_name, client in self.mcp_clients.items():
-            logger.info(f"Attempting to fetch tools from MCP server: {server_name}")
-            mcp_tools = client.list_tools()
-            logger.info(f"MCP tools connected: {', '.join([t.name for t in mcp_tools])}")
-            logger.debug(f"MCP tools: {', '.join([str(t) for t in mcp_tools])}")
+        #
+        # for server_name, server_config in mcp_server_configs.items():
+        #     if server_config.type == MCPServerType.SSE:
+        #         self.mcp_clients[server_name] = SSEMCPClient(server_config)
+        #     elif server_config.type == MCPServerType.STDIO:
+        #         self.mcp_clients[server_name] = StdioMCPClient(server_config)
+        #     else:
+        #         raise ValueError(f"Invalid MCP server config: {server_config}")
+        #
+        #     try:
+        #         self.mcp_clients[server_name].connect_to_server()
+        #     except Exception as e:
+        #         logger.error(e)
+        #         self.mcp_clients.pop(server_name)
+        #
+        # # Print out the tools that are connected
+        # for server_name, client in self.mcp_clients.items():
+        #     logger.info(f"Attempting to fetch tools from MCP server: {server_name}")
+        #     mcp_tools = client.list_tools()
+        #     logger.info(f"MCP tools connected: {', '.join([t.name for t in mcp_tools])}")
+        #     logger.debug(f"MCP tools: {', '.join([str(t) for t in mcp_tools])}")
 
         # TODO: Remove these in memory caches
         self._llm_config_cache = {}
@@ -1181,6 +1219,8 @@ class SyncServer(Server):
             llm_config.max_reasoning_tokens = max_reasoning_tokens
         if enable_reasoner is not None:
             llm_config.enable_reasoner = enable_reasoner
+            if enable_reasoner and llm_config.model_endpoint_type == "anthropic":
+                llm_config.put_inner_thoughts_in_kwargs = False
 
         return llm_config
 
@@ -1562,7 +1602,8 @@ class SyncServer(Server):
             # supports_token_streaming = ["openai", "anthropic", "xai", "deepseek"]
             supports_token_streaming = ["openai", "anthropic", "deepseek"]  # TODO re-enable xAI once streaming is patched
             if stream_tokens and (
-                llm_config.model_endpoint_type not in supports_token_streaming or "inference.memgpt.ai" in llm_config.model_endpoint
+                llm_config.model_endpoint_type not in supports_token_streaming
+                or llm_config.model_endpoint == constants.LETTA_MODEL_ENDPOINT
             ):
                 warnings.warn(
                     f"Token streaming is only supported for models with type {' or '.join(supports_token_streaming)} in the model_endpoint: agent has endpoint type {llm_config.model_endpoint_type} and {llm_config.model_endpoint}. Setting stream_tokens to False."
@@ -1685,7 +1726,7 @@ class SyncServer(Server):
         llm_config = letta_multi_agent.agent_state.llm_config
         supports_token_streaming = ["openai", "anthropic", "deepseek"]
         if stream_tokens and (
-            llm_config.model_endpoint_type not in supports_token_streaming or "inference.memgpt.ai" in llm_config.model_endpoint
+            llm_config.model_endpoint_type not in supports_token_streaming or llm_config.model_endpoint == constants.LETTA_MODEL_ENDPOINT
         ):
             warnings.warn(
                 f"Token streaming is only supported for models with type {' or '.join(supports_token_streaming)} in the model_endpoint: agent has endpoint type {llm_config.model_endpoint_type} and {llm_config.model_endpoint}. Setting stream_tokens to False."

letta/services/helpers/agent_manager_helper.py

@@ -6,7 +6,7 @@ from sqlalchemy import and_, asc, desc, func, literal, or_, select
 from letta import system
 from letta.constants import IN_CONTEXT_MEMORY_KEYWORD, STRUCTURED_OUTPUT_MODELS
 from letta.helpers import ToolRulesSolver
-from letta.helpers.datetime_helpers import get_local_time
+from letta.helpers.datetime_helpers import get_local_time, get_local_time_fast
 from letta.orm.agent import Agent as AgentModel
 from letta.orm.agents_tags import AgentsTags
 from letta.orm.errors import NoResultFound
@@ -119,7 +119,7 @@ def compile_memory_metadata_block(
     # Create a metadata block of info so the agent knows about the metadata of out-of-context memories
     memory_metadata_block = "\n".join(
         [
-            f"### Memory [last modified: {timestamp_str}]",
+            f"### Current Time: {get_local_time_fast()}" f"### Memory [last modified: {timestamp_str}]",
             f"{previous_message_count} previous messages between you and the user are stored in recall memory (use functions to access them)",
             f"{archival_memory_size} total memories you created are stored in archival memory (use functions to access them)",
             (

letta/services/helpers/tool_execution_helper.py

@@ -24,7 +24,7 @@ def find_python_executable(local_configs: LocalSandboxConfig) -> str:
     """
     sandbox_dir = os.path.expanduser(local_configs.sandbox_dir)  # Expand tilde
 
-    if not local_configs.force_create_venv:
+    if not local_configs.use_venv:
         return "python.exe" if platform.system().lower().startswith("win") else "python3"
 
     venv_path = os.path.join(sandbox_dir, local_configs.venv_name)
@@ -96,7 +96,7 @@ def install_pip_requirements_for_sandbox(
     python_exec = find_python_executable(local_configs)
 
     # If using a virtual environment, upgrade pip before installing dependencies.
-    if local_configs.force_create_venv:
+    if local_configs.use_venv:
         ensure_pip_is_up_to_date(python_exec, env=env)
 
     # Construct package list
@@ -108,7 +108,7 @@ def install_pip_requirements_for_sandbox(
         pip_cmd.append("--upgrade")
     pip_cmd += packages
 
-    if user_install_if_no_venv and not local_configs.force_create_venv:
+    if user_install_if_no_venv and not local_configs.use_venv:
         pip_cmd.append("--user")
 
     run_subprocess(pip_cmd, env=env, fail_msg=f"Failed to install packages: {', '.join(packages)}")
@@ -171,3 +171,30 @@ def add_imports_and_pydantic_schemas_for_args(args_json_schema: dict) -> str:
     )
     result = parser.parse()
     return result
+
+
+def prepare_local_sandbox(
+    local_cfg: LocalSandboxConfig,
+    env: Dict[str, str],
+    force_recreate: bool = False,
+) -> None:
+    """
+    Ensure the sandbox virtual-env is freshly created and that
+    requirements are installed.  Uses your existing helpers.
+    """
+    sandbox_dir = os.path.expanduser(local_cfg.sandbox_dir)
+    venv_path = os.path.join(sandbox_dir, local_cfg.venv_name)
+
+    create_venv_for_local_sandbox(
+        sandbox_dir_path=sandbox_dir,
+        venv_path=venv_path,
+        env=env,
+        force_recreate=force_recreate,
+    )
+
+    install_pip_requirements_for_sandbox(
+        local_cfg,
+        upgrade=True,
+        user_install_if_no_venv=False,
+        env=env,
+    )